240 lines
9.3 KiB
Python
Executable File
240 lines
9.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Единая утренняя сводка (пункт 1, 16.11.2026).
|
||
|
||
Заменяет собой разрозненные утренние сообщения: состояние служб, бэкапы,
|
||
ночной разбор журналов, задачи владельца, заявки на подтверждение.
|
||
Одно сообщение в ALERT в 08:00.
|
||
|
||
Источники:
|
||
* /status.json хаба — службы;
|
||
* systemctl --failed — упавшие юниты;
|
||
* /var/lib/vz/dump, second_contour.log, ops/audit/snapshot-* — бэкапы;
|
||
* /var/log/hub/log-digest-YYYY-MM-DD.md — ночной разбор (Фаза 3);
|
||
* /opt/work-hub/work.db — открытые задачи;
|
||
* approvals.py list — заявки на подтверждение.
|
||
|
||
Запуск:
|
||
python3 morning_digest.py # собрать и отправить
|
||
python3 morning_digest.py --dry-run # только напечатать
|
||
"""
|
||
import glob
|
||
import json
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
from datetime import date, datetime, timedelta
|
||
|
||
HUB = "http://127.0.0.1:9120"
|
||
NOTIFY = "/opt/buryan-hub/scripts/notify.py"
|
||
WORK_DB = "/opt/work-hub/work.db"
|
||
OUT = "/var/log/hub/morning-digest-%s.md" % date.today().isoformat()
|
||
APPROVALS = "/opt/music-tools/venv/bin/python"
|
||
|
||
|
||
def sh(cmd, timeout=60):
|
||
try:
|
||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
||
return (r.stdout or "").strip()
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def services():
|
||
import urllib.request as u
|
||
d = None
|
||
for attempt in range(2):
|
||
try:
|
||
with u.urlopen(HUB + "/status.json", timeout=30) as r:
|
||
d = json.loads(r.read())
|
||
break
|
||
except Exception:
|
||
if attempt == 0:
|
||
import time as _t
|
||
_t.sleep(3)
|
||
if d is None:
|
||
return "• Службы: хаб не отвечает (2 попытки)", []
|
||
svc = d.get("services") or []
|
||
down = [s.get("name") for s in svc if not s.get("up")]
|
||
if down:
|
||
return "• Службы: %d из %d, лежат: %s" % (len(svc) - len(down), len(svc),
|
||
", ".join(down[:6])), down
|
||
return "• Службы: %d из %d работают" % (len(svc), len(svc)), down
|
||
|
||
|
||
def failed_units():
|
||
n = sh("systemctl --failed --no-legend --no-pager | wc -l")
|
||
if not n or n == "0":
|
||
return "• Упавших юнитов: нет"
|
||
names = sh("systemctl --failed --no-legend --no-pager | awk '{print $2}' | head -4")
|
||
return "• Упавшие юниты: %s (%s)" % (n, names.replace("\n", ", "))
|
||
|
||
|
||
def backups():
|
||
lines = []
|
||
dump = sh("ls -t /var/lib/vz/dump/*.tar.zst 2>/dev/null | head -1")
|
||
if dump:
|
||
age = (datetime.now().timestamp() - os.path.getmtime(dump)) / 3600
|
||
lines.append("дамп CT %s (%.0f ч)" % (os.path.basename(dump).split("-")[2], age))
|
||
sn = sorted(glob.glob("/root/buryan-infra/ops/audit/snapshot-*.md"))
|
||
if sn:
|
||
lines.append("snapshot %s" % os.path.basename(sn[-1]).replace("snapshot-", "").replace(".md", ""))
|
||
ver = sh("grep -h 'ИТОГ: баз' /var/log/buryan-hub/backups/verify-backups.log 2>/dev/null | tail -1")
|
||
if ver:
|
||
lines.append(re.sub(r"^\[[^]]+\]\s*", "", ver))
|
||
if not lines:
|
||
return "• Бэкапы: свежих данных нет"
|
||
return "• Бэкапы: " + "; ".join(lines)
|
||
|
||
|
||
def log_digest():
|
||
p = "/var/log/hub/log-digest-%s.md" % date.today().isoformat()
|
||
if not os.path.exists(p):
|
||
return "• Ночной разбор: отчёта нет"
|
||
txt = open(p, encoding="utf-8").read()
|
||
head = ""
|
||
m = re.search(r"\*\*(.+?)\*\*", txt)
|
||
if m:
|
||
head = m.group(1).strip()
|
||
counts = ""
|
||
m2 = re.search(r"новых:\s*(\d+)\s*\|\s*всплесков:\s*(\d+)\s*\|\s*корреляций:\s*(\d+)", txt)
|
||
if m2:
|
||
counts = "новых %s, всплесков %s, корреляций %s" % m2.groups()
|
||
if not head:
|
||
return "• Ночной разбор: %s" % (counts or "без находок")
|
||
return "• Ночной разбор: %s%s" % (head, (" (" + counts + ")") if counts else "")
|
||
|
||
|
||
def tasks():
|
||
try:
|
||
c = sqlite3.connect("file:%s?mode=ro" % WORK_DB, uri=True)
|
||
rows = c.execute("select id, text, due from items where status != 'done' "
|
||
"order by id desc limit 5").fetchall()
|
||
c.close()
|
||
except Exception as e:
|
||
return "• Задачи: база недоступна (%s)" % str(e)[:40]
|
||
if not rows:
|
||
return "• Задачи: открытых нет"
|
||
items = []
|
||
for i, text, due in rows[:3]:
|
||
t = re.sub(r"\s+", " ", (text or ""))[:60]
|
||
items.append("#%s %s%s" % (i, t, (" (до %s)" % due) if due else ""))
|
||
return "• Задачи (%d): %s" % (len(rows), " · ".join(items))
|
||
|
||
|
||
def approvals():
|
||
out = sh("%s /opt/hub-agents/approvals.py list 2>/dev/null" % APPROVALS)
|
||
if not out or "заявок нет" in out:
|
||
return "• Заявки на подтверждение: нет"
|
||
lines = [ln for ln in out.splitlines() if ln.startswith("#")]
|
||
return "• Заявки: %d (ждёт «одобри N»): %s" % (len(lines),
|
||
"; ".join(lines[:2])[:160])
|
||
|
||
|
||
def errors_night():
|
||
n = sh("journalctl --since '-24h' -p err --no-pager 2>/dev/null | wc -l")
|
||
return "• Ошибок в журнале за сутки: %s" % (n or "?")
|
||
|
||
|
||
|
||
def pg_conn():
|
||
"""Соединения Postgres (см. pg_conn_monitor)."""
|
||
import subprocess as _sp
|
||
try:
|
||
env = dict(os.environ,
|
||
PGPASSWORD=open("/root/.pg-superuser-pass").read().strip())
|
||
def q(sql):
|
||
r = _sp.run(["psql", "-w", "-h", "192.168.1.151", "-U", "postgres",
|
||
"-d", "postgres", "-t", "-A", "-c", sql],
|
||
capture_output=True, text=True, timeout=20, env=env)
|
||
return (r.stdout or "").strip()
|
||
limit = int(q("SHOW max_connections;") or 0)
|
||
total = int(q("SELECT count(*) FROM pg_stat_activity;") or 0)
|
||
pct = 100.0 * total / limit if limit else 0
|
||
return "• Postgres: %d/%d (%.0f%%)" % (total, limit, pct)
|
||
except Exception:
|
||
return "• Postgres: нет данных"
|
||
|
||
|
||
def llm_stats():
|
||
"""LLM за сутки: вызовы, фолбэки, модели."""
|
||
import json as _j
|
||
from datetime import datetime as _dt, timedelta as _td
|
||
p = "/var/log/hub/llm-fallback.log"
|
||
if not os.path.exists(p):
|
||
return "• LLM: журнала нет"
|
||
since = _dt.now() - _td(days=1)
|
||
calls = fallbacks = 0
|
||
models = {}
|
||
try:
|
||
for line in open(p, encoding="utf-8", errors="replace"):
|
||
try:
|
||
r = _j.loads(line)
|
||
except Exception:
|
||
continue
|
||
ts = r.get("ts")
|
||
if not ts:
|
||
continue
|
||
try:
|
||
if _dt.fromisoformat(ts) < since:
|
||
continue
|
||
except Exception:
|
||
continue
|
||
if not r.get("ok"):
|
||
continue
|
||
calls += 1
|
||
models[r.get("model", "?")] = models.get(r.get("model", "?"), 0) + 1
|
||
if r.get("fallback"):
|
||
fallbacks += 1
|
||
except Exception:
|
||
return "• LLM: журнал не читается"
|
||
if not calls:
|
||
return "• LLM за сутки: вызовов не было"
|
||
top = sorted(models.items(), key=lambda x: -x[1])[:2]
|
||
return ("• LLM за сутки: %d вызовов, фолбэк %d (%.0f%%); %s"
|
||
% (calls, fallbacks, 100.0 * fallbacks / calls,
|
||
", ".join("%s×%d" % (k, v) for k, v in top)))
|
||
|
||
|
||
def rag_stats():
|
||
"""Свежесть RAG-индекса."""
|
||
import subprocess as _sp
|
||
try:
|
||
env = dict(os.environ,
|
||
PGPASSWORD=open("/root/.pg-superuser-pass").read().strip())
|
||
r = _sp.run(["psql", "-w", "-h", "192.168.1.151", "-U", "postgres",
|
||
"-d", "rag", "-t", "-A", "-F", "|", "-c",
|
||
"SELECT count(*), to_char(max(updated),'MM-DD HH24:MI') FROM chunks;"],
|
||
capture_output=True, text=True, timeout=20, env=env)
|
||
n, ts = (r.stdout or "").strip().split("|")[:2]
|
||
return "• RAG: %s чанков, обновлён %s" % (n, ts)
|
||
except Exception:
|
||
return "• RAG: нет данных"
|
||
|
||
def main():
|
||
dry = "--dry-run" in sys.argv
|
||
sline, _ = services()
|
||
parts = ["🌅 Утренняя сводка — %s" % date.today().strftime("%d.%m.%Y"), "",
|
||
sline, failed_units(), backups(), log_digest(),
|
||
pg_conn(), llm_stats(), rag_stats(),
|
||
tasks(), approvals(), errors_night()]
|
||
text = "\n".join(parts)
|
||
print(text)
|
||
try:
|
||
os.makedirs("/var/log/hub", exist_ok=True)
|
||
open(OUT, "w", encoding="utf-8").write(text + "\n")
|
||
except Exception:
|
||
pass
|
||
if dry:
|
||
return 0
|
||
r = subprocess.run([sys.executable, NOTIFY, "--priority", "high",
|
||
"--title", "Утренняя сводка", text],
|
||
capture_output=True, text=True, timeout=180)
|
||
print("\nотправка:", (r.stdout or r.stderr).strip()[:100])
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|