Files
buryan-infra/ops/scripts/selfcheck_platform.py
T

210 lines
7.6 KiB
Python
Executable File

#!/usr/bin/env python3
"""Регрессионная самопроверка платформы Buryans Hub.
Один прогон отвечает на вопрос «что мы сломали» после изменений:
службы, боты, чекеры, отчёты, реестр, задачи, бэкапы, свежесть LLM-контуров.
Запуск:
python3 selfcheck_platform.py # таблица в терминал
python3 selfcheck_platform.py --json # + JSON-отчёт в /var/log/hub
Код возврата: 0 — всё зелёное, 1 — есть провалы.
"""
import glob
import json
import os
import re
import sqlite3
import subprocess
import sys
import time
import urllib.request
from datetime import date
HUB = "http://127.0.0.1:9120"
BOTS = "http://127.0.0.1:9110"
REPO = "/root/buryan-infra"
APPROVALS_PY = "/opt/music-tools/venv/bin/python"
WORK_DB = "/opt/work-hub/work.db"
OUT_JSON = "/var/log/hub/selfcheck-%s.json" % date.today().isoformat()
results = []
def add(name, ok, detail=""):
results.append({"check": name, "ok": bool(ok), "detail": str(detail)[:300]})
return ok
def sh(cmd, timeout=120):
try:
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
return (r.stdout or "") + (r.stderr or "")
except Exception as e:
return "ERR " + str(e)[:100]
def get(url, timeout=20, retries=2):
last = None
for i in range(retries):
try:
with urllib.request.urlopen(url, timeout=timeout) as r:
return json.loads(r.read())
except Exception as e:
last = e
time.sleep(2)
raise last
def check_units():
n = sh("systemctl --failed --no-legend --no-pager | wc -l").strip()
add("упавшие юниты", n == "0", "count=%s" % n)
def check_services():
try:
d = get(HUB + "/status.json", timeout=30)
except Exception as e:
return add("службы (status.json)", False, str(e)[:100])
svc = d.get("services") or []
down = [s.get("name") for s in svc if not s.get("up")]
add("службы", len(svc) > 60 and not down,
"всего=%d лежит=%s" % (len(svc), ",".join(down[:5]) or "0"))
def check_bots():
try:
d = get(BOTS + "/health", timeout=15)
except Exception as e:
return add("боты", False, str(e)[:100])
add("боты", d.get("ok") and not d.get("token_failures") and (d.get("bots_count") or 0) >= 8,
"count=%s failures=%s" % (d.get("bots_count"), d.get("token_failures")))
def check_matrix_sso():
out = sh("/opt/work-hub/venv/bin/python %s/ops/scripts/check_matrix_sso.py" % REPO, timeout=90)
oks = out.count("(ok)")
add("Matrix/SSO чекер", oks == 4 and "СБОЙ" not in out, "ok=%d" % oks)
def check_ollama():
out = sh("/opt/work-hub/venv/bin/python %s/ops/scripts/check_ollama.py" % REPO, timeout=120)
add("Ollama чекер", "в порядке" in out, out.strip().splitlines()[-1][:120] if out.strip() else "нет вывода")
def check_n8n():
sup = open("/root/.pg-superuser-pass").read().strip()
q = ("SELECT count(*) FROM workflow_entity WHERE active=true;")
out = sh('PGPASSWORD=%s psql -w -h 192.168.1.151 -U postgres -d n8n -t -A -c "%s"' % (sup, q))
n = out.strip()
add("n8n активные воркфлоу", n.isdigit() and int(n) >= 1, "count=%s" % n)
def check_digests():
d = date.today().isoformat()
logp = "/var/log/hub/log-digest-%s.md" % d
morp = "/var/log/hub/morning-digest-%s.md" % d
ok_log = os.path.exists(logp) and os.path.getsize(logp) > 50
add("ночной разбор журналов", ok_log, logp)
ok_mor = os.path.exists(morp) and os.path.getsize(morp) > 50
add("утренняя сводка", ok_mor, morp)
def check_reports():
pats = {
"weekly": REPO + "/ops/weekly/WEEKLY-*.md",
"trust": REPO + "/ops/trust/report-*.md",
"research": REPO + "/ops/evolution/research-*.md",
}
for name, pat in pats.items():
files = sorted(glob.glob(pat))
if not files:
add("отчёт %s" % name, False, "нет файлов")
continue
txt = open(files[-1], encoding="utf-8", errors="replace").read()
add("отчёт %s" % name, "## Главное" in txt, os.path.basename(files[-1]))
def check_registry():
out = sh("/usr/bin/python3 %s/ops/scripts/registry_drift.py" % REPO, timeout=300)
add("реестр автоматизаций", "без изменений" in out, out.strip().splitlines()[-1][:120] if out.strip() else "")
def check_cron_dots():
dots = [f for f in os.listdir("/etc/cron.d") if "." in f]
add("cron.d без мёртвых файлов", not dots, ",".join(dots[:5]))
def check_tasks():
try:
c = sqlite3.connect("file:%s?mode=ro" % WORK_DB, uri=True)
rows = c.execute("select count(*) from items where status != 'done'").fetchone()[0]
c.close()
add("база задач work", True, "открытых=%d" % rows)
except Exception as e:
add("база задач work", False, str(e)[:100])
def check_approvals():
out = sh("%s /opt/hub-agents/approvals.py list" % APPROVALS_PY, timeout=60)
ok = "заявок нет" in out or out.strip().startswith("#")
add("очередь заявок", ok, out.strip().splitlines()[0][:100] if out.strip() else "пусто")
def check_hermes():
act = sh("systemctl is-active hermes").strip()
jobs = sh("grep -cE '^[a-z0-9_-]+:' %s/../hermes/jobs.yaml" % REPO).strip() \
if os.path.exists("/opt/hermes/jobs.yaml") else "?"
n = sh("grep -cE '^[a-z0-9_-]+:' /opt/hermes/jobs.yaml").strip()
add("Hermes", act == "active" and n.isdigit() and int(n) >= 50, "active=%s заданий=%s" % (act, n))
def check_errors():
out = sh("journalctl --since '-6h' -p err --no-pager 2>/dev/null | "
"grep -vE 'ssl.c:1012|handshake|hub-second-contour|hub-snapshot' | wc -l").strip()
n = int(out) if out.isdigit() else -1
add("ошибки журнала (6 ч)", 0 <= n <= 20, "count=%s" % n)
def check_disk():
out = sh("df -P / /mnt/backups /mnt/archive 2>/dev/null | tail -3 | awk '{print $5\" \"$6}'")
worst = 0
for line in out.splitlines():
m = re.search(r"(\d+)%", line)
if m:
worst = max(worst, int(m.group(1)))
add("диски (макс. занятость)", worst < 90, "%d%%" % worst)
def main():
print("Самопроверка платформы", date.today().isoformat())
checks = [check_units, check_services, check_bots, check_matrix_sso, check_ollama,
check_n8n, check_digests, check_reports, check_registry, check_cron_dots,
check_tasks, check_approvals, check_hermes, check_errors, check_disk]
for fn in checks:
try:
fn()
except Exception as e:
add(fn.__name__, False, "исключение: %s" % str(e)[:120])
print()
fails = 0
for r in results:
mark = "✅" if r["ok"] else "❌"
print("%s %-28s %s" % (mark, r["check"], r["detail"]))
if not r["ok"]:
fails += 1
print("\nитог: %d проверок, провалов %d" % (len(results), fails))
try:
os.makedirs("/var/log/hub", exist_ok=True)
json.dump({"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "results": results},
open(OUT_JSON, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
print("отчёт:", OUT_JSON)
except Exception:
pass
return 1 if fails else 0
if __name__ == "__main__":
sys.exit(main())