293 lines
12 KiB
Python
Executable File
293 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Регрессионная самопроверка платформы Buryans Hub.
|
|
|
|
Один прогон отвечает на вопрос «что мы сломали» после изменений:
|
|
службы, боты, чекеры, отчёты, реестр, задачи, бэкапы, свежесть LLM-контуров.
|
|
|
|
Запуск:
|
|
python3 selfcheck_platform.py # таблица в терминал
|
|
python3 selfcheck_platform.py --json # + JSON-отчёт в /var/log/hub
|
|
|
|
Код возврата: 0 — всё зелёное, 1 — есть провалы.
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
from datetime import date
|
|
|
|
HUB = "http://127.0.0.1:9120"
|
|
BOTS = "http://127.0.0.1:9110"
|
|
REPO = "/root/buryan-infra"
|
|
APPROVALS_PY = "/opt/music-tools/venv/bin/python"
|
|
WORK_DB = "/opt/work-hub/work.db"
|
|
OUT_JSON = "/var/log/hub/selfcheck-%s.json" % date.today().isoformat()
|
|
|
|
results = []
|
|
|
|
|
|
def add(name, ok, detail=""):
|
|
results.append({"check": name, "ok": bool(ok), "detail": str(detail)[:300]})
|
|
return ok
|
|
|
|
|
|
def sh(cmd, timeout=120):
|
|
try:
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
|
return (r.stdout or "") + (r.stderr or "")
|
|
except Exception as e:
|
|
return "ERR " + str(e)[:100]
|
|
|
|
|
|
def get(url, timeout=20, retries=2):
|
|
last = None
|
|
for i in range(retries):
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=timeout) as r:
|
|
return json.loads(r.read())
|
|
except Exception as e:
|
|
last = e
|
|
time.sleep(2)
|
|
raise last
|
|
|
|
|
|
def check_units():
|
|
n = sh("systemctl --failed --no-legend --no-pager | wc -l").strip()
|
|
add("упавшие юниты", n == "0", "count=%s" % n)
|
|
|
|
|
|
def check_services():
|
|
try:
|
|
d = get(HUB + "/status.json", timeout=30)
|
|
except Exception as e:
|
|
return add("службы (status.json)", False, str(e)[:100])
|
|
svc = d.get("services") or []
|
|
down = [s.get("name") for s in svc if not s.get("up")]
|
|
add("службы", len(svc) > 60 and not down,
|
|
"всего=%d лежит=%s" % (len(svc), ",".join(down[:5]) or "0"))
|
|
|
|
|
|
def check_bots():
|
|
try:
|
|
d = get(BOTS + "/health", timeout=15)
|
|
except Exception as e:
|
|
return add("боты", False, str(e)[:100])
|
|
add("боты", d.get("ok") and not d.get("token_failures") and (d.get("bots_count") or 0) >= 8,
|
|
"count=%s failures=%s" % (d.get("bots_count"), d.get("token_failures")))
|
|
|
|
|
|
def check_matrix_sso():
|
|
out = sh("/opt/work-hub/venv/bin/python %s/ops/scripts/check_matrix_sso.py" % REPO, timeout=90)
|
|
oks = out.count("(ok)")
|
|
add("Matrix/SSO чекер", oks == 4 and "СБОЙ" not in out, "ok=%d" % oks)
|
|
|
|
|
|
def check_ollama():
|
|
out = sh("/opt/work-hub/venv/bin/python %s/ops/scripts/check_ollama.py" % REPO, timeout=120)
|
|
add("Ollama чекер", "в порядке" in out, out.strip().splitlines()[-1][:120] if out.strip() else "нет вывода")
|
|
|
|
|
|
def check_n8n():
|
|
"""n8n выведен 16.11.2026: ожидаем 0 активных воркфлоу и остановленный контейнер."""
|
|
sup = open("/root/.pg-superuser-pass").read().strip()
|
|
out = sh('PGPASSWORD=%s psql -w -h 192.168.1.151 -U postgres -d n8n -t -A '
|
|
'-c "SELECT count(*) FROM workflow_entity WHERE active=true;"' % sup)
|
|
n = out.strip()
|
|
names = sh("pct exec 108 -- docker ps --format '{{.Names}}' 2>/dev/null").split()
|
|
running = "n8n" in names
|
|
add("n8n (интеграции)", n.isdigit() and int(n) >= 1 and running,
|
|
"активных воркфлоу=%s контейнер=%s" % (n, "запущен" if running else "остановлен"))
|
|
|
|
|
|
def check_digests():
|
|
d = date.today().isoformat()
|
|
logp = "/var/log/hub/log-digest-%s.md" % d
|
|
morp = "/var/log/hub/morning-digest-%s.md" % d
|
|
ok_log = os.path.exists(logp) and os.path.getsize(logp) > 50
|
|
add("ночной разбор журналов", ok_log, logp)
|
|
ok_mor = os.path.exists(morp) and os.path.getsize(morp) > 50
|
|
add("утренняя сводка", ok_mor, morp)
|
|
|
|
|
|
def check_reports():
|
|
pats = {
|
|
"weekly": REPO + "/ops/weekly/WEEKLY-*.md",
|
|
"trust": REPO + "/ops/trust/report-*.md",
|
|
"research": REPO + "/ops/evolution/research-*.md",
|
|
}
|
|
for name, pat in pats.items():
|
|
files = sorted(glob.glob(pat))
|
|
if not files:
|
|
add("отчёт %s" % name, False, "нет файлов")
|
|
continue
|
|
txt = open(files[-1], encoding="utf-8", errors="replace").read()
|
|
add("отчёт %s" % name, "## Главное" in txt, os.path.basename(files[-1]))
|
|
|
|
|
|
def check_registry():
|
|
out = sh("/usr/bin/python3 %s/ops/scripts/registry_drift.py" % REPO, timeout=300)
|
|
add("реестр автоматизаций", "без изменений" in out, out.strip().splitlines()[-1][:120] if out.strip() else "")
|
|
|
|
|
|
def check_cron_dots():
|
|
dots = [f for f in os.listdir("/etc/cron.d") if "." in f]
|
|
add("cron.d без мёртвых файлов", not dots, ",".join(dots[:5]))
|
|
|
|
|
|
def check_tasks():
|
|
try:
|
|
c = sqlite3.connect("file:%s?mode=ro" % WORK_DB, uri=True)
|
|
rows = c.execute("select count(*) from items where status != 'done'").fetchone()[0]
|
|
c.close()
|
|
add("база задач work", True, "открытых=%d" % rows)
|
|
except Exception as e:
|
|
add("база задач work", False, str(e)[:100])
|
|
|
|
|
|
def check_approvals():
|
|
out = sh("%s /opt/hub-agents/approvals.py list" % APPROVALS_PY, timeout=60)
|
|
ok = "заявок нет" in out or out.strip().startswith("#")
|
|
add("очередь заявок", ok, out.strip().splitlines()[0][:100] if out.strip() else "пусто")
|
|
|
|
|
|
def check_hermes():
|
|
"""Задания живут в Postgres (hermes.jobs); SQLite — legacy-копия."""
|
|
act = sh("systemctl is-active hermes").strip()
|
|
n = sh("grep -cE '^[a-z0-9_-]+:' /opt/hermes/jobs.yaml").strip()
|
|
sup = open("/root/.pg-superuser-pass").read().strip()
|
|
pg = sh("PGPASSWORD=%s psql -w -h 192.168.1.151 -U postgres -d hermes -t -A "
|
|
"-c 'SELECT count(*) FROM jobs'" % sup).strip()
|
|
add("Hermes", act == "active" and n.isdigit() and int(n) >= 50,
|
|
"active=%s заданий=%s записей=%s" % (act, n, pg))
|
|
# все задания должны иметь запись о запуске
|
|
if pg.isdigit() and n.isdigit():
|
|
add("Hermes: все задания запускались", int(pg) >= int(n), "записей=%s заданий=%s" % (pg, n))
|
|
|
|
|
|
def check_errors():
|
|
out = sh("journalctl --since '-6h' -p err --no-pager 2>/dev/null | "
|
|
"grep -vE 'ssl.c:1012|handshake|hub-second-contour|hub-snapshot' | wc -l").strip()
|
|
n = int(out) if out.isdigit() else -1
|
|
add("ошибки журнала (6 ч)", 0 <= n <= 20, "count=%s" % n)
|
|
|
|
|
|
def check_disk():
|
|
out = sh("df -P / /mnt/backups /mnt/archive 2>/dev/null | tail -3 | awk '{print $5\" \"$6}'")
|
|
worst = 0
|
|
for line in out.splitlines():
|
|
m = re.search(r"(\d+)%", line)
|
|
if m:
|
|
worst = max(worst, int(m.group(1)))
|
|
add("диски (макс. занятость)", worst < 90, "%d%%" % worst)
|
|
|
|
|
|
|
|
def check_lvm():
|
|
"""Свободное место в VG и старые LVM-снимки.
|
|
|
|
Снимки от ручных работ (pre-*) висят неделями и занимают thin pool.
|
|
Плюс после vzdump остаётся мало места — это риск для следующих бэкапов.
|
|
"""
|
|
import datetime as _dt
|
|
# Для thin-пула «свободно в VG» — не показатель: место живёт в пуле.
|
|
# Смотрим заполненность data и метаданных.
|
|
out = sh("lvs --noheadings -o data_percent,metadata_percent pve/data 2>/dev/null").strip()
|
|
dp = md = -1
|
|
try:
|
|
parts = out.split()
|
|
dp = float(parts[0].replace(",", "."))
|
|
md = float(parts[1].replace(",", "."))
|
|
except Exception:
|
|
pass
|
|
add("LVM: thin pool data", 0 <= dp < 85, "%.1f%% (метаданные %.1f%%)" % (dp, md))
|
|
free = sh("vgs --noheadings -o vg_free --units g pve 2>/dev/null | tr -d ' g'").strip()
|
|
try:
|
|
f = float(free.replace(",", "."))
|
|
except Exception:
|
|
f = -1
|
|
add("LVM: свободно в VG pve", f >= 0.5, "%.1f ГБ" % f if f >= 0 else "нет данных")
|
|
|
|
out = sh("lvs --noheadings -o lv_name,lv_time 2>/dev/null | grep 'snap_'")
|
|
total, old, old_names = 0, 0, []
|
|
for line in out.splitlines():
|
|
parts = line.split()
|
|
if len(parts) < 2:
|
|
continue
|
|
total += 1
|
|
m = re.search(r"(\d{4}-\d{2}-\d{2})", line)
|
|
if not m:
|
|
continue
|
|
try:
|
|
d = _dt.date.fromisoformat(m.group(1))
|
|
except Exception:
|
|
continue
|
|
if (date.today() - d).days > 14:
|
|
old += 1
|
|
old_names.append(parts[0])
|
|
add("LVM: старые снимки (>14 дн.)", old == 0,
|
|
"всего=%d старых=%d %s" % (total, old, ",".join(old_names[:3])))
|
|
|
|
|
|
def check_pg_conn():
|
|
"""Соединения Postgres: порог 80% от max_connections (пункт 4)."""
|
|
import subprocess as _sp
|
|
env = dict(os.environ, PGPASSWORD=open("/root/.pg-superuser-pass").read().strip())
|
|
def q(sql):
|
|
r = _sp.run(["psql", "-w", "-h", "192.168.1.151", "-U", "postgres",
|
|
"-d", "postgres", "-t", "-A", "-c", sql],
|
|
capture_output=True, text=True, timeout=20, env=env)
|
|
return (r.stdout or "").strip()
|
|
try:
|
|
limit = int(q("SHOW max_connections;") or 0)
|
|
total = int(q("SELECT count(*) FROM pg_stat_activity;") or 0)
|
|
except Exception as e:
|
|
return add("Postgres соединения", False, str(e)[:80])
|
|
pct = 100.0 * total / limit if limit else 100
|
|
add("Postgres соединения", pct < 80, "%d/%d (%.0f%%)" % (total, limit, pct))
|
|
|
|
|
|
def check_tool_gateway():
|
|
"""Tool Gateway: служба и /health."""
|
|
active = sh("systemctl is-active tool-gateway 2>/dev/null").strip()
|
|
health = sh("curl -s -m 5 http://127.0.0.1:9140/health 2>/dev/null").strip()
|
|
ok = active == "active" and '"ok": true' in health
|
|
add("Tool Gateway", ok, "active=%s health=%s" % (active, health[:60]))
|
|
|
|
def main():
|
|
print("Самопроверка платформы", date.today().isoformat())
|
|
checks = [check_units, check_services, check_bots, check_matrix_sso, check_ollama,
|
|
check_n8n, check_digests, check_reports, check_registry, check_cron_dots,
|
|
check_tasks, check_approvals, check_hermes, check_errors, check_disk,
|
|
check_lvm, check_pg_conn, check_tool_gateway]
|
|
for fn in checks:
|
|
try:
|
|
fn()
|
|
except Exception as e:
|
|
add(fn.__name__, False, "исключение: %s" % str(e)[:120])
|
|
|
|
print()
|
|
fails = 0
|
|
for r in results:
|
|
mark = "✅" if r["ok"] else "❌"
|
|
print("%s %-28s %s" % (mark, r["check"], r["detail"]))
|
|
if not r["ok"]:
|
|
fails += 1
|
|
print("\nитог: %d проверок, провалов %d" % (len(results), fails))
|
|
try:
|
|
os.makedirs("/var/log/hub", exist_ok=True)
|
|
json.dump({"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "results": results},
|
|
open(OUT_JSON, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
|
print("отчёт:", OUT_JSON)
|
|
except Exception:
|
|
pass
|
|
return 1 if fails else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|