Files
buryan-infra/ops/hub-core/reconcile.py
T
buryan-agent 5e4c033944 Сверка сообщает о проблемах боту инфраструктуры
Раньше сверка писала проблемы только в базу — человек о них не узнавал.
Теперь каждая находка уходит событием service.down через маршрутизацию ядра,
а восстановление — одним сообщением «Все службы в порядке».

Проверено: обычный прогон молчит (0 проблем), имитация падения дала
тревогу в боте Инфраструктура, после удаления имитации снова 0.
2026-10-19 20:18:31 +03:00

219 lines
8.9 KiB
Python

"""Ночная сверка: проверяем реальность и чиним безопасное.
Каждое действие пишется в журнал `actions` — с типом, целью и тем,
как откатить. Опасное (удаление, смена настроек) не делаем автоматически.
"""
import json
import os
import subprocess
import sys
import urllib.error
import urllib.request
from datetime import datetime, timezone
import psycopg2
DSN = os.environ.get("CORE_DSN",
"dbname=hub_core user=hub_core password=hub_core host=192.168.1.151 client_encoding=utf8")
STATE = "/var/lib/hub-core/reconcile.json"
# проверяемая служба → (URL, контейнер, хост CT)
CHECKS = [
("authentik", "http://192.168.1.114:9000/-/health/live/", None, 114),
("jellyfin", "http://192.168.1.103:8096/health", "jellyfin", 103),
("paperless", "http://192.168.1.105:8000/api/", "paperless", 105),
("homeassistant", "http://192.168.1.106:8123/manifest.json", "homeassistant", 106),
("uptime-kuma", "http://192.168.1.107:3001/", "uptime_kuma", 107),
("n8n", "http://192.168.1.108:5680/healthz", "n8n", 108),
("nextcloud", "http://192.168.1.109:8080/status.php", "nextcloud", 109),
("vaultwarden", "http://192.168.1.109:8081/alive", "vaultwarden", 109),
("sonarr", "http://192.168.1.117:8989/ping", "sonarr", 117),
("radarr", "http://192.168.1.118:7878/ping", "radarr", 118),
("prowlarr", "http://192.168.1.119:9696/ping", "prowlarr", 119),
("bazarr", "http://192.168.1.120:6767/", "bazarr", 120),
("seerr", "http://192.168.1.121:5055/api/v1/settings/public", "seerr", 121),
("navidrome", "http://192.168.1.125:4533/ping", "navidrome", 125),
("linkding", "http://192.168.1.127:3000/", "linkding", 127),
("prometheus", "http://192.168.1.134:9090/-/healthy", "prometheus", 134),
("qBittorrent", "http://192.168.1.123:8085/", "qbittorrent", 123),
("frigate", "http://192.168.1.126:5000/api/version", "frigate", 126),
]
def db():
con = psycopg2.connect(DSN, connect_timeout=10)
con.autocommit = True
return con
def ensure_journal():
with db() as con, con.cursor() as cur:
cur.execute("""
create table if not exists actions (
id bigserial primary key,
at timestamptz not null default now(),
kind text not null,
target text not null,
detail text,
status text not null default 'done',
rollback text
)""")
cur.execute("""
create table if not exists problems (
id bigserial primary key,
at timestamptz not null default now(),
check_name text not null,
state text not null,
detail text,
resolved_at timestamptz
)""")
def journal(kind, target, detail, status="done", rollback=None):
with db() as con, con.cursor() as cur:
cur.execute("""insert into actions (kind, target, detail, status, rollback)
values (%s,%s,%s,%s,%s) returning id""",
(kind, target, detail, status, rollback))
return cur.fetchone()[0]
def load_state():
try:
return json.load(open("/var/lib/hub-core/reconcile-state.json", encoding="utf-8"))
except OSError:
return {}
def save_state(state):
os.makedirs("/var/lib/hub-core", exist_ok=True)
json.dump(state, open("/var/lib/hub-core/reconcile-state.json", "w", encoding="utf-8"),
ensure_ascii=False)
def probe(url, timeout=8):
try:
req = urllib.request.Request(url, headers={"User-Agent": "hub-reconcile"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return True, r.status
except urllib.error.HTTPError as e:
# 401/403/404 — служба жива, просто закрыта
return e.code < 500, e.code
except Exception as e:
return False, str(e)[:80]
def container_state(ct, name):
try:
out = subprocess.run(["pct", "exec", str(ct), "--", "docker", "inspect", name,
"--format", "{{.State.Status}}"],
capture_output=True, text=True, timeout=25)
return out.stdout.strip() or "нет"
except Exception:
return "нет"
def fix(ct, name):
"""Безопасное исправление: поднять контейнер, если он остановлен."""
try:
r = subprocess.run(["pct", "exec", str(ct), "--", "docker", "start", name],
capture_output=True, text=True, timeout=60)
return r.returncode == 0, (r.stderr or r.stdout).strip()[:120]
except Exception as e:
return False, str(e)[:100]
CORE = os.environ.get("CORE_URL", "http://127.0.0.1:9120")
def notify_core(etype, title, detail):
"""Отправить событие в ядро — оно разошлёт по маршрутам (инфра)."""
body = json.dumps({
"source": "reconcile",
"type": etype,
"entity": {"kind": "проверка", "key": f"reconcile:{title}", "title": title},
"payload": {"detail": detail},
}, ensure_ascii=False).encode()
req = urllib.request.Request(f"{CORE}/event", data=body, method="POST",
headers={"Content-Type": "application/json"})
try:
with urllib.request.urlopen(req, timeout=15) as r:
return r.status in (200, 201)
except Exception:
return False
def main():
ensure_journal()
problems, fixed, checked = [], 0, 0
report = []
for name, url, container, ct in CHECKS:
alive, code = probe(url)
checked += 1
if alive:
continue
state = container_state(ct, container) if container else "?"
detail = f"не отвечает ({code}); контейнер: {state}"
problems.append((name, detail))
# чиним только если контейнер реально не работает
if container and state in ("exited", "created", "dead", "нет"):
ok, out = fix(ct, container)
if ok:
alive2, code2 = probe(url, timeout=15)
journal("restart", f"{ct}:{container}",
f"был {state}, теперь {'отвечает' if alive2 else 'всё ещё молчит'} ({code2})",
status="done" if alive2 else "failed",
rollback=f"pct exec {ct} -- docker stop {container}")
fixed += 1 if alive2 else 0
report.append(f"{name}: поднят контейнер → {'ок' if alive2 else 'не помогло'}")
else:
journal("restart", f"{ct}:{container}", f"ошибка запуска: {out}", status="failed")
report.append(f"{name}: не смог поднять ({out})")
else:
report.append(f"{name}: {detail}")
# фиксируем проблемы в базе
with db() as con, con.cursor() as cur:
for name, detail in problems:
cur.execute("""select id from problems where check_name=%s and resolved_at is null""", (name,))
if cur.fetchone():
continue
cur.execute("""insert into problems (check_name, state, detail) values (%s,%s,%s)""",
(name, "down", detail))
# закрываем те, что уже не проблема
names_down = [p[0] for p in problems]
if names_down:
cur.execute("""update problems set resolved_at = now()
where resolved_at is null and check_name <> all(%s)""", (names_down,))
else:
cur.execute("update problems set resolved_at = now() where resolved_at is null")
cur.execute("select count(*) from problems where resolved_at is null")
open_now = cur.fetchone()[0]
# сообщаем о проблемах и о восстановлении
for name, detail in problems[:5]:
notify_core("service.down", f"{name} недоступен", detail)
if not problems and open_now == 0:
state = load_state()
if state.get("had_problems"):
notify_core("service.up", "Все службы в порядке",
f"проверено {checked}, проблем нет")
state["had_problems"] = False
save_state(state)
elif problems:
state = load_state()
state["had_problems"] = True
save_state(state)
print(f"[{datetime.now(timezone.utc).isoformat(timespec='seconds')}] "
f"проверено {checked}, проблем {len(problems)}, исправлено {fixed}, открытых {open_now}")
for line in report:
print(" -", line)
return 0
if __name__ == "__main__":
sys.exit(main())