"""Ночная сверка: проверяем реальность и чиним безопасное. Каждое действие пишется в журнал `actions` — с типом, целью и тем, как откатить. Опасное (удаление, смена настроек) не делаем автоматически. """ import json import os import subprocess import sys import urllib.error import urllib.request from datetime import datetime, timezone import psycopg2 DSN = os.environ.get("CORE_DSN", "dbname=hub_core user=hub_core password=hub_core host=192.168.1.151 client_encoding=utf8") STATE = "/var/lib/hub-core/reconcile.json" # проверяемая служба → (URL, контейнер, хост CT) CHECKS = [ ("authentik", "http://192.168.1.114:9000/-/health/live/", None, 114), ("jellyfin", "http://192.168.1.103:8096/health", "jellyfin", 103), ("paperless", "http://192.168.1.105:8000/api/", "paperless", 105), ("homeassistant", "http://192.168.1.106:8123/manifest.json", "homeassistant", 106), ("uptime-kuma", "http://192.168.1.107:3001/", "uptime_kuma", 107), ("n8n", "http://192.168.1.108:5680/healthz", "n8n", 108), ("nextcloud", "http://192.168.1.109:8080/status.php", "nextcloud", 109), ("vaultwarden", "http://192.168.1.109:8081/alive", "vaultwarden", 109), ("sonarr", "http://192.168.1.117:8989/ping", "sonarr", 117), ("radarr", "http://192.168.1.118:7878/ping", "radarr", 118), ("prowlarr", "http://192.168.1.119:9696/ping", "prowlarr", 119), ("bazarr", "http://192.168.1.120:6767/", "bazarr", 120), ("seerr", "http://192.168.1.121:5055/api/v1/settings/public", "seerr", 121), ("navidrome", "http://192.168.1.125:4533/ping", "navidrome", 125), ("linkding", "http://192.168.1.127:3000/", "linkding", 127), ("prometheus", "http://192.168.1.134:9090/-/healthy", "prometheus", 134), ("qBittorrent", "http://192.168.1.123:8085/", "qbittorrent", 123), ("frigate", "http://192.168.1.126:5000/api/version", "frigate", 126), ] def db(): con = psycopg2.connect(DSN, connect_timeout=10) con.autocommit = True return con def ensure_journal(): with db() as con, con.cursor() as cur: cur.execute(""" create table if not exists actions ( id bigserial primary key, at timestamptz not null default now(), kind text not null, target text not null, detail text, status text not null default 'done', rollback text )""") cur.execute(""" create table if not exists problems ( id bigserial primary key, at timestamptz not null default now(), check_name text not null, state text not null, detail text, resolved_at timestamptz )""") def journal(kind, target, detail, status="done", rollback=None): with db() as con, con.cursor() as cur: cur.execute("""insert into actions (kind, target, detail, status, rollback) values (%s,%s,%s,%s,%s) returning id""", (kind, target, detail, status, rollback)) return cur.fetchone()[0] def load_state(): try: return json.load(open("/var/lib/hub-core/reconcile-state.json", encoding="utf-8")) except OSError: return {} def save_state(state): os.makedirs("/var/lib/hub-core", exist_ok=True) json.dump(state, open("/var/lib/hub-core/reconcile-state.json", "w", encoding="utf-8"), ensure_ascii=False) STORAGE_LIMIT = 80 # процент, после которого предупреждаем def storage_check(): """Заполненность хранилищ Proxmox: близко к 100% — бэкап не восстановить.""" problems = [] try: out = subprocess.run(["pvesm", "status"], capture_output=True, text=True, timeout=30) except Exception as e: return [(('хранилища', f'не удалось проверить: {str(e)[:60]}'))] for line in out.stdout.splitlines()[1:]: parts = line.split() if len(parts) < 7: continue name, typ, status, total, used, avail, pct = parts[:7] try: p = float(pct.rstrip("%")) except ValueError: continue if p >= STORAGE_LIMIT: problems.append((f"хранилище {name}", f"заполнено {p:.0f}% ({typ}) — восстановление может не пройти")) return problems def probe(url, timeout=8): try: req = urllib.request.Request(url, headers={"User-Agent": "hub-reconcile"}) with urllib.request.urlopen(req, timeout=timeout) as r: return True, r.status except urllib.error.HTTPError as e: # 401/403/404 — служба жива, просто закрыта return e.code < 500, e.code except Exception as e: return False, str(e)[:80] def container_state(ct, name): try: out = subprocess.run(["pct", "exec", str(ct), "--", "docker", "inspect", name, "--format", "{{.State.Status}}"], capture_output=True, text=True, timeout=25) return out.stdout.strip() or "нет" except Exception: return "нет" def fix(ct, name): """Безопасное исправление: поднять контейнер, если он остановлен.""" try: r = subprocess.run(["pct", "exec", str(ct), "--", "docker", "start", name], capture_output=True, text=True, timeout=60) return r.returncode == 0, (r.stderr or r.stdout).strip()[:120] except Exception as e: return False, str(e)[:100] CORE = os.environ.get("CORE_URL", "http://127.0.0.1:9120") def notify_core(etype, title, detail): """Отправить событие в ядро — оно разошлёт по маршрутам (инфра).""" body = json.dumps({ "source": "reconcile", "type": etype, "entity": {"kind": "проверка", "key": f"reconcile:{title}", "title": title}, "payload": {"detail": detail}, }, ensure_ascii=False).encode() req = urllib.request.Request(f"{CORE}/event", data=body, method="POST", headers={"Content-Type": "application/json"}) try: with urllib.request.urlopen(req, timeout=15) as r: return r.status in (200, 201) except Exception: return False def main(): ensure_journal() problems, fixed, checked = [], 0, 0 report = [] for name, url, container, ct in CHECKS: alive, code = probe(url) checked += 1 if alive: continue state = container_state(ct, container) if container else "?" detail = f"не отвечает ({code}); контейнер: {state}" problems.append((name, detail)) # чиним только если контейнер реально не работает if container and state in ("exited", "created", "dead", "нет"): ok, out = fix(ct, container) if ok: alive2, code2 = probe(url, timeout=15) journal("restart", f"{ct}:{container}", f"был {state}, теперь {'отвечает' if alive2 else 'всё ещё молчит'} ({code2})", status="done" if alive2 else "failed", rollback=f"pct exec {ct} -- docker stop {container}") fixed += 1 if alive2 else 0 report.append(f"{name}: поднят контейнер → {'ок' if alive2 else 'не помогло'}") else: journal("restart", f"{ct}:{container}", f"ошибка запуска: {out}", status="failed") report.append(f"{name}: не смог поднять ({out})") else: report.append(f"{name}: {detail}") # заполненность хранилищ for name, detail in storage_check(): problems.append((name, detail)) report.append(f"{name}: {detail}") # фиксируем проблемы в базе with db() as con, con.cursor() as cur: for name, detail in problems: cur.execute("""select id from problems where check_name=%s and resolved_at is null""", (name,)) if cur.fetchone(): continue cur.execute("""insert into problems (check_name, state, detail) values (%s,%s,%s)""", (name, "down", detail)) # закрываем те, что уже не проблема names_down = [p[0] for p in problems] if names_down: cur.execute("""update problems set resolved_at = now() where resolved_at is null and check_name <> all(%s)""", (names_down,)) else: cur.execute("update problems set resolved_at = now() where resolved_at is null") cur.execute("select count(*) from problems where resolved_at is null") open_now = cur.fetchone()[0] # сообщаем о проблемах и о восстановлении for name, detail in problems[:5]: notify_core("service.down", f"{name} недоступен", detail) if not problems and open_now == 0: state = load_state() if state.get("had_problems"): notify_core("service.up", "Все службы в порядке", f"проверено {checked}, проблем нет") state["had_problems"] = False save_state(state) elif problems: state = load_state() state["had_problems"] = True save_state(state) # проверяем, что задачи вообще запускаются try: guard = subprocess.run(["/usr/bin/python3", "/opt/hub-core/cron_guard.py"], capture_output=True, text=True, timeout=60) report.append("сторож расписаний: " + (guard.stdout.strip().splitlines()[-1] if guard.stdout.strip() else "нет вывода")) except Exception as e: report.append(f"сторож расписаний: ошибка {str(e)[:60]}") print(f"[{datetime.now(timezone.utc).isoformat(timespec='seconds')}] " f"проверено {checked}, проблем {len(problems)}, исправлено {fixed}, открытых {open_now}") for line in report: print(" -", line) return 0 if __name__ == "__main__": sys.exit(main())