Раньше сверка писала проблемы только в базу — человек о них не узнавал. Теперь каждая находка уходит событием service.down через маршрутизацию ядра, а восстановление — одним сообщением «Все службы в порядке». Проверено: обычный прогон молчит (0 проблем), имитация падения дала тревогу в боте Инфраструктура, после удаления имитации снова 0.
219 lines
8.9 KiB
Python
219 lines
8.9 KiB
Python
"""Ночная сверка: проверяем реальность и чиним безопасное.
|
|
|
|
Каждое действие пишется в журнал `actions` — с типом, целью и тем,
|
|
как откатить. Опасное (удаление, смена настроек) не делаем автоматически.
|
|
"""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
|
|
import psycopg2
|
|
|
|
DSN = os.environ.get("CORE_DSN",
|
|
"dbname=hub_core user=hub_core password=hub_core host=192.168.1.151 client_encoding=utf8")
|
|
STATE = "/var/lib/hub-core/reconcile.json"
|
|
|
|
# проверяемая служба → (URL, контейнер, хост CT)
|
|
CHECKS = [
|
|
("authentik", "http://192.168.1.114:9000/-/health/live/", None, 114),
|
|
("jellyfin", "http://192.168.1.103:8096/health", "jellyfin", 103),
|
|
("paperless", "http://192.168.1.105:8000/api/", "paperless", 105),
|
|
("homeassistant", "http://192.168.1.106:8123/manifest.json", "homeassistant", 106),
|
|
("uptime-kuma", "http://192.168.1.107:3001/", "uptime_kuma", 107),
|
|
("n8n", "http://192.168.1.108:5680/healthz", "n8n", 108),
|
|
("nextcloud", "http://192.168.1.109:8080/status.php", "nextcloud", 109),
|
|
("vaultwarden", "http://192.168.1.109:8081/alive", "vaultwarden", 109),
|
|
("sonarr", "http://192.168.1.117:8989/ping", "sonarr", 117),
|
|
("radarr", "http://192.168.1.118:7878/ping", "radarr", 118),
|
|
("prowlarr", "http://192.168.1.119:9696/ping", "prowlarr", 119),
|
|
("bazarr", "http://192.168.1.120:6767/", "bazarr", 120),
|
|
("seerr", "http://192.168.1.121:5055/api/v1/settings/public", "seerr", 121),
|
|
("navidrome", "http://192.168.1.125:4533/ping", "navidrome", 125),
|
|
("linkding", "http://192.168.1.127:3000/", "linkding", 127),
|
|
("prometheus", "http://192.168.1.134:9090/-/healthy", "prometheus", 134),
|
|
("qBittorrent", "http://192.168.1.123:8085/", "qbittorrent", 123),
|
|
("frigate", "http://192.168.1.126:5000/api/version", "frigate", 126),
|
|
]
|
|
|
|
|
|
def db():
|
|
con = psycopg2.connect(DSN, connect_timeout=10)
|
|
con.autocommit = True
|
|
return con
|
|
|
|
|
|
def ensure_journal():
|
|
with db() as con, con.cursor() as cur:
|
|
cur.execute("""
|
|
create table if not exists actions (
|
|
id bigserial primary key,
|
|
at timestamptz not null default now(),
|
|
kind text not null,
|
|
target text not null,
|
|
detail text,
|
|
status text not null default 'done',
|
|
rollback text
|
|
)""")
|
|
cur.execute("""
|
|
create table if not exists problems (
|
|
id bigserial primary key,
|
|
at timestamptz not null default now(),
|
|
check_name text not null,
|
|
state text not null,
|
|
detail text,
|
|
resolved_at timestamptz
|
|
)""")
|
|
|
|
|
|
def journal(kind, target, detail, status="done", rollback=None):
|
|
with db() as con, con.cursor() as cur:
|
|
cur.execute("""insert into actions (kind, target, detail, status, rollback)
|
|
values (%s,%s,%s,%s,%s) returning id""",
|
|
(kind, target, detail, status, rollback))
|
|
return cur.fetchone()[0]
|
|
|
|
|
|
def load_state():
|
|
try:
|
|
return json.load(open("/var/lib/hub-core/reconcile-state.json", encoding="utf-8"))
|
|
except OSError:
|
|
return {}
|
|
|
|
|
|
def save_state(state):
|
|
os.makedirs("/var/lib/hub-core", exist_ok=True)
|
|
json.dump(state, open("/var/lib/hub-core/reconcile-state.json", "w", encoding="utf-8"),
|
|
ensure_ascii=False)
|
|
|
|
|
|
def probe(url, timeout=8):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "hub-reconcile"})
|
|
with urllib.request.urlopen(req, timeout=timeout) as r:
|
|
return True, r.status
|
|
except urllib.error.HTTPError as e:
|
|
# 401/403/404 — служба жива, просто закрыта
|
|
return e.code < 500, e.code
|
|
except Exception as e:
|
|
return False, str(e)[:80]
|
|
|
|
|
|
def container_state(ct, name):
|
|
try:
|
|
out = subprocess.run(["pct", "exec", str(ct), "--", "docker", "inspect", name,
|
|
"--format", "{{.State.Status}}"],
|
|
capture_output=True, text=True, timeout=25)
|
|
return out.stdout.strip() or "нет"
|
|
except Exception:
|
|
return "нет"
|
|
|
|
|
|
def fix(ct, name):
|
|
"""Безопасное исправление: поднять контейнер, если он остановлен."""
|
|
try:
|
|
r = subprocess.run(["pct", "exec", str(ct), "--", "docker", "start", name],
|
|
capture_output=True, text=True, timeout=60)
|
|
return r.returncode == 0, (r.stderr or r.stdout).strip()[:120]
|
|
except Exception as e:
|
|
return False, str(e)[:100]
|
|
|
|
|
|
CORE = os.environ.get("CORE_URL", "http://127.0.0.1:9120")
|
|
|
|
|
|
def notify_core(etype, title, detail):
|
|
"""Отправить событие в ядро — оно разошлёт по маршрутам (инфра)."""
|
|
body = json.dumps({
|
|
"source": "reconcile",
|
|
"type": etype,
|
|
"entity": {"kind": "проверка", "key": f"reconcile:{title}", "title": title},
|
|
"payload": {"detail": detail},
|
|
}, ensure_ascii=False).encode()
|
|
req = urllib.request.Request(f"{CORE}/event", data=body, method="POST",
|
|
headers={"Content-Type": "application/json"})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=15) as r:
|
|
return r.status in (200, 201)
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def main():
|
|
ensure_journal()
|
|
problems, fixed, checked = [], 0, 0
|
|
report = []
|
|
|
|
for name, url, container, ct in CHECKS:
|
|
alive, code = probe(url)
|
|
checked += 1
|
|
if alive:
|
|
continue
|
|
|
|
state = container_state(ct, container) if container else "?"
|
|
detail = f"не отвечает ({code}); контейнер: {state}"
|
|
problems.append((name, detail))
|
|
|
|
# чиним только если контейнер реально не работает
|
|
if container and state in ("exited", "created", "dead", "нет"):
|
|
ok, out = fix(ct, container)
|
|
if ok:
|
|
alive2, code2 = probe(url, timeout=15)
|
|
journal("restart", f"{ct}:{container}",
|
|
f"был {state}, теперь {'отвечает' if alive2 else 'всё ещё молчит'} ({code2})",
|
|
status="done" if alive2 else "failed",
|
|
rollback=f"pct exec {ct} -- docker stop {container}")
|
|
fixed += 1 if alive2 else 0
|
|
report.append(f"{name}: поднят контейнер → {'ок' if alive2 else 'не помогло'}")
|
|
else:
|
|
journal("restart", f"{ct}:{container}", f"ошибка запуска: {out}", status="failed")
|
|
report.append(f"{name}: не смог поднять ({out})")
|
|
else:
|
|
report.append(f"{name}: {detail}")
|
|
|
|
# фиксируем проблемы в базе
|
|
with db() as con, con.cursor() as cur:
|
|
for name, detail in problems:
|
|
cur.execute("""select id from problems where check_name=%s and resolved_at is null""", (name,))
|
|
if cur.fetchone():
|
|
continue
|
|
cur.execute("""insert into problems (check_name, state, detail) values (%s,%s,%s)""",
|
|
(name, "down", detail))
|
|
# закрываем те, что уже не проблема
|
|
names_down = [p[0] for p in problems]
|
|
if names_down:
|
|
cur.execute("""update problems set resolved_at = now()
|
|
where resolved_at is null and check_name <> all(%s)""", (names_down,))
|
|
else:
|
|
cur.execute("update problems set resolved_at = now() where resolved_at is null")
|
|
cur.execute("select count(*) from problems where resolved_at is null")
|
|
open_now = cur.fetchone()[0]
|
|
|
|
# сообщаем о проблемах и о восстановлении
|
|
for name, detail in problems[:5]:
|
|
notify_core("service.down", f"{name} недоступен", detail)
|
|
if not problems and open_now == 0:
|
|
state = load_state()
|
|
if state.get("had_problems"):
|
|
notify_core("service.up", "Все службы в порядке",
|
|
f"проверено {checked}, проблем нет")
|
|
state["had_problems"] = False
|
|
save_state(state)
|
|
elif problems:
|
|
state = load_state()
|
|
state["had_problems"] = True
|
|
save_state(state)
|
|
|
|
print(f"[{datetime.now(timezone.utc).isoformat(timespec='seconds')}] "
|
|
f"проверено {checked}, проблем {len(problems)}, исправлено {fixed}, открытых {open_now}")
|
|
for line in report:
|
|
print(" -", line)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|