#!/usr/bin/env python3 """Раннер роя: видит ВСЕ сервисы, помнит ошибки, ищет дубли и мусор. Пишет отчёт в журнал и отдаёт его мозгу. Ничего не меняет без класса A/B. """ import json import os import re import shutil import subprocess import time from datetime import datetime REGISTRY = "/opt/hub-core/services.yaml" JOURNAL = "/var/lib/hub-agents/journal" MEMORY = "/var/lib/hub-agents/memory.json" STATUS = "http://127.0.0.1:9120/status.json" NOTIFY = "http://127.0.0.1:9120/notify" GROUP_TITLES = { "core": "Ядро", "home": "Дом", "media": "Медиа", "knowledge": "Знания", "automation": "Автоматизация", "observability": "Наблюдаемость", "support": "Служебные", "other": "Прочее", } # ---------- реестр ---------- def load_registry(): """Все сервисы из services.yaml: name, group, ct, image, ports, criticality.""" items = [] cur = None try: for line in open(REGISTRY, encoding="utf-8"): if line.startswith(" - key:"): if cur: items.append(cur) cur = {"key": line.split(":", 1)[1].strip().strip('"')} elif cur is not None and line.startswith(" "): k, _, v = line.strip().partition(":") cur[k] = v.strip().strip('"') if cur: items.append(cur) except Exception: return [] return items # ---------- память ---------- def load_memory(): try: return json.load(open(MEMORY, encoding="utf-8")) except Exception: return {"services": {}, "duplicates": [], "solutions": {}} def save_memory(m): os.makedirs(os.path.dirname(MEMORY), exist_ok=True) json.dump(m, open(MEMORY, "w", encoding="utf-8"), ensure_ascii=False, indent=1) def remember(mem, name, ok, reason=""): rec = mem["services"].setdefault(name, {"fails": 0, "first_fail": None, "last_fail": None, "last_ok": None, "reasons": []}) now = datetime.now().isoformat(timespec="seconds") if ok: rec["last_ok"] = now rec["fails"] = 0 else: rec["fails"] = rec.get("fails", 0) + 1 rec["last_fail"] = now if not rec.get("first_fail"): rec["first_fail"] = now if reason: rec["reasons"] = (rec.get("reasons") or [])[-4:] + [reason[:120]] return rec # ---------- проверки ---------- def status_data(): import urllib.request try: with urllib.request.urlopen(STATUS, timeout=25) as r: return json.loads(r.read()) except Exception: return {} def docker_state(name, ct): """Проверяет контейнер напрямую, если его нет в статусе.""" cmd = (["docker", "inspect", name] if ct == 0 else ["pct", "exec", str(ct), "--", "docker", "inspect", name]) r = subprocess.run(cmd + ["--format", "{{.State.Status}}"], capture_output=True, text=True, timeout=20) return (r.stdout or "").strip() or "unknown" def duplicates(services): """Ищет реальные дубли: одинаковый образ в разных CT (двойной экземпляр).""" by_image = {} dups = [] for s in services: img = (s.get("image") or "").split(":")[0] if not img: continue by_image.setdefault(img, []).append(s) for img, items in by_image.items(): if len(items) < 2: continue # служебные образы, которые нормально живут в каждом CT if any(k in img for k in ("promtail", "oauth2-proxy", "exportarr", "redis")): continue cts = {int(i.get("ct") or 0) for i in items} names = [i.get("name") for i in items] # дубль: один и тот же сервис в разных местах (хост + CT) if 0 in cts and len(cts) > 1: dups.append({"type": "double_instance", "image": img, "services": names, "hint": "один экземпляр на хосте, другой в CT"}) elif len(set(names)) == 1 and len(items) > 1: dups.append({"type": "same_name", "image": img, "services": names}) return dups def junk(): """Мусор: большие логи, висячие образы.""" out = {"big_logs": [], "dangling": 0} r = subprocess.run(["docker", "system", "df"], capture_output=True, text=True, timeout=40) m = re.search(r"Dangling Images\s+\d+\s+(\d+)", r.stdout or "") if m: out["dangling"] = int(m.group(1)) return out CERT_HOSTS = [ "authentik.buryans.ru", "git.buryans.ru", "grocy.buryans.ru", "vikunja.buryans.ru", "homebox.buryans.ru", "prometheus.buryans.ru", "media.buryans.ru", "home.buryans.ru", "hub.buryans.ru", "docs.buryans.ru", "finance.buryans.ru", "n8n.buryans.ru", "rss.buryans.ru", "books.buryans.ru", ] def cert_alert(host, days): """Сообщает в Matrix о скором истечении сертификата.""" import json as _json import urllib.request as _rq body = _json.dumps({ "title": "Сертификат истекает", "message": f"{host}: осталось {days} дн. Нужно обновить.", "priority": "high", }).encode() req = _rq.Request("http://127.0.0.1:9120/notify", data=body, method="POST", headers={"Content-Type": "application/json"}) try: _rq.urlopen(req, timeout=15).read() except Exception: pass def certs_days(): """Минимальный остаток дней по сертификатам (None, если проверить нельзя).""" import socket import ssl from datetime import datetime left = [] for host in CERT_HOSTS: try: ctx = ssl.create_default_context() with socket.create_connection((host, 443), timeout=8) as sock: with ctx.wrap_socket(sock, server_hostname=host) as ss: exp = ss.getpeercert().get("notAfter") dt = datetime.strptime(exp, "%b %d %H:%M:%S %Y %Z") left.append((host, (dt - datetime.utcnow()).days)) except Exception: continue if not left: return None, [] left.sort(key=lambda x: x[1]) return left[0][1], left def backup_freshness(max_hours=26): import glob paths = glob.glob("/mnt/backups/dump/*.zst") if not paths: return None, None newest = max(paths, key=os.path.getmtime) return round((time.time() - os.path.getmtime(newest)) / 3600, 1), os.path.basename(newest) def disk_pct(path="/"): try: st = os.statvfs(path) return round((1 - st.f_bavail / st.f_blocks) * 100, 1) except Exception: return None # ---------- отчёт ---------- def main(): services = load_registry() st = status_data() up_map = {s["name"]: s.get("up") for s in st.get("services", [])} mem = load_memory() groups = {} problems = [] for s in services: name = s.get("name") group = s.get("group", "other") ct = int(s.get("ct") or 0) up = up_map.get(name) reason = "" if up is None: state = docker_state(name, ct) up = state == "running" reason = f"docker state: {state}" rec = remember(mem, name, bool(up), reason) groups.setdefault(group, {"total": 0, "up": 0}) groups[group]["total"] += 1 groups[group]["up"] += 1 if up else 0 if not up: problems.append({ "name": name, "group": group, "ct": ct, "fails": rec.get("fails", 1), "reason": reason or "не в статусе", "since": rec.get("first_fail"), }) dups = duplicates(services) junk_info = junk() bkp_age, bkp_file = backup_freshness() cert_min, cert_all = certs_days() disk = disk_pct("/") mem["duplicates"] = dups save_memory(mem) os.makedirs(JOURNAL, exist_ok=True) stamp = datetime.now().strftime("%Y-%m-%d") report = { "ts": datetime.now().isoformat(timespec="seconds"), "services_total": len(services), "services_up": sum(g["up"] for g in groups.values()), "groups": groups, "problems": problems, "duplicates": dups, "junk": junk_info, "disk_pct": disk, "backup_age_hours": bkp_age, "cert_min_days": cert_min, "certs": cert_all[:3], "backup_file": bkp_file, } json.dump(report, open(os.path.join(JOURNAL, f"{stamp}-runner.json"), "w"), ensure_ascii=False, indent=1) print(f"сервисов: {report['services_up']}/{report['services_total']}") for g, v in sorted(groups.items()): mark = "✅" if v["up"] == v["total"] else "⚠️" print(f" {mark} {GROUP_TITLES.get(g, g)}: {v['up']}/{v['total']}") if problems: print("проблемы:") for p in problems[:10]: print(f" ✗ {p['name']} (CT{p['ct']}, сбоев {p['fails']}): {p['reason']}") if dups: print("дубли:") for d in dups[:5]: print(f" ⧉ {d['type']}: {', '.join(d['services'])}") if bkp_age is not None: print(f"бэкап: {bkp_age} ч назад") print(f"диск: {disk}% | висячих образов: {junk_info['dangling']}") if cert_min is not None: print(f"сертификаты: минимум {cert_min} дн. ({cert_all[0][0]})") if cert_min < 21: cert_alert(cert_all[0][0], cert_min) if __name__ == "__main__": main()