Files

287 lines
9.8 KiB
Python

#!/usr/bin/env python3
"""Раннер роя: видит ВСЕ сервисы, помнит ошибки, ищет дубли и мусор.
Пишет отчёт в журнал и отдаёт его мозгу. Ничего не меняет без класса A/B.
"""
import json
import os
import re
import shutil
import subprocess
import time
from datetime import datetime
REGISTRY = "/opt/hub-core/services.yaml"
JOURNAL = "/var/lib/hub-agents/journal"
MEMORY = "/var/lib/hub-agents/memory.json"
STATUS = "http://127.0.0.1:9120/status.json"
NOTIFY = "http://127.0.0.1:9120/notify"
GROUP_TITLES = {
"core": "Ядро", "home": "Дом", "media": "Медиа", "knowledge": "Знания",
"automation": "Автоматизация", "observability": "Наблюдаемость",
"support": "Служебные", "other": "Прочее",
}
# ---------- реестр ----------
def load_registry():
"""Все сервисы из services.yaml: name, group, ct, image, ports, criticality."""
items = []
cur = None
try:
for line in open(REGISTRY, encoding="utf-8"):
if line.startswith(" - key:"):
if cur:
items.append(cur)
cur = {"key": line.split(":", 1)[1].strip().strip('"')}
elif cur is not None and line.startswith(" "):
k, _, v = line.strip().partition(":")
cur[k] = v.strip().strip('"')
if cur:
items.append(cur)
except Exception:
return []
return items
# ---------- память ----------
def load_memory():
try:
return json.load(open(MEMORY, encoding="utf-8"))
except Exception:
return {"services": {}, "duplicates": [], "solutions": {}}
def save_memory(m):
os.makedirs(os.path.dirname(MEMORY), exist_ok=True)
json.dump(m, open(MEMORY, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
def remember(mem, name, ok, reason=""):
rec = mem["services"].setdefault(name, {"fails": 0, "first_fail": None,
"last_fail": None, "last_ok": None,
"reasons": []})
now = datetime.now().isoformat(timespec="seconds")
if ok:
rec["last_ok"] = now
rec["fails"] = 0
else:
rec["fails"] = rec.get("fails", 0) + 1
rec["last_fail"] = now
if not rec.get("first_fail"):
rec["first_fail"] = now
if reason:
rec["reasons"] = (rec.get("reasons") or [])[-4:] + [reason[:120]]
return rec
# ---------- проверки ----------
def status_data():
import urllib.request
try:
with urllib.request.urlopen(STATUS, timeout=25) as r:
return json.loads(r.read())
except Exception:
return {}
def docker_state(name, ct):
"""Проверяет контейнер напрямую, если его нет в статусе."""
cmd = (["docker", "inspect", name] if ct == 0
else ["pct", "exec", str(ct), "--", "docker", "inspect", name])
r = subprocess.run(cmd + ["--format", "{{.State.Status}}"], capture_output=True,
text=True, timeout=20)
return (r.stdout or "").strip() or "unknown"
def duplicates(services):
"""Ищет реальные дубли: одинаковый образ в разных CT (двойной экземпляр)."""
by_image = {}
dups = []
for s in services:
img = (s.get("image") or "").split(":")[0]
if not img:
continue
by_image.setdefault(img, []).append(s)
for img, items in by_image.items():
if len(items) < 2:
continue
# служебные образы, которые нормально живут в каждом CT
if any(k in img for k in ("promtail", "oauth2-proxy", "exportarr", "redis")):
continue
cts = {int(i.get("ct") or 0) for i in items}
names = [i.get("name") for i in items]
# дубль: один и тот же сервис в разных местах (хост + CT)
if 0 in cts and len(cts) > 1:
dups.append({"type": "double_instance", "image": img,
"services": names, "hint": "один экземпляр на хосте, другой в CT"})
elif len(set(names)) == 1 and len(items) > 1:
dups.append({"type": "same_name", "image": img, "services": names})
return dups
def junk():
"""Мусор: большие логи, висячие образы."""
out = {"big_logs": [], "dangling": 0}
r = subprocess.run(["docker", "system", "df"], capture_output=True, text=True, timeout=40)
m = re.search(r"Dangling Images\s+\d+\s+(\d+)", r.stdout or "")
if m:
out["dangling"] = int(m.group(1))
return out
CERT_HOSTS = [
"authentik.buryans.ru", "git.buryans.ru", "grocy.buryans.ru",
"vikunja.buryans.ru", "homebox.buryans.ru", "prometheus.buryans.ru",
"media.buryans.ru", "home.buryans.ru", "hub.buryans.ru",
"docs.buryans.ru", "finance.buryans.ru", "n8n.buryans.ru",
"rss.buryans.ru", "books.buryans.ru",
]
def cert_alert(host, days):
"""Сообщает в Matrix о скором истечении сертификата."""
import json as _json
import urllib.request as _rq
body = _json.dumps({
"title": "Сертификат истекает",
"message": f"{host}: осталось {days} дн. Нужно обновить.",
"priority": "high",
}).encode()
req = _rq.Request("http://127.0.0.1:9120/notify", data=body, method="POST",
headers={"Content-Type": "application/json"})
try:
_rq.urlopen(req, timeout=15).read()
except Exception:
pass
def certs_days():
"""Минимальный остаток дней по сертификатам (None, если проверить нельзя)."""
import socket
import ssl
from datetime import datetime
left = []
for host in CERT_HOSTS:
try:
ctx = ssl.create_default_context()
with socket.create_connection((host, 443), timeout=8) as sock:
with ctx.wrap_socket(sock, server_hostname=host) as ss:
exp = ss.getpeercert().get("notAfter")
dt = datetime.strptime(exp, "%b %d %H:%M:%S %Y %Z")
left.append((host, (dt - datetime.utcnow()).days))
except Exception:
continue
if not left:
return None, []
left.sort(key=lambda x: x[1])
return left[0][1], left
def backup_freshness(max_hours=26):
import glob
paths = glob.glob("/mnt/backups/dump/*.zst")
if not paths:
return None, None
newest = max(paths, key=os.path.getmtime)
return round((time.time() - os.path.getmtime(newest)) / 3600, 1), os.path.basename(newest)
def disk_pct(path="/"):
try:
st = os.statvfs(path)
return round((1 - st.f_bavail / st.f_blocks) * 100, 1)
except Exception:
return None
# ---------- отчёт ----------
def main():
services = load_registry()
st = status_data()
up_map = {s["name"]: s.get("up") for s in st.get("services", [])}
mem = load_memory()
groups = {}
problems = []
for s in services:
name = s.get("name")
group = s.get("group", "other")
ct = int(s.get("ct") or 0)
up = up_map.get(name)
reason = ""
if up is None:
state = docker_state(name, ct)
up = state == "running"
reason = f"docker state: {state}"
rec = remember(mem, name, bool(up), reason)
groups.setdefault(group, {"total": 0, "up": 0})
groups[group]["total"] += 1
groups[group]["up"] += 1 if up else 0
if not up:
problems.append({
"name": name, "group": group, "ct": ct,
"fails": rec.get("fails", 1),
"reason": reason or "не в статусе",
"since": rec.get("first_fail"),
})
dups = duplicates(services)
junk_info = junk()
bkp_age, bkp_file = backup_freshness()
cert_min, cert_all = certs_days()
disk = disk_pct("/")
mem["duplicates"] = dups
save_memory(mem)
os.makedirs(JOURNAL, exist_ok=True)
stamp = datetime.now().strftime("%Y-%m-%d")
report = {
"ts": datetime.now().isoformat(timespec="seconds"),
"services_total": len(services),
"services_up": sum(g["up"] for g in groups.values()),
"groups": groups,
"problems": problems,
"duplicates": dups,
"junk": junk_info,
"disk_pct": disk,
"backup_age_hours": bkp_age,
"cert_min_days": cert_min,
"certs": cert_all[:3],
"backup_file": bkp_file,
}
json.dump(report, open(os.path.join(JOURNAL, f"{stamp}-runner.json"), "w"),
ensure_ascii=False, indent=1)
print(f"сервисов: {report['services_up']}/{report['services_total']}")
for g, v in sorted(groups.items()):
mark = "✅" if v["up"] == v["total"] else "⚠️"
print(f" {mark} {GROUP_TITLES.get(g, g)}: {v['up']}/{v['total']}")
if problems:
print("проблемы:")
for p in problems[:10]:
print(f" ✗ {p['name']} (CT{p['ct']}, сбоев {p['fails']}): {p['reason']}")
if dups:
print("дубли:")
for d in dups[:5]:
print(f" ⧉ {d['type']}: {', '.join(d['services'])}")
if bkp_age is not None:
print(f"бэкап: {bkp_age} ч назад")
print(f"диск: {disk}% | висячих образов: {junk_info['dangling']}")
if cert_min is not None:
print(f"сертификаты: минимум {cert_min} дн. ({cert_all[0][0]})")
if cert_min < 21:
cert_alert(cert_all[0][0], cert_min)
if __name__ == "__main__":
main()