287 lines
9.8 KiB
Python
287 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Раннер роя: видит ВСЕ сервисы, помнит ошибки, ищет дубли и мусор.
|
|
|
|
Пишет отчёт в журнал и отдаёт его мозгу. Ничего не меняет без класса A/B.
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
from datetime import datetime
|
|
|
|
REGISTRY = "/opt/hub-core/services.yaml"
|
|
JOURNAL = "/var/lib/hub-agents/journal"
|
|
MEMORY = "/var/lib/hub-agents/memory.json"
|
|
STATUS = "http://127.0.0.1:9120/status.json"
|
|
NOTIFY = "http://127.0.0.1:9120/notify"
|
|
|
|
GROUP_TITLES = {
|
|
"core": "Ядро", "home": "Дом", "media": "Медиа", "knowledge": "Знания",
|
|
"automation": "Автоматизация", "observability": "Наблюдаемость",
|
|
"support": "Служебные", "other": "Прочее",
|
|
}
|
|
|
|
|
|
# ---------- реестр ----------
|
|
|
|
def load_registry():
|
|
"""Все сервисы из services.yaml: name, group, ct, image, ports, criticality."""
|
|
items = []
|
|
cur = None
|
|
try:
|
|
for line in open(REGISTRY, encoding="utf-8"):
|
|
if line.startswith(" - key:"):
|
|
if cur:
|
|
items.append(cur)
|
|
cur = {"key": line.split(":", 1)[1].strip().strip('"')}
|
|
elif cur is not None and line.startswith(" "):
|
|
k, _, v = line.strip().partition(":")
|
|
cur[k] = v.strip().strip('"')
|
|
if cur:
|
|
items.append(cur)
|
|
except Exception:
|
|
return []
|
|
return items
|
|
|
|
|
|
# ---------- память ----------
|
|
|
|
def load_memory():
|
|
try:
|
|
return json.load(open(MEMORY, encoding="utf-8"))
|
|
except Exception:
|
|
return {"services": {}, "duplicates": [], "solutions": {}}
|
|
|
|
|
|
def save_memory(m):
|
|
os.makedirs(os.path.dirname(MEMORY), exist_ok=True)
|
|
json.dump(m, open(MEMORY, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
|
|
|
|
|
|
def remember(mem, name, ok, reason=""):
|
|
rec = mem["services"].setdefault(name, {"fails": 0, "first_fail": None,
|
|
"last_fail": None, "last_ok": None,
|
|
"reasons": []})
|
|
now = datetime.now().isoformat(timespec="seconds")
|
|
if ok:
|
|
rec["last_ok"] = now
|
|
rec["fails"] = 0
|
|
else:
|
|
rec["fails"] = rec.get("fails", 0) + 1
|
|
rec["last_fail"] = now
|
|
if not rec.get("first_fail"):
|
|
rec["first_fail"] = now
|
|
if reason:
|
|
rec["reasons"] = (rec.get("reasons") or [])[-4:] + [reason[:120]]
|
|
return rec
|
|
|
|
|
|
# ---------- проверки ----------
|
|
|
|
def status_data():
|
|
import urllib.request
|
|
try:
|
|
with urllib.request.urlopen(STATUS, timeout=25) as r:
|
|
return json.loads(r.read())
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def docker_state(name, ct):
|
|
"""Проверяет контейнер напрямую, если его нет в статусе."""
|
|
cmd = (["docker", "inspect", name] if ct == 0
|
|
else ["pct", "exec", str(ct), "--", "docker", "inspect", name])
|
|
r = subprocess.run(cmd + ["--format", "{{.State.Status}}"], capture_output=True,
|
|
text=True, timeout=20)
|
|
return (r.stdout or "").strip() or "unknown"
|
|
|
|
|
|
def duplicates(services):
|
|
"""Ищет реальные дубли: одинаковый образ в разных CT (двойной экземпляр)."""
|
|
by_image = {}
|
|
dups = []
|
|
for s in services:
|
|
img = (s.get("image") or "").split(":")[0]
|
|
if not img:
|
|
continue
|
|
by_image.setdefault(img, []).append(s)
|
|
|
|
for img, items in by_image.items():
|
|
if len(items) < 2:
|
|
continue
|
|
# служебные образы, которые нормально живут в каждом CT
|
|
if any(k in img for k in ("promtail", "oauth2-proxy", "exportarr", "redis")):
|
|
continue
|
|
cts = {int(i.get("ct") or 0) for i in items}
|
|
names = [i.get("name") for i in items]
|
|
# дубль: один и тот же сервис в разных местах (хост + CT)
|
|
if 0 in cts and len(cts) > 1:
|
|
dups.append({"type": "double_instance", "image": img,
|
|
"services": names, "hint": "один экземпляр на хосте, другой в CT"})
|
|
elif len(set(names)) == 1 and len(items) > 1:
|
|
dups.append({"type": "same_name", "image": img, "services": names})
|
|
return dups
|
|
|
|
|
|
def junk():
|
|
"""Мусор: большие логи, висячие образы."""
|
|
out = {"big_logs": [], "dangling": 0}
|
|
r = subprocess.run(["docker", "system", "df"], capture_output=True, text=True, timeout=40)
|
|
m = re.search(r"Dangling Images\s+\d+\s+(\d+)", r.stdout or "")
|
|
if m:
|
|
out["dangling"] = int(m.group(1))
|
|
return out
|
|
|
|
|
|
|
|
CERT_HOSTS = [
|
|
"authentik.buryans.ru", "git.buryans.ru", "grocy.buryans.ru",
|
|
"vikunja.buryans.ru", "homebox.buryans.ru", "prometheus.buryans.ru",
|
|
"media.buryans.ru", "home.buryans.ru", "hub.buryans.ru",
|
|
"docs.buryans.ru", "finance.buryans.ru", "n8n.buryans.ru",
|
|
"rss.buryans.ru", "books.buryans.ru",
|
|
]
|
|
|
|
|
|
def cert_alert(host, days):
|
|
"""Сообщает в Matrix о скором истечении сертификата."""
|
|
import json as _json
|
|
import urllib.request as _rq
|
|
|
|
body = _json.dumps({
|
|
"title": "Сертификат истекает",
|
|
"message": f"{host}: осталось {days} дн. Нужно обновить.",
|
|
"priority": "high",
|
|
}).encode()
|
|
req = _rq.Request("http://127.0.0.1:9120/notify", data=body, method="POST",
|
|
headers={"Content-Type": "application/json"})
|
|
try:
|
|
_rq.urlopen(req, timeout=15).read()
|
|
except Exception:
|
|
pass
|
|
|
|
def certs_days():
|
|
"""Минимальный остаток дней по сертификатам (None, если проверить нельзя)."""
|
|
import socket
|
|
import ssl
|
|
from datetime import datetime
|
|
|
|
left = []
|
|
for host in CERT_HOSTS:
|
|
try:
|
|
ctx = ssl.create_default_context()
|
|
with socket.create_connection((host, 443), timeout=8) as sock:
|
|
with ctx.wrap_socket(sock, server_hostname=host) as ss:
|
|
exp = ss.getpeercert().get("notAfter")
|
|
dt = datetime.strptime(exp, "%b %d %H:%M:%S %Y %Z")
|
|
left.append((host, (dt - datetime.utcnow()).days))
|
|
except Exception:
|
|
continue
|
|
if not left:
|
|
return None, []
|
|
left.sort(key=lambda x: x[1])
|
|
return left[0][1], left
|
|
|
|
|
|
def backup_freshness(max_hours=26):
|
|
import glob
|
|
paths = glob.glob("/mnt/backups/dump/*.zst")
|
|
if not paths:
|
|
return None, None
|
|
newest = max(paths, key=os.path.getmtime)
|
|
return round((time.time() - os.path.getmtime(newest)) / 3600, 1), os.path.basename(newest)
|
|
|
|
|
|
def disk_pct(path="/"):
|
|
try:
|
|
st = os.statvfs(path)
|
|
return round((1 - st.f_bavail / st.f_blocks) * 100, 1)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
# ---------- отчёт ----------
|
|
|
|
def main():
|
|
services = load_registry()
|
|
st = status_data()
|
|
up_map = {s["name"]: s.get("up") for s in st.get("services", [])}
|
|
mem = load_memory()
|
|
|
|
groups = {}
|
|
problems = []
|
|
for s in services:
|
|
name = s.get("name")
|
|
group = s.get("group", "other")
|
|
ct = int(s.get("ct") or 0)
|
|
up = up_map.get(name)
|
|
reason = ""
|
|
if up is None:
|
|
state = docker_state(name, ct)
|
|
up = state == "running"
|
|
reason = f"docker state: {state}"
|
|
rec = remember(mem, name, bool(up), reason)
|
|
groups.setdefault(group, {"total": 0, "up": 0})
|
|
groups[group]["total"] += 1
|
|
groups[group]["up"] += 1 if up else 0
|
|
if not up:
|
|
problems.append({
|
|
"name": name, "group": group, "ct": ct,
|
|
"fails": rec.get("fails", 1),
|
|
"reason": reason or "не в статусе",
|
|
"since": rec.get("first_fail"),
|
|
})
|
|
|
|
dups = duplicates(services)
|
|
junk_info = junk()
|
|
bkp_age, bkp_file = backup_freshness()
|
|
cert_min, cert_all = certs_days()
|
|
disk = disk_pct("/")
|
|
mem["duplicates"] = dups
|
|
save_memory(mem)
|
|
|
|
os.makedirs(JOURNAL, exist_ok=True)
|
|
stamp = datetime.now().strftime("%Y-%m-%d")
|
|
report = {
|
|
"ts": datetime.now().isoformat(timespec="seconds"),
|
|
"services_total": len(services),
|
|
"services_up": sum(g["up"] for g in groups.values()),
|
|
"groups": groups,
|
|
"problems": problems,
|
|
"duplicates": dups,
|
|
"junk": junk_info,
|
|
"disk_pct": disk,
|
|
"backup_age_hours": bkp_age,
|
|
"cert_min_days": cert_min,
|
|
"certs": cert_all[:3],
|
|
"backup_file": bkp_file,
|
|
}
|
|
json.dump(report, open(os.path.join(JOURNAL, f"{stamp}-runner.json"), "w"),
|
|
ensure_ascii=False, indent=1)
|
|
|
|
print(f"сервисов: {report['services_up']}/{report['services_total']}")
|
|
for g, v in sorted(groups.items()):
|
|
mark = "✅" if v["up"] == v["total"] else "⚠️"
|
|
print(f" {mark} {GROUP_TITLES.get(g, g)}: {v['up']}/{v['total']}")
|
|
if problems:
|
|
print("проблемы:")
|
|
for p in problems[:10]:
|
|
print(f" ✗ {p['name']} (CT{p['ct']}, сбоев {p['fails']}): {p['reason']}")
|
|
if dups:
|
|
print("дубли:")
|
|
for d in dups[:5]:
|
|
print(f" ⧉ {d['type']}: {', '.join(d['services'])}")
|
|
if bkp_age is not None:
|
|
print(f"бэкап: {bkp_age} ч назад")
|
|
print(f"диск: {disk}% | висячих образов: {junk_info['dangling']}")
|
|
if cert_min is not None:
|
|
print(f"сертификаты: минимум {cert_min} дн. ({cert_all[0][0]})")
|
|
if cert_min < 21:
|
|
cert_alert(cert_all[0][0], cert_min)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|