Wiki в Gitea: 9 страниц (архитектура, сервисы, агенты, дорожная карта, регламенты, ADR, онбординг, работа). Инициализирована через API, затем пушится как git-репозиторий. Статус-страница hub.buryans.ru/status: 62 сервиса по группам, состояние берётся из docker ps (TCP-проверка давала ложные срабатывания для CT111), кэш 60 с, JSON на /status.json. Рой агентов: groups.yaml (6 групп), policy.yaml (классы A/B/C/D, лимиты, тихие часы), ТЗ шести секретарей и Головы, раннер (наблюдение) и мозг на DeepSeek (json-ответ: диагноз, severity, предложенные действия). Таймеры: раннер каждые 30 минут, мозг ежечасно. В v0 ничего не выполняется — только диагностика и предложения; уведомление уходит при crit. Найдено и исправлено агентом: freshrss, catalog и их oauth2-proxy были недоступны — перезапущены, статус 62/62.
218 lines
7.0 KiB
Python
218 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Раннер роя: читает группы и политику, проверяет сервисы, пишет журнал.
|
|
|
|
Режим v0 — только наблюдение: ничего не меняет, только собирает факты
|
|
и готовит отчёт Голове.
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime
|
|
|
|
AGENTS_DIR = "/opt/hub-agents"
|
|
JOURNAL = "/var/lib/hub-agents/journal"
|
|
STATUS_URL = "http://127.0.0.1:9120/status.json"
|
|
NOTIFY = "http://127.0.0.1:9120/notify"
|
|
|
|
|
|
def read_yaml(path):
|
|
"""Минимальный YAML-парсер под наши файлы (без зависимостей)."""
|
|
out, cur, stack = {}, None, []
|
|
try:
|
|
lines = open(path, encoding="utf-8").read().splitlines()
|
|
except Exception:
|
|
return {}
|
|
for raw in lines:
|
|
if not raw.strip() or raw.strip().startswith("#"):
|
|
continue
|
|
indent = len(raw) - len(raw.lstrip())
|
|
line = raw.strip()
|
|
key, _, val = line.partition(":")
|
|
val = val.strip()
|
|
if indent == 0:
|
|
cur = None
|
|
if val:
|
|
out[key] = parse_scalar(val)
|
|
else:
|
|
out[key] = {}
|
|
stack = [(0, out[key])]
|
|
continue
|
|
# вложенность
|
|
while stack and indent <= stack[-1][0]:
|
|
stack.pop()
|
|
if not stack:
|
|
continue
|
|
target = stack[-1][1]
|
|
if val.startswith("["):
|
|
target[key] = [x.strip().strip("'\"") for x in val.strip("[]").split(",") if x.strip()]
|
|
elif val:
|
|
target[key] = parse_scalar(val)
|
|
else:
|
|
target[key] = {}
|
|
stack.append((indent, target[key]))
|
|
return out
|
|
|
|
|
|
def parse_scalar(v):
|
|
v = v.strip()
|
|
if v in ("true", "yes"):
|
|
return True
|
|
if v in ("false", "no"):
|
|
return False
|
|
if v.isdigit():
|
|
return int(v)
|
|
return v.strip("'\"")
|
|
|
|
|
|
def status():
|
|
import urllib.request
|
|
try:
|
|
with urllib.request.urlopen(STATUS_URL, timeout=25) as r:
|
|
return json.loads(r.read())
|
|
except Exception:
|
|
return {"services": [], "groups": {}, "total": 0, "up": 0}
|
|
|
|
|
|
def disk():
|
|
st = os.statvfs("/")
|
|
used = 1 - st.f_bavail / st.f_blocks
|
|
return round(used * 100, 1)
|
|
|
|
|
|
def swap():
|
|
try:
|
|
for line in open("/proc/meminfo"):
|
|
if line.startswith("SwapTotal"):
|
|
total = int(line.split()[1])
|
|
if line.startswith("SwapFree"):
|
|
free = int(line.split()[1])
|
|
return round(100 * (total - free) / total, 1) if total else 0.0
|
|
except Exception:
|
|
return 0.0
|
|
|
|
|
|
def backup_age_hours():
|
|
"""Сколько часов самому свежему дампу/архиву."""
|
|
best = None
|
|
for root in ("/mnt/backups", "/root/matrix-backups"):
|
|
for dirpath, _, files in os.walk(root):
|
|
for f in files:
|
|
if not (f.endswith((".gz", ".tar.gz", ".sql.gz", ".zst", ".tar"))):
|
|
continue
|
|
p = os.path.join(dirpath, f)
|
|
try:
|
|
m = os.path.getmtime(p)
|
|
except Exception:
|
|
continue
|
|
if best is None or m > best:
|
|
best = m
|
|
if not best:
|
|
return None
|
|
return round((time.time() - best) / 3600, 1)
|
|
|
|
|
|
def tcp(ip, port, timeout=1.2):
|
|
try:
|
|
with socket.create_connection((ip, port), timeout=timeout):
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def notify(title, message):
|
|
import urllib.request
|
|
body = json.dumps({"title": title, "message": message}).encode()
|
|
req = urllib.request.Request(NOTIFY, data=body, method="POST",
|
|
headers={"Content-Type": "application/json"})
|
|
try:
|
|
urllib.request.urlopen(req, timeout=15).read()
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def check_group(name, group, st):
|
|
"""Проверка группы: сервисы + доп. проверки."""
|
|
wanted = [x for x in (group.get("services") or []) if x]
|
|
services = st.get("services") or []
|
|
checked, down = 0, []
|
|
for svc in wanted:
|
|
matches = [s for s in services
|
|
if svc in (s.get("name") or "") or (s.get("name") or "") in svc]
|
|
if not matches:
|
|
continue
|
|
checked += len(matches)
|
|
for m in matches:
|
|
if not m.get("up"):
|
|
down.append(m.get("name"))
|
|
down = sorted(set(down))
|
|
extra = {}
|
|
checks = group.get("extra_checks") or []
|
|
if "disk" in checks:
|
|
extra["disk_used_pct"] = disk()
|
|
if "swap" in checks:
|
|
extra["swap_used_pct"] = swap()
|
|
if "backups" in checks:
|
|
extra["newest_backup_hours"] = backup_age_hours()
|
|
if "tls" in checks:
|
|
extra["tls_ok"] = tcp("127.0.0.1", 443)
|
|
if "ha_api" in checks:
|
|
extra["ha_ok"] = tcp("192.168.1.106", 8123)
|
|
if "zigbee" in checks:
|
|
extra["zigbee_bridge"] = tcp("192.168.1.10", 8098)
|
|
return {
|
|
"group": name,
|
|
"checked": checked,
|
|
"down": down,
|
|
"extra": extra,
|
|
}
|
|
|
|
|
|
def main():
|
|
groups_doc = read_yaml(os.path.join(AGENTS_DIR, "groups.yaml"))
|
|
policy = read_yaml(os.path.join(AGENTS_DIR, "policy.yaml"))
|
|
st = status()
|
|
os.makedirs(JOURNAL, exist_ok=True)
|
|
stamp = datetime.now().strftime("%Y-%m-%d")
|
|
reports = []
|
|
|
|
for name, group in (groups_doc.get("groups") or {}).items():
|
|
if not isinstance(group, dict):
|
|
continue
|
|
rep = check_group(name, group, st)
|
|
rep["agent"] = group.get("agent")
|
|
rep["ts"] = datetime.now().isoformat(timespec="seconds")
|
|
reports.append(rep)
|
|
path = os.path.join(JOURNAL, f"{stamp}-{rep['agent']}.json")
|
|
json.dump(rep, open(path, "w", encoding="utf-8"), ensure_ascii=False)
|
|
|
|
total_down = sum(len(r["down"]) for r in reports)
|
|
total_checked = sum(r["checked"] for r in reports)
|
|
print(f"проверено сервисов: {total_checked}, проблем: {total_down}")
|
|
for r in reports:
|
|
tail = f" — {', '.join(r['down'])}" if r["down"] else ""
|
|
print(f" {r['group']:<14} {r['checked'] - len(r['down'])}/{r['checked']}{tail}")
|
|
for k, v in (r["extra"] or {}).items():
|
|
print(f" {k}: {v}")
|
|
|
|
# Голова: сообщает только если есть проблемы и не тихие часы
|
|
hour = datetime.now().hour
|
|
quiet = hour >= 23 or hour < 8
|
|
if total_down and not quiet:
|
|
lines = [f"⚠️ Проблемы: {total_down}"]
|
|
for r in reports:
|
|
if r["down"]:
|
|
lines.append(f"• {r['group']}: {', '.join(r['down'])}")
|
|
notify("Страж: проблемы в сервисах", "\n".join(lines))
|
|
print(" уведомление отправлено")
|
|
else:
|
|
print(" уведомление не нужно")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|