#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Hermes — шина событий, планировщик и агентский слой Buryan Hub. Зачем: события (алерты, бэкапы, обновления, домашние) сейчас размазаны по хуку, ботам и cron. Hermes собирает их в один журнал, устраняет дубли, маршрутизирует в доставку через хаб (/notify) и запускает периодические задания с ретраями и историей. Что умеет: POST /event — принять событие (source, type, severity, title, body) POST /alertmanager — принять вебхук Alertmanager и превратить в события GET /events — журнал (limit, severity) GET /health — состояние GET /metrics — метрики Prometheus POST /jobs//run — ручной запуск задания GET /approvals — список ожидающих решений Задания описываются в jobs.yaml (интервал или время суток), ретраи и таймаут. Доставка — через http://127.0.0.1:9120/notify (хаб знает про Matrix/Telegram). """ import json import os import re import sqlite3 import subprocess import threading import time import urllib.error import urllib.request from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer BASE = "/opt/hermes" DB = f"{BASE}/hermes.db" JOBS = f"{BASE}/jobs.yaml" CONF = f"{BASE}/hermes.yaml" PORT = int(os.environ.get("HERMES_PORT", "9130")) HOST = os.environ.get("HERMES_HOST", "0.0.0.0") _lock = threading.Lock() def conf(): try: import yaml return yaml.safe_load(open(CONF, encoding="utf-8")) or {} except Exception: return {} CFG = conf() NOTIFY_URL = CFG.get("notify_url", "http://127.0.0.1:9120/notify") QUIET = CFG.get("quiet_hours", []) # [["23:00","08:00"]] DEDUP_SEC = int(CFG.get("dedup_seconds", 900)) RETENTION_DAYS = int(CFG.get("retention_days", 30)) def db(): conn = sqlite3.connect(DB, timeout=10) conn.row_factory = sqlite3.Row return conn def init(): os.makedirs(BASE, exist_ok=True) with db() as c: c.executescript(""" create table if not exists events( id integer primary key autoincrement, ts real not null, source text, type text, severity text, title text, body text, dedup_key text, delivered integer default 0, delivery text, payload text); create index if not exists ev_ts on events(ts desc); create index if not exists ev_dedup on events(dedup_key, ts desc); create table if not exists jobs( name text primary key, last_run real, last_status text, last_ms integer, runs integer default 0, fails integer default 0); create table if not exists approvals( id integer primary key autoincrement, ts real, kind text, title text, details text, status text default 'pending'); """) def now(): return time.time() def in_quiet(ts=None): if not QUIET: return False t = time.localtime(ts or now()) cur = t.tm_hour * 60 + t.tm_min for span in QUIET: try: a, b = span h1, m1 = map(int, a.split(":")) h2, m2 = map(int, b.split(":")) start, end = h1 * 60 + m1, h2 * 60 + m2 if start <= end: if start <= cur < end: return True else: # через полночь if cur >= start or cur < end: return True except Exception: continue return False def notify(title, message, priority="normal", dedup_key=None): """Доставка через хаб. Возвращает (delivered, note).""" if in_quiet() and priority != "critical": return 0, "quiet-hours" data = json.dumps({"title": title, "message": message, "priority": priority}).encode() req = urllib.request.Request(NOTIFY_URL, data=data, method="POST", headers={"Content-Type": "application/json"}) try: with urllib.request.urlopen(req, timeout=10) as r: return 1, f"http:{r.status}" except Exception as e: return 0, f"error:{str(e)[:60]}" def add_event(source, etype, severity, title, body="", dedup_key=None, payload=None): """Записать событие; при дубле в окне dedup — не доставлять повторно.""" key = dedup_key or f"{source}:{etype}:{title}" with _lock, db() as c: prev = c.execute( "select id, delivered from events where dedup_key=? and ts>? order by ts desc limit 1", (key, now() - DEDUP_SEC)).fetchone() if prev and prev["delivered"]: c.execute("insert into events(ts,source,type,severity,title,body,dedup_key,delivered,delivery,payload)" " values(?,?,?,?,?,?,?,0,'dedup',?)", (now(), source, etype, severity, title, body, key, json.dumps(payload or {}, ensure_ascii=False))) return False cur = c.execute( "insert into events(ts,source,type,severity,title,body,dedup_key,payload)" " values(?,?,?,?,?,?,?,?)", (now(), source, etype, severity, title, body, key, json.dumps(payload or {}, ensure_ascii=False))) eid = cur.lastrowid prio = "critical" if severity in ("critical", "high") else "normal" ok, note = notify(title, body or title, prio, key) with db() as c: c.execute("update events set delivered=?, delivery=? where id=?", (ok, note, eid)) return True # ---------------- задания ---------------- def load_jobs(): try: import yaml return yaml.safe_load(open(JOBS, encoding="utf-8")) or {} except Exception: return {} def run_job(name, job): started = now() cmd = job.get("command") timeout = int(job.get("timeout", 600)) retries = int(job.get("retries", 0)) status, out = "ok", "" for attempt in range(retries + 1): try: r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) out = ((r.stdout or "") + (r.stderr or "")).strip() status = "ok" if r.returncode == 0 else f"fail:{r.returncode}" if status == "ok": break except subprocess.TimeoutExpired: status, out = "timeout", f"таймаут {timeout}с" except Exception as e: status, out = "error", str(e)[:150] time.sleep(2) ms = int((now() - started) * 1000) with db() as c: c.execute("""insert into jobs(name,last_run,last_status,last_ms,runs,fails) values(?,?,?,?,1,?) on conflict(name) do update set last_run=excluded.last_run, last_status=excluded.last_status, last_ms=excluded.last_ms, runs=runs+1, fails=fails+excluded.fails""", (name, started, status, ms, 0 if status == "ok" else 1)) if status != "ok": add_event("hermes", "job", "medium", f"Задание провалено: {name}", out[:800], dedup_key=f"job:{name}:{status}") elif job.get("notify_on_success"): add_event("hermes", "job", "low", f"Задание выполнено: {name}", out[:400], dedup_key=f"job-ok:{name}:{int(started)//3600}") return status, out def scheduler_loop(): while True: try: jobs = load_jobs() t = time.localtime() for name, job in (jobs or {}).items(): if not isinstance(job, dict) or not job.get("command"): continue with db() as c: row = c.execute("select last_run from jobs where name=?", (name,)).fetchone() last = row["last_run"] if row else 0 due = False if job.get("every_seconds"): due = (now() - (last or 0)) >= int(job["every_seconds"]) elif job.get("daily_at"): hh, mm = map(int, job["daily_at"].split(":")) today = time.mktime((t.tm_year, t.tm_mon, t.tm_mday, hh, mm, 0, 0, 0, -1)) due = now() >= today and (last or 0) < today if due: threading.Thread(target=run_job, args=(name, job), daemon=True).start() time.sleep(1) except Exception as e: print("scheduler:", e, flush=True) time.sleep(30) def cleanup_loop(): while True: try: with db() as c: c.execute("delete from events where ts < ?", (now() - RETENTION_DAYS * 86400,)) except Exception: pass time.sleep(3600) # ---------------- HTTP ---------------- class Handler(BaseHTTPRequestHandler): server_version = "hermes/0.1" def log_message(self, *a): pass def _send(self, code, obj): body = json.dumps(obj, ensure_ascii=False).encode() self.send_response(code) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def _text(self, code, text): body = text.encode() self.send_response(code) self.send_header("Content-Type", "text/plain; charset=utf-8") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def do_GET(self): p = self.path.split("?")[0] q = dict(re.findall(r"([^?&=]+)=([^&]*)", self.path)) if p == "/health": with db() as c: n = c.execute("select count(*) n from events where ts>?", (now() - 3600,)).fetchone()["n"] pend = c.execute("select count(*) n from approvals where status='pending'").fetchone()["n"] return self._send(200, {"status": "ok", "events_1h": n, "approvals_pending": pend, "quiet": in_quiet()}) if p == "/events": limit = min(int(q.get("limit", 50)), 500) with db() as c: rows = c.execute("select * from events order by ts desc limit ?", (limit,)).fetchall() return self._send(200, {"count": len(rows), "items": [dict(r) for r in rows]}) if p == "/approvals": with db() as c: rows = c.execute("select * from approvals order by ts desc limit 50").fetchall() return self._send(200, {"count": len(rows), "items": [dict(r) for r in rows]}) if p == "/metrics": with db() as c: total = c.execute("select count(*) n from events").fetchone()["n"] last_h = c.execute("select count(*) n from events where ts>?", (now() - 3600,)).fetchone()["n"] undel = c.execute("select count(*) n from events where delivered=0").fetchone()["n"] jobs = c.execute("select name,fails,runs,last_status from jobs").fetchall() lines = ["# HELP hermes_events_total событий в журнале", "# TYPE hermes_events_total gauge", f"hermes_events_total {total}", f"hermes_events_1h {last_h}", f"hermes_events_undelivered {undel}", f"hermes_quiet_active {1 if in_quiet() else 0}"] for j in jobs: lines.append(f'hermes_job_runs{{job="{j["name"]}",status="{j["last_status"]}"}} {j["runs"]}') lines.append(f'hermes_job_fails{{job="{j["name"]}"}} {j["fails"]}') return self._text(200, "\n".join(lines) + "\n") return self._send(404, {"error": "not found"}) def do_POST(self): p = self.path.split("?")[0] length = int(self.headers.get("Content-Length") or 0) raw = self.rfile.read(length) if length else b"{}" try: data = json.loads(raw or b"{}") except Exception: return self._send(400, {"error": "bad json"}) if p == "/event": ok = add_event(data.get("source", "external"), data.get("type", "info"), data.get("severity", "low"), data.get("title", "событие"), data.get("body", ""), data.get("dedup_key"), data) return self._send(200, {"accepted": True, "delivered": ok}) if p == "/alertmanager": n = 0 for a in data.get("alerts", []): if a.get("status") == "resolved": continue labels = a.get("labels") or {} ann = a.get("annotations") or {} sev = labels.get("severity", "warning") add_event("prometheus", labels.get("alertname", "alert"), "critical" if sev == "critical" else "medium", ann.get("summary") or labels.get("alertname", "алерт"), ann.get("description", ""), dedup_key=f"am:{labels.get('alertname')}:{labels.get('instance','')}") n += 1 return self._send(200, {"accepted": n}) m = re.match(r"^/jobs/([A-Za-z0-9_.-]+)/run$", p) if m: name = m.group(1) jobs = load_jobs() or {} if name not in jobs: return self._send(404, {"error": "unknown job"}) threading.Thread(target=run_job, args=(name, jobs[name]), daemon=True).start() return self._send(202, {"started": name}) return self._send(404, {"error": "not found"}) def main(): init() threading.Thread(target=scheduler_loop, daemon=True).start() threading.Thread(target=cleanup_loop, daemon=True).start() srv = ThreadingHTTPServer((HOST, PORT), Handler) print(f"hermes слушает {HOST}:{PORT}", flush=True) add_event("hermes", "start", "low", "Hermes запущен", f"порт {PORT}", dedup_key=f"hermes-start:{int(now())//600}") srv.serve_forever() if __name__ == "__main__": main()