Files
buryan-infra/ops/hermes/hermes.py
T

351 lines
14 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Hermes — шина событий, планировщик и агентский слой Buryan Hub.
Зачем: события (алерты, бэкапы, обновления, домашние) сейчас размазаны по
хуку, ботам и cron. Hermes собирает их в один журнал, устраняет дубли,
маршрутизирует в доставку через хаб (/notify) и запускает периодические
задания с ретраями и историей.
Что умеет:
POST /event — принять событие (source, type, severity, title, body)
POST /alertmanager — принять вебхук Alertmanager и превратить в события
GET /events — журнал (limit, severity)
GET /health — состояние
GET /metrics — метрики Prometheus
POST /jobs/<name>/run — ручной запуск задания
GET /approvals — список ожидающих решений
Задания описываются в jobs.yaml (интервал или время суток), ретраи и таймаут.
Доставка — через http://127.0.0.1:9120/notify (хаб знает про Matrix/Telegram).
"""
import json
import os
import re
import sqlite3
import subprocess
import threading
import time
import urllib.error
import urllib.request
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
BASE = "/opt/hermes"
DB = f"{BASE}/hermes.db"
JOBS = f"{BASE}/jobs.yaml"
CONF = f"{BASE}/hermes.yaml"
PORT = int(os.environ.get("HERMES_PORT", "9130"))
HOST = os.environ.get("HERMES_HOST", "0.0.0.0")
_lock = threading.Lock()
def conf():
try:
import yaml
return yaml.safe_load(open(CONF, encoding="utf-8")) or {}
except Exception:
return {}
CFG = conf()
NOTIFY_URL = CFG.get("notify_url", "http://127.0.0.1:9120/notify")
QUIET = CFG.get("quiet_hours", []) # [["23:00","08:00"]]
DEDUP_SEC = int(CFG.get("dedup_seconds", 900))
RETENTION_DAYS = int(CFG.get("retention_days", 30))
def db():
conn = sqlite3.connect(DB, timeout=10)
conn.row_factory = sqlite3.Row
return conn
def init():
os.makedirs(BASE, exist_ok=True)
with db() as c:
c.executescript("""
create table if not exists events(
id integer primary key autoincrement,
ts real not null, source text, type text, severity text,
title text, body text, dedup_key text, delivered integer default 0,
delivery text, payload text);
create index if not exists ev_ts on events(ts desc);
create index if not exists ev_dedup on events(dedup_key, ts desc);
create table if not exists jobs(
name text primary key, last_run real, last_status text,
last_ms integer, runs integer default 0, fails integer default 0);
create table if not exists approvals(
id integer primary key autoincrement, ts real, kind text,
title text, details text, status text default 'pending');
""")
def now():
return time.time()
def in_quiet(ts=None):
if not QUIET:
return False
t = time.localtime(ts or now())
cur = t.tm_hour * 60 + t.tm_min
for span in QUIET:
try:
a, b = span
h1, m1 = map(int, a.split(":"))
h2, m2 = map(int, b.split(":"))
start, end = h1 * 60 + m1, h2 * 60 + m2
if start <= end:
if start <= cur < end:
return True
else: # через полночь
if cur >= start or cur < end:
return True
except Exception:
continue
return False
def notify(title, message, priority="normal", dedup_key=None):
"""Доставка через хаб. Возвращает (delivered, note)."""
if in_quiet() and priority != "critical":
return 0, "quiet-hours"
data = json.dumps({"title": title, "message": message, "priority": priority}).encode()
req = urllib.request.Request(NOTIFY_URL, data=data, method="POST",
headers={"Content-Type": "application/json"})
try:
with urllib.request.urlopen(req, timeout=10) as r:
return 1, f"http:{r.status}"
except Exception as e:
return 0, f"error:{str(e)[:60]}"
def add_event(source, etype, severity, title, body="", dedup_key=None, payload=None):
"""Записать событие; при дубле в окне dedup — не доставлять повторно."""
key = dedup_key or f"{source}:{etype}:{title}"
with _lock, db() as c:
prev = c.execute(
"select id, delivered from events where dedup_key=? and ts>? order by ts desc limit 1",
(key, now() - DEDUP_SEC)).fetchone()
if prev and prev["delivered"]:
c.execute("insert into events(ts,source,type,severity,title,body,dedup_key,delivered,delivery,payload)"
" values(?,?,?,?,?,?,?,0,'dedup',?)",
(now(), source, etype, severity, title, body, key,
json.dumps(payload or {}, ensure_ascii=False)))
return False
cur = c.execute(
"insert into events(ts,source,type,severity,title,body,dedup_key,payload)"
" values(?,?,?,?,?,?,?,?)",
(now(), source, etype, severity, title, body, key,
json.dumps(payload or {}, ensure_ascii=False)))
eid = cur.lastrowid
prio = "critical" if severity in ("critical", "high") else "normal"
ok, note = notify(title, body or title, prio, key)
with db() as c:
c.execute("update events set delivered=?, delivery=? where id=?", (ok, note, eid))
return True
# ---------------- задания ----------------
def load_jobs():
try:
import yaml
return yaml.safe_load(open(JOBS, encoding="utf-8")) or {}
except Exception:
return {}
def run_job(name, job):
started = now()
cmd = job.get("command")
timeout = int(job.get("timeout", 600))
retries = int(job.get("retries", 0))
status, out = "ok", ""
for attempt in range(retries + 1):
try:
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
out = ((r.stdout or "") + (r.stderr or "")).strip()
status = "ok" if r.returncode == 0 else f"fail:{r.returncode}"
if status == "ok":
break
except subprocess.TimeoutExpired:
status, out = "timeout", f"таймаут {timeout}с"
except Exception as e:
status, out = "error", str(e)[:150]
time.sleep(2)
ms = int((now() - started) * 1000)
with db() as c:
c.execute("""insert into jobs(name,last_run,last_status,last_ms,runs,fails)
values(?,?,?,?,1,?)
on conflict(name) do update set
last_run=excluded.last_run, last_status=excluded.last_status,
last_ms=excluded.last_ms, runs=runs+1,
fails=fails+excluded.fails""",
(name, started, status, ms, 0 if status == "ok" else 1))
if status != "ok":
add_event("hermes", "job", "medium", f"Задание провалено: {name}", out[:800],
dedup_key=f"job:{name}:{status}")
elif job.get("notify_on_success"):
add_event("hermes", "job", "low", f"Задание выполнено: {name}", out[:400],
dedup_key=f"job-ok:{name}:{int(started)//3600}")
return status, out
def scheduler_loop():
while True:
try:
jobs = load_jobs()
t = time.localtime()
for name, job in (jobs or {}).items():
if not isinstance(job, dict) or not job.get("command"):
continue
with db() as c:
row = c.execute("select last_run from jobs where name=?", (name,)).fetchone()
last = row["last_run"] if row else 0
due = False
if job.get("every_seconds"):
due = (now() - (last or 0)) >= int(job["every_seconds"])
elif job.get("daily_at"):
hh, mm = map(int, job["daily_at"].split(":"))
today = time.mktime((t.tm_year, t.tm_mon, t.tm_mday, hh, mm, 0, 0, 0, -1))
due = now() >= today and (last or 0) < today
if due:
threading.Thread(target=run_job, args=(name, job), daemon=True).start()
time.sleep(1)
except Exception as e:
print("scheduler:", e, flush=True)
time.sleep(30)
def cleanup_loop():
while True:
try:
with db() as c:
c.execute("delete from events where ts < ?", (now() - RETENTION_DAYS * 86400,))
except Exception:
pass
time.sleep(3600)
# ---------------- HTTP ----------------
class Handler(BaseHTTPRequestHandler):
server_version = "hermes/0.1"
def log_message(self, *a):
pass
def _send(self, code, obj):
body = json.dumps(obj, ensure_ascii=False).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _text(self, code, text):
body = text.encode()
self.send_response(code)
self.send_header("Content-Type", "text/plain; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_GET(self):
p = self.path.split("?")[0]
q = dict(re.findall(r"([^?&=]+)=([^&]*)", self.path))
if p == "/health":
with db() as c:
n = c.execute("select count(*) n from events where ts>?", (now() - 3600,)).fetchone()["n"]
pend = c.execute("select count(*) n from approvals where status='pending'").fetchone()["n"]
return self._send(200, {"status": "ok", "events_1h": n, "approvals_pending": pend,
"quiet": in_quiet()})
if p == "/events":
limit = min(int(q.get("limit", 50)), 500)
with db() as c:
rows = c.execute("select * from events order by ts desc limit ?", (limit,)).fetchall()
return self._send(200, {"count": len(rows), "items": [dict(r) for r in rows]})
if p == "/approvals":
with db() as c:
rows = c.execute("select * from approvals order by ts desc limit 50").fetchall()
return self._send(200, {"count": len(rows), "items": [dict(r) for r in rows]})
if p == "/metrics":
with db() as c:
total = c.execute("select count(*) n from events").fetchone()["n"]
last_h = c.execute("select count(*) n from events where ts>?", (now() - 3600,)).fetchone()["n"]
undel = c.execute("select count(*) n from events where delivered=0").fetchone()["n"]
jobs = c.execute("select name,fails,runs,last_status from jobs").fetchall()
lines = ["# HELP hermes_events_total событий в журнале",
"# TYPE hermes_events_total gauge",
f"hermes_events_total {total}",
f"hermes_events_1h {last_h}",
f"hermes_events_undelivered {undel}",
f"hermes_quiet_active {1 if in_quiet() else 0}"]
for j in jobs:
lines.append(f'hermes_job_runs{{job="{j["name"]}",status="{j["last_status"]}"}} {j["runs"]}')
lines.append(f'hermes_job_fails{{job="{j["name"]}"}} {j["fails"]}')
return self._text(200, "\n".join(lines) + "\n")
return self._send(404, {"error": "not found"})
def do_POST(self):
p = self.path.split("?")[0]
length = int(self.headers.get("Content-Length") or 0)
raw = self.rfile.read(length) if length else b"{}"
try:
data = json.loads(raw or b"{}")
except Exception:
return self._send(400, {"error": "bad json"})
if p == "/event":
ok = add_event(data.get("source", "external"), data.get("type", "info"),
data.get("severity", "low"), data.get("title", "событие"),
data.get("body", ""), data.get("dedup_key"), data)
return self._send(200, {"accepted": True, "delivered": ok})
if p == "/alertmanager":
n = 0
for a in data.get("alerts", []):
if a.get("status") == "resolved":
continue
labels = a.get("labels") or {}
ann = a.get("annotations") or {}
sev = labels.get("severity", "warning")
add_event("prometheus", labels.get("alertname", "alert"),
"critical" if sev == "critical" else "medium",
ann.get("summary") or labels.get("alertname", "алерт"),
ann.get("description", ""),
dedup_key=f"am:{labels.get('alertname')}:{labels.get('instance','')}")
n += 1
return self._send(200, {"accepted": n})
m = re.match(r"^/jobs/([A-Za-z0-9_.-]+)/run$", p)
if m:
name = m.group(1)
jobs = load_jobs() or {}
if name not in jobs:
return self._send(404, {"error": "unknown job"})
threading.Thread(target=run_job, args=(name, jobs[name]), daemon=True).start()
return self._send(202, {"started": name})
return self._send(404, {"error": "not found"})
def main():
init()
threading.Thread(target=scheduler_loop, daemon=True).start()
threading.Thread(target=cleanup_loop, daemon=True).start()
srv = ThreadingHTTPServer((HOST, PORT), Handler)
print(f"hermes слушает {HOST}:{PORT}", flush=True)
add_event("hermes", "start", "low", "Hermes запущен", f"порт {PORT}",
dedup_key=f"hermes-start:{int(now())//600}")
srv.serve_forever()
if __name__ == "__main__":
main()