351 lines
14 KiB
Python
Executable File
351 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""Hermes — шина событий, планировщик и агентский слой Buryan Hub.
|
||
|
||
Зачем: события (алерты, бэкапы, обновления, домашние) сейчас размазаны по
|
||
хуку, ботам и cron. Hermes собирает их в один журнал, устраняет дубли,
|
||
маршрутизирует в доставку через хаб (/notify) и запускает периодические
|
||
задания с ретраями и историей.
|
||
|
||
Что умеет:
|
||
POST /event — принять событие (source, type, severity, title, body)
|
||
POST /alertmanager — принять вебхук Alertmanager и превратить в события
|
||
GET /events — журнал (limit, severity)
|
||
GET /health — состояние
|
||
GET /metrics — метрики Prometheus
|
||
POST /jobs/<name>/run — ручной запуск задания
|
||
GET /approvals — список ожидающих решений
|
||
|
||
Задания описываются в jobs.yaml (интервал или время суток), ретраи и таймаут.
|
||
Доставка — через http://127.0.0.1:9120/notify (хаб знает про Matrix/Telegram).
|
||
"""
|
||
import json
|
||
import os
|
||
import re
|
||
import sqlite3
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
|
||
BASE = "/opt/hermes"
|
||
DB = f"{BASE}/hermes.db"
|
||
JOBS = f"{BASE}/jobs.yaml"
|
||
CONF = f"{BASE}/hermes.yaml"
|
||
PORT = int(os.environ.get("HERMES_PORT", "9130"))
|
||
HOST = os.environ.get("HERMES_HOST", "0.0.0.0")
|
||
|
||
_lock = threading.Lock()
|
||
|
||
|
||
def conf():
|
||
try:
|
||
import yaml
|
||
return yaml.safe_load(open(CONF, encoding="utf-8")) or {}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
CFG = conf()
|
||
NOTIFY_URL = CFG.get("notify_url", "http://127.0.0.1:9120/notify")
|
||
QUIET = CFG.get("quiet_hours", []) # [["23:00","08:00"]]
|
||
DEDUP_SEC = int(CFG.get("dedup_seconds", 900))
|
||
RETENTION_DAYS = int(CFG.get("retention_days", 30))
|
||
|
||
|
||
def db():
|
||
conn = sqlite3.connect(DB, timeout=10)
|
||
conn.row_factory = sqlite3.Row
|
||
return conn
|
||
|
||
|
||
def init():
|
||
os.makedirs(BASE, exist_ok=True)
|
||
with db() as c:
|
||
c.executescript("""
|
||
create table if not exists events(
|
||
id integer primary key autoincrement,
|
||
ts real not null, source text, type text, severity text,
|
||
title text, body text, dedup_key text, delivered integer default 0,
|
||
delivery text, payload text);
|
||
create index if not exists ev_ts on events(ts desc);
|
||
create index if not exists ev_dedup on events(dedup_key, ts desc);
|
||
create table if not exists jobs(
|
||
name text primary key, last_run real, last_status text,
|
||
last_ms integer, runs integer default 0, fails integer default 0);
|
||
create table if not exists approvals(
|
||
id integer primary key autoincrement, ts real, kind text,
|
||
title text, details text, status text default 'pending');
|
||
""")
|
||
|
||
|
||
def now():
|
||
return time.time()
|
||
|
||
|
||
def in_quiet(ts=None):
|
||
if not QUIET:
|
||
return False
|
||
t = time.localtime(ts or now())
|
||
cur = t.tm_hour * 60 + t.tm_min
|
||
for span in QUIET:
|
||
try:
|
||
a, b = span
|
||
h1, m1 = map(int, a.split(":"))
|
||
h2, m2 = map(int, b.split(":"))
|
||
start, end = h1 * 60 + m1, h2 * 60 + m2
|
||
if start <= end:
|
||
if start <= cur < end:
|
||
return True
|
||
else: # через полночь
|
||
if cur >= start or cur < end:
|
||
return True
|
||
except Exception:
|
||
continue
|
||
return False
|
||
|
||
|
||
def notify(title, message, priority="normal", dedup_key=None):
|
||
"""Доставка через хаб. Возвращает (delivered, note)."""
|
||
if in_quiet() and priority != "critical":
|
||
return 0, "quiet-hours"
|
||
data = json.dumps({"title": title, "message": message, "priority": priority}).encode()
|
||
req = urllib.request.Request(NOTIFY_URL, data=data, method="POST",
|
||
headers={"Content-Type": "application/json"})
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=10) as r:
|
||
return 1, f"http:{r.status}"
|
||
except Exception as e:
|
||
return 0, f"error:{str(e)[:60]}"
|
||
|
||
|
||
def add_event(source, etype, severity, title, body="", dedup_key=None, payload=None):
|
||
"""Записать событие; при дубле в окне dedup — не доставлять повторно."""
|
||
key = dedup_key or f"{source}:{etype}:{title}"
|
||
with _lock, db() as c:
|
||
prev = c.execute(
|
||
"select id, delivered from events where dedup_key=? and ts>? order by ts desc limit 1",
|
||
(key, now() - DEDUP_SEC)).fetchone()
|
||
if prev and prev["delivered"]:
|
||
c.execute("insert into events(ts,source,type,severity,title,body,dedup_key,delivered,delivery,payload)"
|
||
" values(?,?,?,?,?,?,?,0,'dedup',?)",
|
||
(now(), source, etype, severity, title, body, key,
|
||
json.dumps(payload or {}, ensure_ascii=False)))
|
||
return False
|
||
cur = c.execute(
|
||
"insert into events(ts,source,type,severity,title,body,dedup_key,payload)"
|
||
" values(?,?,?,?,?,?,?,?)",
|
||
(now(), source, etype, severity, title, body, key,
|
||
json.dumps(payload or {}, ensure_ascii=False)))
|
||
eid = cur.lastrowid
|
||
|
||
prio = "critical" if severity in ("critical", "high") else "normal"
|
||
ok, note = notify(title, body or title, prio, key)
|
||
with db() as c:
|
||
c.execute("update events set delivered=?, delivery=? where id=?", (ok, note, eid))
|
||
return True
|
||
|
||
|
||
# ---------------- задания ----------------
|
||
|
||
def load_jobs():
|
||
try:
|
||
import yaml
|
||
return yaml.safe_load(open(JOBS, encoding="utf-8")) or {}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def run_job(name, job):
|
||
started = now()
|
||
cmd = job.get("command")
|
||
timeout = int(job.get("timeout", 600))
|
||
retries = int(job.get("retries", 0))
|
||
status, out = "ok", ""
|
||
for attempt in range(retries + 1):
|
||
try:
|
||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
||
out = ((r.stdout or "") + (r.stderr or "")).strip()
|
||
status = "ok" if r.returncode == 0 else f"fail:{r.returncode}"
|
||
if status == "ok":
|
||
break
|
||
except subprocess.TimeoutExpired:
|
||
status, out = "timeout", f"таймаут {timeout}с"
|
||
except Exception as e:
|
||
status, out = "error", str(e)[:150]
|
||
time.sleep(2)
|
||
|
||
ms = int((now() - started) * 1000)
|
||
with db() as c:
|
||
c.execute("""insert into jobs(name,last_run,last_status,last_ms,runs,fails)
|
||
values(?,?,?,?,1,?)
|
||
on conflict(name) do update set
|
||
last_run=excluded.last_run, last_status=excluded.last_status,
|
||
last_ms=excluded.last_ms, runs=runs+1,
|
||
fails=fails+excluded.fails""",
|
||
(name, started, status, ms, 0 if status == "ok" else 1))
|
||
|
||
if status != "ok":
|
||
add_event("hermes", "job", "medium", f"Задание провалено: {name}", out[:800],
|
||
dedup_key=f"job:{name}:{status}")
|
||
elif job.get("notify_on_success"):
|
||
add_event("hermes", "job", "low", f"Задание выполнено: {name}", out[:400],
|
||
dedup_key=f"job-ok:{name}:{int(started)//3600}")
|
||
return status, out
|
||
|
||
|
||
def scheduler_loop():
|
||
while True:
|
||
try:
|
||
jobs = load_jobs()
|
||
t = time.localtime()
|
||
for name, job in (jobs or {}).items():
|
||
if not isinstance(job, dict) or not job.get("command"):
|
||
continue
|
||
with db() as c:
|
||
row = c.execute("select last_run from jobs where name=?", (name,)).fetchone()
|
||
last = row["last_run"] if row else 0
|
||
due = False
|
||
if job.get("every_seconds"):
|
||
due = (now() - (last or 0)) >= int(job["every_seconds"])
|
||
elif job.get("daily_at"):
|
||
hh, mm = map(int, job["daily_at"].split(":"))
|
||
today = time.mktime((t.tm_year, t.tm_mon, t.tm_mday, hh, mm, 0, 0, 0, -1))
|
||
due = now() >= today and (last or 0) < today
|
||
if due:
|
||
threading.Thread(target=run_job, args=(name, job), daemon=True).start()
|
||
time.sleep(1)
|
||
except Exception as e:
|
||
print("scheduler:", e, flush=True)
|
||
time.sleep(30)
|
||
|
||
|
||
def cleanup_loop():
|
||
while True:
|
||
try:
|
||
with db() as c:
|
||
c.execute("delete from events where ts < ?", (now() - RETENTION_DAYS * 86400,))
|
||
except Exception:
|
||
pass
|
||
time.sleep(3600)
|
||
|
||
|
||
# ---------------- HTTP ----------------
|
||
|
||
class Handler(BaseHTTPRequestHandler):
|
||
server_version = "hermes/0.1"
|
||
|
||
def log_message(self, *a):
|
||
pass
|
||
|
||
def _send(self, code, obj):
|
||
body = json.dumps(obj, ensure_ascii=False).encode()
|
||
self.send_response(code)
|
||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.end_headers()
|
||
self.wfile.write(body)
|
||
|
||
def _text(self, code, text):
|
||
body = text.encode()
|
||
self.send_response(code)
|
||
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.end_headers()
|
||
self.wfile.write(body)
|
||
|
||
def do_GET(self):
|
||
p = self.path.split("?")[0]
|
||
q = dict(re.findall(r"([^?&=]+)=([^&]*)", self.path))
|
||
if p == "/health":
|
||
with db() as c:
|
||
n = c.execute("select count(*) n from events where ts>?", (now() - 3600,)).fetchone()["n"]
|
||
pend = c.execute("select count(*) n from approvals where status='pending'").fetchone()["n"]
|
||
return self._send(200, {"status": "ok", "events_1h": n, "approvals_pending": pend,
|
||
"quiet": in_quiet()})
|
||
if p == "/events":
|
||
limit = min(int(q.get("limit", 50)), 500)
|
||
with db() as c:
|
||
rows = c.execute("select * from events order by ts desc limit ?", (limit,)).fetchall()
|
||
return self._send(200, {"count": len(rows), "items": [dict(r) for r in rows]})
|
||
if p == "/approvals":
|
||
with db() as c:
|
||
rows = c.execute("select * from approvals order by ts desc limit 50").fetchall()
|
||
return self._send(200, {"count": len(rows), "items": [dict(r) for r in rows]})
|
||
if p == "/metrics":
|
||
with db() as c:
|
||
total = c.execute("select count(*) n from events").fetchone()["n"]
|
||
last_h = c.execute("select count(*) n from events where ts>?", (now() - 3600,)).fetchone()["n"]
|
||
undel = c.execute("select count(*) n from events where delivered=0").fetchone()["n"]
|
||
jobs = c.execute("select name,fails,runs,last_status from jobs").fetchall()
|
||
lines = ["# HELP hermes_events_total событий в журнале",
|
||
"# TYPE hermes_events_total gauge",
|
||
f"hermes_events_total {total}",
|
||
f"hermes_events_1h {last_h}",
|
||
f"hermes_events_undelivered {undel}",
|
||
f"hermes_quiet_active {1 if in_quiet() else 0}"]
|
||
for j in jobs:
|
||
lines.append(f'hermes_job_runs{{job="{j["name"]}",status="{j["last_status"]}"}} {j["runs"]}')
|
||
lines.append(f'hermes_job_fails{{job="{j["name"]}"}} {j["fails"]}')
|
||
return self._text(200, "\n".join(lines) + "\n")
|
||
return self._send(404, {"error": "not found"})
|
||
|
||
def do_POST(self):
|
||
p = self.path.split("?")[0]
|
||
length = int(self.headers.get("Content-Length") or 0)
|
||
raw = self.rfile.read(length) if length else b"{}"
|
||
try:
|
||
data = json.loads(raw or b"{}")
|
||
except Exception:
|
||
return self._send(400, {"error": "bad json"})
|
||
|
||
if p == "/event":
|
||
ok = add_event(data.get("source", "external"), data.get("type", "info"),
|
||
data.get("severity", "low"), data.get("title", "событие"),
|
||
data.get("body", ""), data.get("dedup_key"), data)
|
||
return self._send(200, {"accepted": True, "delivered": ok})
|
||
|
||
if p == "/alertmanager":
|
||
n = 0
|
||
for a in data.get("alerts", []):
|
||
if a.get("status") == "resolved":
|
||
continue
|
||
labels = a.get("labels") or {}
|
||
ann = a.get("annotations") or {}
|
||
sev = labels.get("severity", "warning")
|
||
add_event("prometheus", labels.get("alertname", "alert"),
|
||
"critical" if sev == "critical" else "medium",
|
||
ann.get("summary") or labels.get("alertname", "алерт"),
|
||
ann.get("description", ""),
|
||
dedup_key=f"am:{labels.get('alertname')}:{labels.get('instance','')}")
|
||
n += 1
|
||
return self._send(200, {"accepted": n})
|
||
|
||
m = re.match(r"^/jobs/([A-Za-z0-9_.-]+)/run$", p)
|
||
if m:
|
||
name = m.group(1)
|
||
jobs = load_jobs() or {}
|
||
if name not in jobs:
|
||
return self._send(404, {"error": "unknown job"})
|
||
threading.Thread(target=run_job, args=(name, jobs[name]), daemon=True).start()
|
||
return self._send(202, {"started": name})
|
||
|
||
return self._send(404, {"error": "not found"})
|
||
|
||
|
||
def main():
|
||
init()
|
||
threading.Thread(target=scheduler_loop, daemon=True).start()
|
||
threading.Thread(target=cleanup_loop, daemon=True).start()
|
||
srv = ThreadingHTTPServer((HOST, PORT), Handler)
|
||
print(f"hermes слушает {HOST}:{PORT}", flush=True)
|
||
add_event("hermes", "start", "low", "Hermes запущен", f"порт {PORT}",
|
||
dedup_key=f"hermes-start:{int(now())//600}")
|
||
srv.serve_forever()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|