Files
buryan-infra/ops/hub-core/core.py
T
buryan-agent b11899cec7 Панель наружу через status.buryans.ru/hub/ + блок «что есть»
Добавлен location /hub/ в существующий домен status.buryans.ru — новых DNS
и сертификатов не потребовалось. Снаружи доступ закрыт входом Authentik,
изнутри сети панель как раньше на 9120.

На панель добавлен блок «Что есть в сервисах» из последней инвентаризации
(Jellyfin, Navidrome, Calibre, Immich, Paperless, Frigate).

Проверено: локально 200 и блок на месте, через HTTPS с нужным Host — 200,
публично — 302 на вход. nginx -t успешно, конфиг забэкаплен.
2026-10-19 20:15:56 +03:00

602 lines
26 KiB
Python

"""Buryan Hub Core: шина событий и граф сущностей.
Один процесс, три задачи:
1) принимать события от сервисов и складывать в общую базу;
2) хранить карточки сущностей и связи между ними;
3) публиковать значимое в Matrix (личка владельцу).
Запуск: python3 core.py (HTTP на порту 9120)
"""
import json
import os
import threading
import urllib.request
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
import psycopg2
import psycopg2.extras
DSN = os.environ.get("CORE_DSN", "dbname=hub_core user=hub_core password=hub_core host=127.0.0.1")
PORT = int(os.environ.get("CORE_PORT", "9120"))
MATRIX_HS = os.environ.get("MATRIX_HS", "https://matrix.buryans.ru")
NOTIFY_TOKEN = os.environ.get("NOTIFY_TOKEN", "")
NOTIFY_ROOM = os.environ.get("NOTIFY_ROOM", "")
# какие события стоит показывать человеку сразу
LOUD = {
"media.imported", "media.requested", "media.failed",
"backup.done", "backup.failed", "service.down", "service.up",
"photo.imported", "book.added", "document.added",
}
def db():
con = psycopg2.connect(DSN, connect_timeout=10)
con.autocommit = True
return con
def ensure_schema():
sql = open(os.path.join(os.path.dirname(__file__), "schema.sql"), encoding="utf-8").read()
with db() as con, con.cursor() as cur:
cur.execute(sql)
def publish(text):
"""Короткое уведомление в личку владельцу."""
if not (NOTIFY_TOKEN and NOTIFY_ROOM):
return
txn = int(datetime.now().timestamp() * 1000)
import urllib.parse
rid = urllib.parse.quote(NOTIFY_ROOM, safe="")
body = json.dumps({"msgtype": "m.text", "body": text,
"format": "org.matrix.custom.html",
"formatted_body": text.replace("\n", "<br>")}).encode()
req = urllib.request.Request(
f"{MATRIX_HS}/_matrix/client/v3/rooms/{rid}/send/m.room.message/{txn}",
data=body, method="PUT",
headers={"Authorization": f"Bearer {NOTIFY_TOKEN}", "Content-Type": "application/json"})
try:
urllib.request.urlopen(req, timeout=10)
except Exception:
pass
def human_event(ev):
e = ev.get("entity") or {}
title = e.get("title") or e.get("key") or "объект"
t = ev.get("type", "")
icons = {"media.imported": "🎬", "media.requested": "🕓", "media.failed": "⚠️",
"backup.done": "💾", "backup.failed": "⚠️",
"service.down": "🔴", "service.up": "✅",
"photo.imported": "📷", "book.added": "📚", "document.added": "📄"}
icon = icons.get(t, "•")
src = ev.get("source", "?")
extra = ev.get("payload") or {}
line = f"{icon} **{title}** — {t} ({src})"
if extra.get("detail"):
line += f"\n{extra['detail']}"
return line
def do_search(q, limit=20):
"""Поиск по карточкам: название и внешние ключи."""
like = f"%{q.strip().lower()}%"
con = db()
cur = con.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
cur.execute("""select id, kind, title, status, external_ids, updated_at
from entities where search_text like %s
order by updated_at desc limit %s""", (like, limit))
items = [dict(r) for r in cur.fetchall()]
con.close()
return {"query": q, "count": len(items), "items": items}
def do_entity(eid):
"""Карточка: сама сущность, её связи и история событий."""
con = db()
cur = con.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
cur.execute("select * from entities where id = %s", (eid,))
row = cur.fetchone()
if not row:
con.close()
return None
card = dict(row)
cur.execute("""select e.id, e.kind, e.title, l.relation
from entity_links l
join entities e on e.id = case when l.from_id = %s then l.to_id else l.from_id end
where l.from_id = %s or l.to_id = %s""", (eid, eid, eid))
links = [dict(r) for r in cur.fetchall()]
cur.execute("""select at, source, type, payload from events
where entity_id = %s order by id desc limit 50""", (eid,))
history = [dict(r) for r in cur.fetchall()]
con.close()
return {"card": card, "links": links, "history": history}
DATA_DIR = os.environ.get("PROJECT_DATA", "/root/buryan-infra/data")
def _ensure_doc_table():
con = db()
with con.cursor() as cur:
cur.execute("""
create table if not exists data_docs (
id bigserial primary key,
collection text not null,
key text not null,
payload jsonb not null,
tags text[] not null default '{}',
created_at timestamptz not null default now(),
updated_at timestamptz not null default now(),
unique (collection, key)
)""")
cur.execute("create index if not exists idx_docs_collection on data_docs (collection)")
con.close()
def do_data_put(collection, data):
"""Сохранить документ: {key, payload, tags}."""
_ensure_doc_table()
key = str(data.get("key") or "").strip()
if not key:
return None, "нужен key"
payload = data.get("payload") or {}
tags = data.get("tags") or []
con = db()
cur = con.cursor()
cur.execute("""insert into data_docs (collection, key, payload, tags)
values (%s, %s, %s, %s)
on conflict (collection, key) do update
set payload = excluded.payload,
tags = excluded.tags,
updated_at = now()
returning id, updated_at""",
(collection, key, json.dumps(payload, ensure_ascii=False), tags))
row = cur.fetchone()
con.close()
return {"id": row[0], "updated_at": str(row[1])}, None
def do_data_list(collection, limit=100, tag=None):
_ensure_doc_table()
con = db()
cur = con.cursor(cursor_factory=psycopg2.extras.RealDictCursor)
if tag:
cur.execute("""select id, key, payload, tags, updated_at from data_docs
where collection=%s and %s = any(tags)
order by updated_at desc limit %s""", (collection, tag, limit))
else:
cur.execute("""select id, key, payload, tags, updated_at from data_docs
where collection=%s order by updated_at desc limit %s""",
(collection, limit))
items = [dict(r) for r in cur.fetchall()]
con.close()
return {"collection": collection, "count": len(items), "items": items}, None
def _jsonable(obj):
if isinstance(obj, dict):
return {k: _jsonable(v) for k, v in obj.items()}
if isinstance(obj, list):
return [_jsonable(v) for v in obj]
return str(obj) if hasattr(obj, "isoformat") else obj
def do_export(collection, fmt="json", filename=None, limit=500):
"""Выгрузить коллекцию в папку проекта."""
data, err = do_data_list(collection, limit=limit)
if err:
return None, err
items = _jsonable(data["items"])
folder = os.path.join(DATA_DIR, collection)
os.makedirs(folder, exist_ok=True)
stamp = datetime.now().strftime("%Y-%m-%d")
if fmt == "md":
name = filename or f"{collection}-{stamp}.md"
lines = [f"# {collection}", "", f"записей: {len(items)}", ""]
for it in items:
lines.append(f"## {it.get('key')}")
lines.append(f"обновлено: {it.get('updated_at')}")
payload = it.get("payload") or {}
for k, v in payload.items():
lines.append(f"* **{k}**: {v}")
lines.append("")
open(os.path.join(folder, name), "w", encoding="utf-8").write("\n".join(lines))
else:
name = filename or f"{collection}-{stamp}.json"
with open(os.path.join(folder, name), "w", encoding="utf-8") as f:
json.dump({"collection": collection, "exported_at": str(datetime.now()),
"count": len(items), "items": items},
f, ensure_ascii=False, indent=1, default=str)
path = os.path.join(folder, name)
return {"path": path, "count": len(items), "format": fmt}, None
def render_panel():
"""HTML-страница: состояние системы в одном месте."""
try:
con = db()
cur = con.cursor()
cur.execute("select count(*) from entities")
entities = cur.fetchone()[0]
cur.execute("select count(*) from events")
events = cur.fetchone()[0]
cur.execute("select count(*) from events where at > now() - interval '24 hours'")
events24 = cur.fetchone()[0]
cur.execute("select count(*) from problems where resolved_at is null")
open_problems = cur.fetchone()[0]
cur.execute("""select check_name, detail, at from problems
where resolved_at is null order by at desc limit 10""")
problems = cur.fetchall()
cur.execute("""select at, source, type, entity->>'title'
from events order by id desc limit 25""")
recent = cur.fetchall()
cur.execute("""select source, count(*) from events
where at > now() - interval '24 hours' group by source order by 2 desc""")
by_source = cur.fetchall()
try:
cur.execute("""select payload from data_docs where collection='inventory'
order by updated_at desc limit 1""")
invrow = cur.fetchone()
inventory = invrow[0] if invrow else None
except Exception:
inventory = None
cur.execute("select count(*) from entity_links")
links = cur.fetchone()[0]
cur.execute("""select at, kind, target, detail, status from actions
order by id desc limit 10""")
actions = cur.fetchall()
con.close()
err = None
except Exception as e:
entities = events = events24 = open_problems = 0
problems, recent, by_source, actions, links, inventory = [], [], [], [], 0, None
err = str(e)[:200]
def esc(x):
return (str(x) if x is not None else "—").replace("&", "&amp;").replace("<", "&lt;")
rows = "".join(
f"<tr><td>{esc(at.strftime('%d.%m %H:%M'))}</td><td>{esc(src)}</td>"
f"<td>{esc(typ)}</td><td>{esc(title)}</td></tr>"
for at, src, typ, title in recent) or "<tr><td colspan=4>событий пока нет</td></tr>"
inv_rows = ""
if isinstance(inventory, dict):
jf = inventory.get("jellyfin") or {}
nv = inventory.get("navidrome") or {}
cb = inventory.get("calibre") or {}
im = inventory.get("immich") or {}
pl = inventory.get("paperless") or {}
fr = inventory.get("frigate") or {}
inv_rows = (
f"<tr><td>Jellyfin</td><td>{jf.get('Movie')} фильмов, {jf.get('Series')} сериалов, "
f"{jf.get('Episode')} серий</td></tr>"
f"<tr><td>Navidrome</td><td>{nv.get('albums')} альбомов, {nv.get('tracks')} треков</td></tr>"
f"<tr><td>Calibre</td><td>{cb.get('books')} книг</td></tr>"
f"<tr><td>Immich</td><td>{im.get('assets')} снимков, {im.get('albums')} альбомов</td></tr>"
f"<tr><td>Paperless</td><td>{pl.get('documents')} документов</td></tr>"
f"<tr><td>Frigate</td><td>{fr.get('cameras')} камер</td></tr>")
if not inv_rows:
inv_rows = '<tr><td colspan=2>инвентаризация ещё не собиралась</td></tr>'
arows = "".join(
f"<tr><td>{esc(at.strftime('%d.%m %H:%M'))}</td><td>{esc(kind)}</td>"
f"<td>{esc(target)}</td><td>{esc(detail)}</td>"
f"<td class=\"{'bad' if st != 'done' else ''}\">{esc(st)}</td></tr>"
for at, kind, target, detail, st in actions) or "<tr><td colspan=5>действий не было</td></tr>"
prow = "".join(
f"<tr><td>{esc(name)}</td><td>{esc(detail)}</td></tr>"
for name, detail, _ in problems) or "<tr><td colspan=2>открытых проблем нет</td></tr>"
srcs = ", ".join(f"{esc(s)}: {n}" for s, n in by_source) or "нет данных"
return f"""<!doctype html><html lang="ru"><meta charset="utf-8">
<title>Buryan Hub</title>
<style>
body{{font-family:system-ui,sans-serif;background:#101418;color:#d8e0e8;margin:0;padding:24px}}
h1{{font-size:20px;margin:0 0 4px}} h2{{font-size:15px;margin:28px 0 8px;color:#8fb4d9}}
.cards{{display:flex;gap:12px;flex-wrap:wrap;margin-top:16px}}
.card{{background:#182027;border-radius:10px;padding:14px 18px;min-width:120px}}
.card b{{display:block;font-size:24px}} .card span{{color:#7d8b99;font-size:12px}}
table{{width:100%;border-collapse:collapse;font-size:13px}}
td,th{{text-align:left;padding:6px 8px;border-bottom:1px solid #223}}
th{{color:#7d8b99;font-weight:500}}
.bad{{color:#ff8a80}} .muted{{color:#7d8b99;font-size:12px}}
</style>
<h1>Buryan Hub</h1>
<div class="muted">ядро событий и карточек · обновлено сейчас</div>
{'<p class="bad">Ошибка базы: ' + esc(err) + '</p>' if err else ''}
<div class="cards">
<div class="card"><b>{entities}</b><span>карточек</span></div>
<div class="card"><b>{events}</b><span>событий всего</span></div>
<div class="card"><b>{events24}</b><span>за 24 часа</span></div>
<div class="card"><b class="{'bad' if open_problems else ''}">{open_problems}</b><span>проблем</span></div>
<div class="card"><b>{links}</b><span>связей</span></div>
</div>
<h2>Проблемы</h2><table><tr><th>проверка</th><th>что не так</th></tr>{prow}</table>
<h2>Что есть в сервисах</h2>
<table><tr><th>сервис</th><th>что есть</th></tr>{inv_rows}</table>
<h2>Действия сверки и автоисправлений</h2>
<table><tr><th>когда</th><th>что</th><th>цель</th><th>результат</th><th>статус</th></tr>{arows}</table>
<h2>Последние события</h2>
<table><tr><th>когда</th><th>источник</th><th>тип</th><th>объект</th></tr>{rows}</table>
<h2>Источники за сутки</h2><div>{srcs}</div>
"""
JELLYFIN_URL = os.environ.get("JELLYFIN_URL", "http://192.168.1.103:8096")
JELLYFIN_KEY = os.environ.get("JELLYFIN_KEY", "")
def _jellyfin_scan():
"""Просим Jellyfin пересканировать библиотеку."""
if not JELLYFIN_KEY:
return False, "нет ключа Jellyfin"
req = urllib.request.Request(
f"{JELLYFIN_URL}/Library/Refresh", data=b"", method="POST",
headers={"Authorization": f'MediaBrowser Token="{JELLYFIN_KEY}"'})
try:
with urllib.request.urlopen(req, timeout=20) as r:
return r.status in (200, 204), f"HTTP {r.status}"
except urllib.error.HTTPError as e:
return False, f"HTTP {e.code}"
except Exception as e:
return False, str(e)[:100]
def _journal(kind, target, detail, status="done", rollback=None):
try:
con = db()
with con.cursor() as cur:
cur.execute("""insert into actions (kind, target, detail, status, rollback)
values (%s,%s,%s,%s,%s)""", (kind, target, detail, status, rollback))
con.close()
except Exception:
pass
def auto_actions(ev, etype):
"""Что сделать сразу после события."""
ent = ev.get("entity") or {}
title = ent.get("title") or ent.get("key") or "объект"
if etype == "media.imported":
ok, detail = _jellyfin_scan()
_journal("scan", "jellyfin", f"после импорта «{title}»: {detail}",
"done" if ok else "failed", None)
return ok
if etype == "media.requested":
# заявку просто фиксируем: одобрение — решение человека
_journal("note", "seerr", f"заявка «{title}» принята", "done", None)
return True
return False
class Handler(BaseHTTPRequestHandler):
def log_message(self, *a):
pass
def _json(self, code, obj):
body = json.dumps(obj, ensure_ascii=False, default=str).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def do_GET(self):
if self.path == "/" or self.path.startswith("/panel"):
body = render_panel().encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if self.path.startswith("/data/"):
import urllib.parse as up
parts = self.path.split("?")[0].split("/data/")
coll = parts[1] if len(parts) > 1 else ""
qs = up.parse_qs(self.path.split("?")[1]) if "?" in self.path else {}
tag = (qs.get("tag") or [None])[0]
limit = int((qs.get("limit") or ["100"])[0])
try:
res, err = do_data_list(coll, limit, tag)
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
return self._json(200 if not err else 500, res or {"error": err})
if self.path.startswith("/health"):
try:
with db() as con, con.cursor() as cur:
cur.execute("select count(*) from events")
n = cur.fetchone()[0]
return self._json(200, {"ok": True, "events": n})
except Exception as e:
return self._json(500, {"ok": False, "error": str(e)[:200]})
if self.path.startswith("/search"):
import urllib.parse as up
q = (up.parse_qs(self.path.split("?", 1)[1]).get("q") or [""])[0] if "?" in self.path else ""
if not q:
return self._json(400, {"error": "нужен параметр q"})
try:
return self._json(200, do_search(q))
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
if self.path.startswith("/entity/"):
try:
eid = int(self.path.split("/entity/")[1].split("?")[0])
except ValueError:
return self._json(400, {"error": "нужен числовой id"})
try:
data = do_entity(eid)
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
if data is None:
return self._json(404, {"error": "карточка не найдена"})
return self._json(200, data)
if self.path.startswith("/entities"):
q = ""
if "?" in self.path:
import urllib.parse
q = (urllib.parse.parse_qs(self.path.split("?", 1)[1]).get("q") or [""])[0]
try:
with db() as con, con.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
if q:
cur.execute("""select id, kind, title, status, external_ids, updated_at
from entities where search_text like %s
order by updated_at desc limit 20""", (f"%{q.lower()}%",))
else:
cur.execute("""select id, kind, title, status, external_ids, updated_at
from entities order by updated_at desc limit 20""")
return self._json(200, {"items": [dict(r) for r in cur.fetchall()]})
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
if self.path.startswith("/events"):
try:
with db() as con, con.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
cur.execute("""select id, at, source, type, entity, payload
from events order by id desc limit 30""")
return self._json(200, {"items": [dict(r) for r in cur.fetchall()]})
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
return self._json(404, {"error": "нет такого пути"})
def do_POST(self):
length = int(self.headers.get("Content-Length", 0))
raw = self.rfile.read(length) if length else b"{}"
try:
data = json.loads(raw or b"{}")
except json.JSONDecodeError:
return self._json(400, {"error": "не JSON"})
if self.path.startswith("/event"):
return self._event(data)
if self.path.startswith("/entity"):
return self._entity(data)
if self.path.startswith("/link"):
return self._link(data)
if self.path.startswith("/data/"):
coll = self.path.split("/data/")[1].split("?")[0]
if not coll:
return self._json(400, {"error": "нужна коллекция"})
res, err = do_data_put(coll, data)
return self._json(201 if not err else 500, res or {"error": err})
if self.path.startswith("/export"):
coll = data.get("collection") or ""
if not coll:
return self._json(400, {"error": "нужна collection"})
res, err = do_export(coll, data.get("format") or "json", data.get("filename"))
return self._json(200 if not err else 500, res or {"error": err})
return self._json(404, {"error": "нет такого пути"})
# ── события ──
def _event(self, ev):
src = (ev.get("source") or "").strip()
typ = (ev.get("type") or "").strip()
if not (src and typ):
return self._json(400, {"error": "нужны source и type"})
now = datetime.now(timezone.utc)
ent = ev.get("entity") or {}
payload = ev.get("payload") or {}
try:
with db() as con, con.cursor() as cur:
cur.execute(
"""insert into events (at, source, type, entity, payload, actor)
values (%s, %s, %s, %s, %s, %s) returning id""",
(now, src, typ, json.dumps(ent, ensure_ascii=False),
json.dumps(payload, ensure_ascii=False), ev.get("actor") or "system"))
eid = cur.fetchone()[0]
# событие, связанное с сущностью, обновляет её статус
if ent.get("kind") and ent.get("key"):
cur.execute(
"""insert into entities (kind, key, title, status, external_ids, search_text, updated_at)
values (%s, %s, %s, %s, %s, %s, %s)
on conflict (kind, key) do update
set status = excluded.status,
title = coalesce(excluded.title, entities.title),
updated_at = excluded.updated_at
returning id""",
(ent["kind"], ent["key"], ent.get("title") or ent["key"],
typ, json.dumps({src: ent.get("key")}, ensure_ascii=False),
f"{ent.get('title') or ''} {ent.get('key')}".lower(), now))
ent_id = cur.fetchone()[0]
cur.execute("update events set entity_id = %s where id = %s", (ent_id, eid))
threading.Thread(target=auto_actions, args=(ev, typ), daemon=True).start()
if typ in LOUD:
threading.Thread(target=publish, args=(human_event(ev),), daemon=True).start()
return self._json(201, {"ok": True, "id": eid})
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
# ── сущности ──
def _entity(self, data):
kind = (data.get("kind") or "").strip()
key = (data.get("key") or "").strip()
if not (kind and key):
return self._json(400, {"error": "нужны kind и key"})
now = datetime.now(timezone.utc)
title = data.get("title") or key
try:
with db() as con, con.cursor() as cur:
cur.execute(
"""insert into entities (kind, key, title, status, external_ids, search_text, updated_at)
values (%s, %s, %s, %s, %s, %s, %s)
on conflict (kind, key) do update
set title = excluded.title,
status = coalesce(excluded.status, entities.status),
external_ids = entities.external_ids || excluded.external_ids,
search_text = excluded.search_text,
updated_at = excluded.updated_at
returning id""",
(kind, key, title, data.get("status"),
json.dumps(data.get("external_ids") or {}, ensure_ascii=False),
f"{title} {key}".lower(), now))
rid = cur.fetchone()[0]
return self._json(201, {"ok": True, "id": rid})
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
def _link(self, data):
a, b = data.get("from"), data.get("to")
rel = data.get("relation") or "related"
if not (a and b):
return self._json(400, {"error": "нужны from и to (id)"})
try:
with db() as con, con.cursor() as cur:
cur.execute(
"""insert into entity_links (from_id, to_id, relation)
values (%s, %s, %s) on conflict do nothing returning id""",
(a, b, rel))
row = cur.fetchone()
return self._json(201, {"ok": True, "id": row[0] if row else None})
except Exception as e:
return self._json(500, {"error": str(e)[:200]})
def main():
ensure_schema()
srv = HTTPServer(("0.0.0.0", PORT), Handler)
print(f"hub-core слушает {PORT}", flush=True)
srv.serve_forever()
if __name__ == "__main__":
main()