Hub Core: шина событий и граф сущностей в PostgreSQL
Первый шаг проекта: база hub_core (UTF8) на общем PostgreSQL, служба hub-core на порту 9120. Умеет: принимать события от сервисов (POST /event), вести карточки сущностей (kind + key, статус, внешние ключи), связывать карточки, искать и показывать последние события, публиковать важное в личку владельцу через Matrix. Проверено: событие radarr/media.imported записалось, карточка «фильм | Дюна: часть вторая» создалась с кириллицей. По пути: systemd разбивал DSN по пробелам (нужны кавычки), база была создана в SQL_ASCII — пересоздана в UTF8.
This commit is contained in:
1 parent
92bc3aa4e9
commit
125ca73696
4 files changed
+328
No files matched your search
@@ -0,0 +1,26 @@
|
||||
# Buryan Hub Core
|
||||
|
||||
Шина событий и граф сущностей.
|
||||
|
||||
## Что делает
|
||||
|
||||
* `POST /event` — принять событие от сервиса, записать в `events`,
|
||||
обновить карточку сущности, показать важное в Matrix.
|
||||
* `POST /entity` — создать/обновить карточку (kind + key).
|
||||
* `POST /link` — связать две карточки (например, фильм и книга).
|
||||
* `GET /entities?q=` — поиск по карточкам.
|
||||
* `GET /events` — последние события.
|
||||
* `GET /health` — состояние.
|
||||
|
||||
## База
|
||||
|
||||
PostgreSQL на CT 151, база `hub_core` (UTF8), таблицы:
|
||||
`entities`, `entity_links`, `events`, `sync_state`.
|
||||
|
||||
## Служба
|
||||
|
||||
`/etc/systemd/system/hub-core.service`, порт 9120,
|
||||
лог `/var/log/hub-core.log`, настройки `/etc/hub-core/env`.
|
||||
|
||||
Важно: в systemd DSN должен быть в кавычках (иначе разбивается по пробелам),
|
||||
и база обязательно UTF8 — иначе кириллица падает с UnicodeEncodeError.
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Buryan Hub Core: шина событий и граф сущностей.
|
||||
|
||||
Один процесс, три задачи:
|
||||
1) принимать события от сервисов и складывать в общую базу;
|
||||
2) хранить карточки сущностей и связи между ними;
|
||||
3) публиковать значимое в Matrix (личка владельцу).
|
||||
|
||||
Запуск: python3 core.py (HTTP на порту 9120)
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
|
||||
DSN = os.environ.get("CORE_DSN", "dbname=hub_core user=hub_core password=hub_core host=127.0.0.1")
|
||||
PORT = int(os.environ.get("CORE_PORT", "9120"))
|
||||
MATRIX_HS = os.environ.get("MATRIX_HS", "https://matrix.buryans.ru")
|
||||
NOTIFY_TOKEN = os.environ.get("NOTIFY_TOKEN", "")
|
||||
NOTIFY_ROOM = os.environ.get("NOTIFY_ROOM", "")
|
||||
|
||||
# какие события стоит показывать человеку сразу
|
||||
LOUD = {
|
||||
"media.imported", "media.requested", "media.failed",
|
||||
"backup.done", "backup.failed", "service.down", "service.up",
|
||||
"photo.imported", "book.added", "document.added",
|
||||
}
|
||||
|
||||
|
||||
def db():
|
||||
con = psycopg2.connect(DSN, connect_timeout=10)
|
||||
con.autocommit = True
|
||||
return con
|
||||
|
||||
|
||||
def ensure_schema():
|
||||
sql = open(os.path.join(os.path.dirname(__file__), "schema.sql"), encoding="utf-8").read()
|
||||
with db() as con, con.cursor() as cur:
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
def publish(text):
|
||||
"""Короткое уведомление в личку владельцу."""
|
||||
if not (NOTIFY_TOKEN and NOTIFY_ROOM):
|
||||
return
|
||||
txn = int(datetime.now().timestamp() * 1000)
|
||||
import urllib.parse
|
||||
rid = urllib.parse.quote(NOTIFY_ROOM, safe="")
|
||||
body = json.dumps({"msgtype": "m.text", "body": text,
|
||||
"format": "org.matrix.custom.html",
|
||||
"formatted_body": text.replace("\n", "<br>")}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{MATRIX_HS}/_matrix/client/v3/rooms/{rid}/send/m.room.message/{txn}",
|
||||
data=body, method="PUT",
|
||||
headers={"Authorization": f"Bearer {NOTIFY_TOKEN}", "Content-Type": "application/json"})
|
||||
try:
|
||||
urllib.request.urlopen(req, timeout=10)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def human_event(ev):
|
||||
e = ev.get("entity") or {}
|
||||
title = e.get("title") or e.get("key") or "объект"
|
||||
t = ev.get("type", "")
|
||||
icons = {"media.imported": "🎬", "media.requested": "🕓", "media.failed": "⚠️",
|
||||
"backup.done": "💾", "backup.failed": "⚠️",
|
||||
"service.down": "🔴", "service.up": "✅",
|
||||
"photo.imported": "📷", "book.added": "📚", "document.added": "📄"}
|
||||
icon = icons.get(t, "•")
|
||||
src = ev.get("source", "?")
|
||||
extra = ev.get("payload") or {}
|
||||
line = f"{icon} **{title}** — {t} ({src})"
|
||||
if extra.get("detail"):
|
||||
line += f"\n{extra['detail']}"
|
||||
return line
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
def _json(self, code, obj):
|
||||
body = json.dumps(obj, ensure_ascii=False).encode()
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self):
|
||||
if self.path.startswith("/health"):
|
||||
try:
|
||||
with db() as con, con.cursor() as cur:
|
||||
cur.execute("select count(*) from events")
|
||||
n = cur.fetchone()[0]
|
||||
return self._json(200, {"ok": True, "events": n})
|
||||
except Exception as e:
|
||||
return self._json(500, {"ok": False, "error": str(e)[:200]})
|
||||
if self.path.startswith("/entities"):
|
||||
q = ""
|
||||
if "?" in self.path:
|
||||
import urllib.parse
|
||||
q = (urllib.parse.parse_qs(self.path.split("?", 1)[1]).get("q") or [""])[0]
|
||||
try:
|
||||
with db() as con, con.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
if q:
|
||||
cur.execute("""select id, kind, title, status, external_ids, updated_at
|
||||
from entities where search_text like %s
|
||||
order by updated_at desc limit 20""", (f"%{q.lower()}%",))
|
||||
else:
|
||||
cur.execute("""select id, kind, title, status, external_ids, updated_at
|
||||
from entities order by updated_at desc limit 20""")
|
||||
return self._json(200, {"items": [dict(r) for r in cur.fetchall()]})
|
||||
except Exception as e:
|
||||
return self._json(500, {"error": str(e)[:200]})
|
||||
if self.path.startswith("/events"):
|
||||
try:
|
||||
with db() as con, con.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur:
|
||||
cur.execute("""select id, at, source, type, entity, payload
|
||||
from events order by id desc limit 30""")
|
||||
return self._json(200, {"items": [dict(r) for r in cur.fetchall()]})
|
||||
except Exception as e:
|
||||
return self._json(500, {"error": str(e)[:200]})
|
||||
return self._json(404, {"error": "нет такого пути"})
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
raw = self.rfile.read(length) if length else b"{}"
|
||||
try:
|
||||
data = json.loads(raw or b"{}")
|
||||
except json.JSONDecodeError:
|
||||
return self._json(400, {"error": "не JSON"})
|
||||
|
||||
if self.path.startswith("/event"):
|
||||
return self._event(data)
|
||||
if self.path.startswith("/entity"):
|
||||
return self._entity(data)
|
||||
if self.path.startswith("/link"):
|
||||
return self._link(data)
|
||||
return self._json(404, {"error": "нет такого пути"})
|
||||
|
||||
# ── события ──
|
||||
def _event(self, ev):
|
||||
src = (ev.get("source") or "").strip()
|
||||
typ = (ev.get("type") or "").strip()
|
||||
if not (src and typ):
|
||||
return self._json(400, {"error": "нужны source и type"})
|
||||
now = datetime.now(timezone.utc)
|
||||
ent = ev.get("entity") or {}
|
||||
payload = ev.get("payload") or {}
|
||||
try:
|
||||
with db() as con, con.cursor() as cur:
|
||||
cur.execute(
|
||||
"""insert into events (at, source, type, entity, payload, actor)
|
||||
values (%s, %s, %s, %s, %s, %s) returning id""",
|
||||
(now, src, typ, json.dumps(ent, ensure_ascii=False),
|
||||
json.dumps(payload, ensure_ascii=False), ev.get("actor") or "system"))
|
||||
eid = cur.fetchone()[0]
|
||||
|
||||
# событие, связанное с сущностью, обновляет её статус
|
||||
if ent.get("kind") and ent.get("key"):
|
||||
cur.execute(
|
||||
"""insert into entities (kind, key, title, status, external_ids, search_text, updated_at)
|
||||
values (%s, %s, %s, %s, %s, %s, %s)
|
||||
on conflict (kind, key) do update
|
||||
set status = excluded.status,
|
||||
title = coalesce(excluded.title, entities.title),
|
||||
updated_at = excluded.updated_at
|
||||
returning id""",
|
||||
(ent["kind"], ent["key"], ent.get("title") or ent["key"],
|
||||
typ, json.dumps({src: ent.get("key")}, ensure_ascii=False),
|
||||
f"{ent.get('title') or ''} {ent.get('key')}".lower(), now))
|
||||
ent_id = cur.fetchone()[0]
|
||||
cur.execute("update events set entity_id = %s where id = %s", (ent_id, eid))
|
||||
if typ in LOUD:
|
||||
threading.Thread(target=publish, args=(human_event(ev),), daemon=True).start()
|
||||
return self._json(201, {"ok": True, "id": eid})
|
||||
except Exception as e:
|
||||
return self._json(500, {"error": str(e)[:200]})
|
||||
|
||||
# ── сущности ──
|
||||
def _entity(self, data):
|
||||
kind = (data.get("kind") or "").strip()
|
||||
key = (data.get("key") or "").strip()
|
||||
if not (kind and key):
|
||||
return self._json(400, {"error": "нужны kind и key"})
|
||||
now = datetime.now(timezone.utc)
|
||||
title = data.get("title") or key
|
||||
try:
|
||||
with db() as con, con.cursor() as cur:
|
||||
cur.execute(
|
||||
"""insert into entities (kind, key, title, status, external_ids, search_text, updated_at)
|
||||
values (%s, %s, %s, %s, %s, %s, %s)
|
||||
on conflict (kind, key) do update
|
||||
set title = excluded.title,
|
||||
status = coalesce(excluded.status, entities.status),
|
||||
external_ids = entities.external_ids || excluded.external_ids,
|
||||
search_text = excluded.search_text,
|
||||
updated_at = excluded.updated_at
|
||||
returning id""",
|
||||
(kind, key, title, data.get("status"),
|
||||
json.dumps(data.get("external_ids") or {}, ensure_ascii=False),
|
||||
f"{title} {key}".lower(), now))
|
||||
rid = cur.fetchone()[0]
|
||||
return self._json(201, {"ok": True, "id": rid})
|
||||
except Exception as e:
|
||||
return self._json(500, {"error": str(e)[:200]})
|
||||
|
||||
def _link(self, data):
|
||||
a, b = data.get("from"), data.get("to")
|
||||
rel = data.get("relation") or "related"
|
||||
if not (a and b):
|
||||
return self._json(400, {"error": "нужны from и to (id)"})
|
||||
try:
|
||||
with db() as con, con.cursor() as cur:
|
||||
cur.execute(
|
||||
"""insert into entity_links (from_id, to_id, relation)
|
||||
values (%s, %s, %s) on conflict do nothing returning id""",
|
||||
(a, b, rel))
|
||||
row = cur.fetchone()
|
||||
return self._json(201, {"ok": True, "id": row[0] if row else None})
|
||||
except Exception as e:
|
||||
return self._json(500, {"error": str(e)[:200]})
|
||||
|
||||
|
||||
def main():
|
||||
ensure_schema()
|
||||
srv = HTTPServer(("0.0.0.0", PORT), Handler)
|
||||
print(f"hub-core слушает {PORT}", flush=True)
|
||||
srv.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Buryan Hub Core — шина событий и граф сущностей
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
Environment="CORE_DSN=dbname=hub_core user=hub_core password=hub_core host=192.168.1.151 client_encoding=utf8"
|
||||
Environment=CORE_PORT=9120
|
||||
EnvironmentFile=-/etc/hub-core/env
|
||||
WorkingDirectory=/opt/hub-core
|
||||
ExecStart=/usr/bin/python3 /opt/hub-core/core.py
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=append:/var/log/hub-core.log
|
||||
StandardError=append:/var/log/hub-core.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,44 @@
|
||||
-- Ядро Buryan Hub: сущности, связи, события.
|
||||
create table if not exists entities (
|
||||
id bigserial primary key,
|
||||
kind text not null, -- фильм, сериал, книга, фото, документ, закладка, устройство
|
||||
key text not null, -- tmdb:693134, calibre:88
|
||||
title text not null,
|
||||
status text, -- последнее известное состояние
|
||||
external_ids jsonb not null default '{}'::jsonb,
|
||||
search_text text not null default '',
|
||||
updated_at timestamptz not null default now(),
|
||||
unique (kind, key)
|
||||
);
|
||||
create index if not exists idx_entities_search on entities using gin (to_tsvector('simple', search_text));
|
||||
create index if not exists idx_entities_kind on entities (kind);
|
||||
|
||||
create table if not exists entity_links (
|
||||
id bigserial primary key,
|
||||
from_id bigint not null references entities(id) on delete cascade,
|
||||
to_id bigint not null references entities(id) on delete cascade,
|
||||
relation text not null default 'related',
|
||||
created_at timestamptz not null default now(),
|
||||
unique (from_id, to_id, relation)
|
||||
);
|
||||
create index if not exists idx_links_from on entity_links (from_id);
|
||||
create index if not exists idx_links_to on entity_links (to_id);
|
||||
|
||||
create table if not exists events (
|
||||
id bigserial primary key,
|
||||
at timestamptz not null default now(),
|
||||
source text not null, -- sonarr, radarr, seerr, jellyfin, immich, alertmanager
|
||||
type text not null, -- media.imported, service.down
|
||||
entity jsonb not null default '{}'::jsonb,
|
||||
payload jsonb not null default '{}'::jsonb,
|
||||
actor text not null default 'system',
|
||||
entity_id bigint references entities(id) on delete set null
|
||||
);
|
||||
create index if not exists idx_events_at on events (at desc);
|
||||
create index if not exists idx_events_source on events (source);
|
||||
|
||||
create table if not exists sync_state (
|
||||
key text primary key,
|
||||
value jsonb not null default '{}'::jsonb,
|
||||
updated_at timestamptz not null default now()
|
||||
);
|
||||
Reference in new issue
Block a user