Источников стало девять: jellyfin, sonarr, radarr, navidrome, calibre, linkding, paperless, immich (альбомы), seerr (заявки — видно, что просили, даже если ещё не скачано). Все девять собираются без ошибок (проверено). По пути поправил Navidrome (в схеме нет artist_id, есть album_artist) и Immich (psql не понимает кириллицу в именах колонок без кавычек — сделал помощник на sh). Недельная сводка: понедельник 09:30, отдельный файл weekly.py.
297 lines
11 KiB
Python
297 lines
11 KiB
Python
"""Сквозной индекс контента: всё, что есть в сервисах, в одной базе.
|
|
|
|
Зачем: искать по одному месту, а не по семи, и видеть «где что лежит».
|
|
Индекс строится по расписанию и по команде; каждый источник опционален —
|
|
если сервис недоступен, остальные всё равно попадают в базу.
|
|
|
|
Выгрузки из баз делают маленькие скрипты-помощники (hub/helpers): они
|
|
кладутся в контейнеры при развёртывании и печатают JSON.
|
|
"""
|
|
import datetime
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
|
|
from . import local, util
|
|
from .sources import CFG, RADARR, SONARR, Jellyfin
|
|
|
|
DB_PATH = os.environ.get("INDEX_DB", "/var/lib/hub-bots/index.db")
|
|
|
|
HELPER_LOCAL = "/opt/hub-helpers"
|
|
|
|
SCHEMA = """
|
|
create table if not exists items (
|
|
id integer primary key autoincrement,
|
|
kind text not null,
|
|
source text not null,
|
|
title text not null,
|
|
subtitle text,
|
|
url text,
|
|
external_id text,
|
|
updated_at text,
|
|
search_text text,
|
|
unique(source, external_id)
|
|
);
|
|
create index if not exists idx_items_title on items(title);
|
|
create index if not exists idx_items_kind on items(kind);
|
|
create table if not exists meta (key text primary key, value text);
|
|
"""
|
|
|
|
|
|
def _connect():
|
|
os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
|
|
con = sqlite3.connect(DB_PATH, timeout=20)
|
|
con.executescript(SCHEMA)
|
|
# миграция для баз, созданных до появления поиска
|
|
cols = {r[1] for r in con.execute("pragma table_info(items)")}
|
|
if "search_text" not in cols:
|
|
con.execute("alter table items add column search_text text")
|
|
con.commit()
|
|
return con
|
|
|
|
|
|
def _norm(title, subtitle):
|
|
"""Приводим к нижнему регистру заранее: SQLite не умеет lower() для кириллицы."""
|
|
return f"{title or ''} {subtitle or ''}".strip().lower()
|
|
|
|
|
|
def _year(*values):
|
|
for v in values:
|
|
if v:
|
|
return str(v)[:4]
|
|
return ""
|
|
|
|
|
|
# ─────────────────────────── источники ───────────────────────────
|
|
|
|
def _jellyfin():
|
|
data, err = util.get(
|
|
f"{CFG.jellyfin_url}/Items?Recursive=true&Limit=5000"
|
|
f"&IncludeItemTypes=Movie,Series,Audio&Fields=ProductionYear&SortBy=SortName",
|
|
{"Authorization": f'MediaBrowser Token="{CFG.jellyfin_key}"'}, ttl=600)
|
|
if err:
|
|
return [], err
|
|
kinds = {"Movie": "фильм", "Series": "сериал", "Audio": "трек"}
|
|
rows = []
|
|
for it in (data or {}).get("Items", []):
|
|
rows.append({
|
|
"kind": kinds.get(it.get("Type"), "медиа"),
|
|
"source": "jellyfin",
|
|
"title": it.get("Name", ""),
|
|
"subtitle": _year(it.get("ProductionYear")),
|
|
"url": f"https://media.buryans.ru/web/index.html#!/details?id={it.get('Id')}",
|
|
"external_id": str(it.get("Id")),
|
|
})
|
|
return rows, None
|
|
|
|
|
|
def _sonarr():
|
|
items, err = SONARR.series()
|
|
if err:
|
|
return [], err
|
|
rows = [{
|
|
"kind": "сериал", "source": "sonarr",
|
|
"title": s.get("title", ""), "subtitle": _year(s.get("year")),
|
|
"url": "https://sonarr.buryans.ru/series", "external_id": str(s.get("id")),
|
|
} for s in items or []]
|
|
return rows, None
|
|
|
|
|
|
def _radarr():
|
|
items, err = RADARR.movies()
|
|
if err:
|
|
return [], err
|
|
rows = [{
|
|
"kind": "фильм", "source": "radarr",
|
|
"title": m.get("title", ""), "subtitle": _year(m.get("year")),
|
|
"url": "https://radarr.buryans.ru/movie", "external_id": str(m.get("id")),
|
|
} for m in items or []]
|
|
return rows, None
|
|
|
|
|
|
def _navidrome():
|
|
out, err = local.pct_exec(125, ["docker", "exec", "navidrome", "sqlite3",
|
|
"-separator", "|", "/data/navidrome.db",
|
|
"select ifnull(album_artist,''), name, id from album limit 5000"])
|
|
if err:
|
|
return [], err
|
|
rows = []
|
|
for line in (out or "").splitlines():
|
|
parts = line.split("|")
|
|
if len(parts) >= 3:
|
|
rows.append({
|
|
"kind": "музыка", "source": "navidrome",
|
|
"title": parts[1], "subtitle": parts[0],
|
|
"url": "https://music.buryans.ru", "external_id": parts[2],
|
|
})
|
|
return rows, None
|
|
|
|
|
|
def _helper(ct, script, kind, source, make_row):
|
|
out, err = local.pct_exec(ct, ["python3", f"{HELPER_LOCAL}/{script}"], timeout=40)
|
|
if err:
|
|
return [], err
|
|
try:
|
|
data = json.loads(out or "[]")
|
|
except json.JSONDecodeError:
|
|
return [], "не разобрал ответ помощника"
|
|
rows = []
|
|
for rec in data:
|
|
r = make_row(rec)
|
|
if r and r.get("title"):
|
|
r.update({"kind": kind, "source": source})
|
|
rows.append(r)
|
|
return rows, None
|
|
|
|
|
|
def _calibre():
|
|
return _helper(111, "calibre_dump.py", "книга", "calibre", lambda rec: {
|
|
"title": rec[1], "subtitle": rec[2],
|
|
"url": f"https://books.buryans.ru/book/{rec[0]}", "external_id": str(rec[0]),
|
|
})
|
|
|
|
|
|
def _linkding():
|
|
return _helper(127, "linkding_dump.py", "закладка", "linkding", lambda rec: {
|
|
"title": rec[1] or rec[2], "subtitle": (rec[3] or "")[:80],
|
|
"url": rec[2], "external_id": str(rec[0]),
|
|
})
|
|
|
|
|
|
def _paperless():
|
|
data, err = util.get(
|
|
f"{CFG.paperless_url}/api/documents/?page_size=500&ordering=-created",
|
|
{"Authorization": f"Token {CFG.paperless_token}"}, ttl=600)
|
|
if err:
|
|
return [], err
|
|
rows = [{
|
|
"kind": "документ", "source": "paperless",
|
|
"title": d.get("title", ""), "subtitle": (d.get("created") or "")[:10],
|
|
"url": f"https://docs.buryans.ru/documents/{d.get('id')}", "external_id": str(d.get("id")),
|
|
} for d in (data or {}).get("results", [])]
|
|
return rows, None
|
|
|
|
|
|
def _immich_albums():
|
|
"""Альбомы Immich: сами снимки названий не имеют, а альбомы имеют."""
|
|
out, err = local.pct_exec(151, ["sh", f"{HELPER_LOCAL}/immich_albums.sh"], timeout=40)
|
|
if err:
|
|
return [], err
|
|
rows = []
|
|
for line in (out or "").splitlines():
|
|
parts = line.split("|")
|
|
if len(parts) >= 3:
|
|
rows.append({
|
|
"kind": "фотоальбом", "source": "immich",
|
|
"title": parts[1], "subtitle": parts[2],
|
|
"url": "https://immich.buryans.ru/albums",
|
|
"external_id": parts[0],
|
|
})
|
|
return rows, None
|
|
|
|
|
|
def _seerr_requests():
|
|
"""Заявки: что просили, даже если ещё не скачано."""
|
|
data, err = util.get(
|
|
f"{CFG.seerr_url}/api/v1/request?take=200&filter=all&sort=added",
|
|
{"X-Api-Key": CFG.seerr_key}, ttl=300)
|
|
if err:
|
|
return [], err
|
|
status = {1: "ожидает", 2: "одобрено", 3: "отклонено", 4: "частично"}
|
|
rows = []
|
|
for r in (data or {}).get("results", []):
|
|
media = r.get("media") or {}
|
|
title = media.get("title") or media.get("name") or ""
|
|
if not title:
|
|
continue
|
|
rows.append({
|
|
"kind": "заявка", "source": "seerr",
|
|
"title": title,
|
|
"subtitle": status.get(r.get("status"), "?"),
|
|
"url": "https://ombi.buryans.ru",
|
|
"external_id": str(r.get("id")),
|
|
})
|
|
return rows, None
|
|
|
|
|
|
SOURCES = {
|
|
"jellyfin": _jellyfin,
|
|
"sonarr": _sonarr,
|
|
"radarr": _radarr,
|
|
"navidrome": _navidrome,
|
|
"calibre": _calibre,
|
|
"linkding": _linkding,
|
|
"paperless": _paperless,
|
|
"immich": _immich_albums,
|
|
"seerr": _seerr_requests,
|
|
}
|
|
|
|
|
|
# ─────────────────────────── работа с индексом ───────────────────────────
|
|
|
|
def rebuild(only=None):
|
|
"""Перестраиваем индекс. Возвращает отчёт по источникам."""
|
|
con = _connect()
|
|
report = {}
|
|
now = datetime.datetime.now().isoformat(timespec="seconds")
|
|
for name, fn in SOURCES.items():
|
|
if only and name not in only:
|
|
continue
|
|
try:
|
|
rows, err = fn()
|
|
except Exception as e:
|
|
rows, err = [], f"{type(e).__name__}: {e}"
|
|
if err:
|
|
report[name] = {"error": str(err)[:120], "count": 0}
|
|
continue
|
|
con.execute("delete from items where source = ?", (name,))
|
|
con.executemany(
|
|
"insert or replace into items "
|
|
"(kind, source, title, subtitle, url, external_id, updated_at, search_text) "
|
|
"values (:kind, :source, :title, :subtitle, :url, :external_id, :updated_at, :search_text)",
|
|
[{**r, "updated_at": now, "search_text": _norm(r["title"], r.get("subtitle"))}
|
|
for r in rows])
|
|
report[name] = {"count": len(rows), "error": None}
|
|
con.execute("insert or replace into meta (key, value) values ('built_at', ?)", (now,))
|
|
con.commit()
|
|
con.close()
|
|
return report
|
|
|
|
|
|
def stats():
|
|
con = _connect()
|
|
total = con.execute("select count(*) from items").fetchone()[0]
|
|
by_kind = con.execute("select kind, count(*) from items group by kind order by 2 desc").fetchall()
|
|
by_source = con.execute("select source, count(*) from items group by source order by 2 desc").fetchall()
|
|
built = con.execute("select value from meta where key='built_at'").fetchone()
|
|
con.close()
|
|
return {"total": total, "kinds": by_kind, "sources": by_source,
|
|
"built_at": built[0] if built else None, "path": DB_PATH}
|
|
|
|
|
|
def search(term, limit=15):
|
|
con = _connect()
|
|
like = f"%{term.strip().lower()}%"
|
|
rows = con.execute(
|
|
"select kind, source, title, subtitle, url from items "
|
|
"where search_text like ? order by kind, title limit ?",
|
|
(like, limit)).fetchall()
|
|
con.close()
|
|
return [{"kind": k, "source": s, "title": t, "subtitle": sub, "url": u}
|
|
for k, s, t, sub, u in rows]
|
|
|
|
|
|
def links(term, limit=30):
|
|
"""Кросс-связи: одно и то же название в разных видах контента."""
|
|
con = _connect()
|
|
like = f"%{term.strip().lower()}%"
|
|
rows = con.execute(
|
|
"select kind, title, subtitle, url from items where search_text like ? "
|
|
"order by kind limit ?",
|
|
(like, limit)).fetchall()
|
|
con.close()
|
|
grouped = {}
|
|
for kind, title, subtitle, url in rows:
|
|
grouped.setdefault(kind, []).append({"title": title, "subtitle": subtitle, "url": url})
|
|
return grouped
|