Files
buryan-infra/ops/hub/search_index.py

106 lines
3.8 KiB
Python

#!/usr/bin/env python3
"""Индекс поиска: книги, документы, рецепты (SQLite FTS5)."""
import json
import os
import sqlite3
import subprocess
import urllib.request
DB = "/var/lib/hub-search/index.db"
DUMP_BOOKS = '''import sqlite3, json
c = sqlite3.connect("/opt/books/library/metadata.db")
out = []
for bid, title, author, path in c.execute(
"select id,title,author_sort,path from books").fetchall():
authors = [r[0] for r in c.execute(
"select a.name from authors a join books_authors_link l on l.author=a.id "
"where l.book=?", (bid,))]
series = [s[0] for s in c.execute(
"select s.name from series s join books_series_link l on l.series=s.id "
"where l.book=?", (bid,)).fetchall()]
out.append({"id": bid, "title": title, "authors": authors,
"series": series, "path": path})
print(json.dumps(out, ensure_ascii=False))
'''
def books():
open("/tmp/dump_books.py", "w", encoding="utf-8").write(DUMP_BOOKS)
subprocess.run(["pct", "push", "111", "/tmp/dump_books.py", "/tmp/dump_books.py",
"--perms", "644"], capture_output=True, timeout=60)
r = subprocess.run(["pct", "exec", "111", "--", "python3", "/tmp/dump_books.py"],
capture_output=True, text=True, timeout=120)
try:
return json.loads(r.stdout or "[]")
except Exception:
return []
def documents():
tok = open("/root/.paperless-token", encoding="utf-8").read().strip()
rows, page = [], 1
while page <= 5:
req = urllib.request.Request(
f"http://192.168.1.105:8000/api/documents/?page_size=100&page={page}",
headers={"Authorization": "Token " + tok})
try:
with urllib.request.urlopen(req, timeout=30) as r:
d = json.loads(r.read() or b"{}")
except Exception:
break
for doc in d.get("results", []):
rows.append((doc.get("id"), doc.get("title") or "",
(doc.get("content") or "")[:3000]))
if not d.get("next"):
break
page += 1
return rows
def recipes():
sql = "select slug, name, coalesce(description,'') from recipes"
r = subprocess.run(
["bash", "-lc",
f"PGPASSWORD='{open('/root/.pg-superuser-pass').read().strip()}' psql -h 192.168.1.151 "
f"-U postgres -d mealie_db -t -A -F '|||' -c \"{sql}\""],
capture_output=True, text=True, timeout=60)
out = []
for line in (r.stdout or "").splitlines():
parts = line.split("|||")
if len(parts) >= 2 and parts[0].strip():
out.append((parts[0], parts[1], parts[2] if len(parts) > 2 else ""))
return out
def main():
os.makedirs(os.path.dirname(DB), exist_ok=True)
c = sqlite3.connect(DB)
c.execute("""create virtual table if not exists docs using fts5(
source, ref, title, body, tokenize='unicode61 remove_diacritics 2')""")
c.execute("delete from docs")
b = books()
c.executemany("insert into docs(source,ref,title,body) values(?,?,?,?)",
[("book", str(x["id"]), x["title"] or "",
" ".join(x.get("authors") or []) + " " +
" ".join(x.get("series") or []) + " " + (x.get("path") or ""))
for x in b])
d = documents()
c.executemany("insert into docs(source,ref,title,body) values(?,?,?,?)",
[("doc", str(x[0]), x[1], x[2]) for x in d])
r = recipes()
c.executemany("insert into docs(source,ref,title,body) values(?,?,?,?)",
[("recipe", str(x[0]), x[1], x[2]) for x in r])
c.commit()
total = c.execute("select count(*) from docs").fetchone()[0]
c.close()
print(f"индекс: {total} (книги {len(b)}, документы {len(d)}, рецепты {len(r)})")
if __name__ == "__main__":
main()