#!/usr/bin/env python3 """RAG-индекс Buryans Hub в PostgreSQL (pgvector). Единственный индексатор: раньше их было два (SQLite и Postgres), и агент искал в пустой базе. Теперь одна база — Postgres на 192.168.1.151. Хранит: rag_docs(path, mtime, size, hash) — что уже проиндексировано chunks(source, ref, text, embedding) — фрагменты с векторами (pgvector) Использование: rag_index_pg.py # инкрементально rag_index_pg.py --rebuild # пересобрать rag_index_pg.py --seed # только запомнить хеши (без эмбеддингов) """ import hashlib import json import os import sys import time import urllib.request sys.path.insert(0, "/opt/hub-core") import rag # noqa: E402 import psycopg2 # noqa: E402 LLM = "http://192.168.1.229:11434" EMB_MODEL = "nomic-embed-text" ROOT = "/root/buryan-infra" DIRS = ["ops", "docs", "deploy"] EXTRA_FILES = ["AGENTS.md", "README.md"] CHUNK = 900 OVERLAP = 120 SOURCE = "ops-rag" def embed(text): body = json.dumps({"model": EMB_MODEL, "prompt": text[:4000]}).encode() req = urllib.request.Request(LLM + "/api/embeddings", data=body, method="POST") req.add_header("Content-Type", "application/json") for attempt in range(3): try: with urllib.request.urlopen(req, timeout=180) as r: return json.loads(r.read().decode()).get("embedding") or [] except Exception: if attempt == 2: return [] time.sleep(3) return [] def collect_files(): files = [] for d in DIRS: base = os.path.join(ROOT, d) if not os.path.isdir(base): continue for dirpath, dirnames, filenames in os.walk(base): dirnames[:] = [x for x in dirnames if x not in (".git", "node_modules", "__pycache__", "rag")] for fn in filenames: if fn.endswith((".md", ".txt", ".yml", ".yaml")) and not fn.startswith("."): p = os.path.join(dirpath, fn) if os.path.getsize(p) < 200: continue files.append(p) journal = "/var/log/hermes/actions.jsonl" if os.path.isfile(journal): try: eps = [] lines = open(journal, encoding="utf-8", errors="ignore").readlines()[-400:] for ln in lines: try: e = json.loads(ln) except Exception: continue if e.get("event") not in ("done", "failed", "denied"): continue eps.append("%s | %s | %s | %s" % ( e.get("ts", ""), e.get("rule", ""), e.get("event", ""), str(e.get("reason") or e.get("detail") or "")[:200])) if eps: ep = ROOT + "/ops/rag/episodes-actions.txt" os.makedirs(os.path.dirname(ep), exist_ok=True) open(ep, "w", encoding="utf-8").write("\n".join(eps)) files.append(ep) except Exception: pass for f in EXTRA_FILES: p = os.path.join(ROOT, f) if os.path.isfile(p): files.append(p) return sorted(set(files)) def chunk_text(text): text = text.replace("\r\n", "\n") out, i = [], 0 while i < len(text): piece = text[i:i + CHUNK] if piece.strip(): out.append(piece) i += CHUNK - OVERLAP return out def vec_lit(v): return "[" + ",".join("%.6f" % x for x in v) + "]" def main(): rebuild = "--rebuild" in sys.argv seed = "--seed" in sys.argv pg = psycopg2.connect(rag.DSN) cur = pg.cursor() cur.execute("""CREATE TABLE IF NOT EXISTS rag_docs( path TEXT PRIMARY KEY, mtime DOUBLE PRECISION, size BIGINT, hash TEXT)""") pg.commit() if rebuild: cur.execute("DELETE FROM chunks WHERE source=%s", (SOURCE,)) cur.execute("DELETE FROM rag_docs") pg.commit() files = collect_files() cur.execute("SELECT path, mtime, size, hash FROM rag_docs") known = {r[0]: r for r in cur.fetchall()} added = skipped = failed = chunks = 0 for p in files: rel = os.path.relpath(p, ROOT) try: st = os.stat(p) raw = open(p, encoding="utf-8", errors="ignore").read() except Exception: failed += 1 continue h = hashlib.sha1(raw.encode("utf-8", "ignore")).hexdigest() old = known.get(rel) if old and old[3] == h and not rebuild: skipped += 1 continue if seed: cur.execute("INSERT INTO rag_docs(path,mtime,size,hash) VALUES(%s,%s,%s,%s) " "ON CONFLICT (path) DO UPDATE SET mtime=EXCLUDED.mtime, " "size=EXCLUDED.size, hash=EXCLUDED.hash", (rel, st.st_mtime, st.st_size, h)) pg.commit() skipped += 1 continue cur.execute("DELETE FROM chunks WHERE source=%s AND ref LIKE %s", (SOURCE, rel + "#%")) for idx, piece in enumerate(chunk_text(raw)): v = embed(piece) if not v: failed += 1 continue cur.execute("INSERT INTO chunks(source,ref,text,embedding) VALUES(%s,%s,%s,%s) " "ON CONFLICT (source,ref) DO UPDATE SET text=EXCLUDED.text, " "embedding=EXCLUDED.embedding, updated=now()", (SOURCE, "%s#%s" % (rel, idx), piece, vec_lit(v))) chunks += 1 cur.execute("INSERT INTO rag_docs(path,mtime,size,hash) VALUES(%s,%s,%s,%s) " "ON CONFLICT (path) DO UPDATE SET mtime=EXCLUDED.mtime, " "size=EXCLUDED.size, hash=EXCLUDED.hash", (rel, st.st_mtime, st.st_size, h)) pg.commit() added += 1 if added % 10 == 0: print(" файлов обновлено: %d (чанков %d)" % (added, chunks)) cur.execute("SELECT count(*) FROM chunks WHERE source=%s", (SOURCE,)) total = cur.fetchone()[0] pg.close() print("итог: обновлено %d, пропущено %d, ошибок %d | чанков: %d" % (added, skipped, failed, total)) return 0 if __name__ == "__main__": raise SystemExit(main())