#!/usr/bin/env python3 """RAG-индекс по документации Buryans Hub (память мозга). Собирает markdown/text из ops/, docs/, deploy/, корня репозитория, разбивает на чанки, считает эмбеддинги локальной моделью nomic-embed-text и складывает в SQLite. Использование: rag_index.py # обновить индекс (инкрементально) rag_index.py --rebuild # пересобрать с нуля """ import hashlib import json import os import struct import sys import time import urllib.request LLM = "http://192.168.1.229:11434" EMB_MODEL = "nomic-embed-text" ROOT = "/root/buryan-infra" DB = ROOT + "/ops/rag/index.db" DIRS = ["ops", "docs", "deploy"] EXTRA_FILES = ["AGENTS.md", "README.md"] CHUNK = 900 OVERLAP = 120 def embed(text): body = json.dumps({"model": EMB_MODEL, "prompt": text[:4000]}).encode() req = urllib.request.Request(LLM + "/api/embeddings", data=body, method="POST") req.add_header("Content-Type", "application/json") for attempt in range(3): try: with urllib.request.urlopen(req, timeout=120) as r: return json.loads(r.read().decode()).get("embedding") or [] except Exception: if attempt == 2: return [] time.sleep(2) return [] def pack(vec): return struct.pack("%df" % len(vec), *vec) def unpack(blob): n = len(blob) // 4 return list(struct.unpack("%df" % n, blob)) def collect_files(): files = [] for d in DIRS: base = os.path.join(ROOT, d) if not os.path.isdir(base): continue for dirpath, dirnames, filenames in os.walk(base): # исключаем только служебное; отчёты гигиены/развития/доверия — это ЭПИЗОДИЧЕСКАЯ память dirnames[:] = [x for x in dirnames if x not in (".git", "node_modules", "__pycache__", "index.db")] for fn in filenames: if fn.endswith((".md", ".txt", ".yml", ".yaml")) and not fn.startswith("."): p = os.path.join(dirpath, fn) if os.path.getsize(p) < 200: continue files.append(p) # эпизодическая память: журнал действий агента journal = "/var/log/hermes/actions.jsonl" if os.path.isfile(journal): try: eps = [] lines = open(journal, encoding="utf-8", errors="ignore").readlines()[-400:] for ln in lines: try: e = json.loads(ln) except Exception: continue if e.get("event") not in ("done", "failed", "denied"): continue eps.append("%s | %s | %s | %s" % ( e.get("ts", ""), e.get("rule", ""), e.get("event", ""), str(e.get("reason") or e.get("detail") or "")[:200])) if eps: ep_path = ROOT + "/ops/rag/episodes-actions.txt" open(ep_path, "w", encoding="utf-8").write("\n".join(eps)) files.append(ep_path) except Exception: pass for f in EXTRA_FILES: p = os.path.join(ROOT, f) if os.path.isfile(p): files.append(p) return sorted(set(files)) def chunk_text(text): text = text.replace("\r\n", "\n") out, i = [], 0 while i < len(text): piece = text[i:i + CHUNK] if piece.strip(): out.append(piece) i += CHUNK - OVERLAP return out def main(): rebuild = "--rebuild" in sys.argv os.makedirs(os.path.dirname(DB), exist_ok=True) import sqlite3 con = sqlite3.connect(DB) cur = con.cursor() cur.execute("""CREATE TABLE IF NOT EXISTS docs( path TEXT PRIMARY KEY, mtime REAL, size INTEGER, hash TEXT)""") cur.execute("""CREATE TABLE IF NOT EXISTS chunks( id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT, idx INTEGER, text TEXT, vec BLOB)""") cur.execute("""CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT)""") if rebuild: cur.execute("DELETE FROM chunks") cur.execute("DELETE FROM docs") con.commit() files = collect_files() known = {r[0]: r for r in cur.execute("SELECT path, mtime, size, hash FROM docs")} added = skipped = failed = 0 ch_total = 0 for p in files: rel = os.path.relpath(p, ROOT) try: st = os.stat(p) raw = open(p, encoding="utf-8", errors="ignore").read() except Exception: failed += 1 continue h = hashlib.sha1(raw.encode("utf-8", "ignore")).hexdigest() old = known.get(rel) if old and old[3] == h and not rebuild: skipped += 1 continue cur.execute("DELETE FROM chunks WHERE path=?", (rel,)) n = 0 for idx, chunk in enumerate(chunk_text(raw)): vec = embed(chunk) if not vec: failed += 1 continue cur.execute("INSERT INTO chunks(path, idx, text, vec) VALUES(?,?,?,?)", (rel, idx, chunk, pack(vec))) n += 1 ch_total += 1 cur.execute("INSERT OR REPLACE INTO docs(path, mtime, size, hash) VALUES(?,?,?,?)", (rel, st.st_mtime, st.st_size, h)) con.commit() added += 1 print(" + %-60s чанков: %d" % (rel[:60], n), flush=True) cur.execute("INSERT OR REPLACE INTO meta(k, v) VALUES('updated', ?)", (time.strftime("%Y-%m-%dT%H:%M:%S"),)) con.commit() total_chunks = cur.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] con.close() print("\nфайлов: обновлено %d, пропущено %d, ошибок %d | чанков всего: %d" % (added, skipped, failed, total_chunks)) if __name__ == "__main__": main()