Files
buryan-infra/ops/scripts/rag_index.py

174 lines
6.0 KiB
Python
Executable File

#!/usr/bin/env python3
"""RAG-индекс по документации Buryans Hub (память мозга).
Собирает markdown/text из ops/, docs/, deploy/, корня репозитория,
разбивает на чанки, считает эмбеддинги локальной моделью nomic-embed-text
и складывает в SQLite.
Использование:
rag_index.py # обновить индекс (инкрементально)
rag_index.py --rebuild # пересобрать с нуля
"""
import hashlib
import json
import os
import struct
import sys
import time
import urllib.request
LLM = "http://192.168.1.229:11434"
EMB_MODEL = "nomic-embed-text"
ROOT = "/root/buryan-infra"
DB = ROOT + "/ops/rag/index.db"
DIRS = ["ops", "docs", "deploy"]
EXTRA_FILES = ["AGENTS.md", "README.md"]
CHUNK = 900
OVERLAP = 120
def embed(text):
body = json.dumps({"model": EMB_MODEL, "prompt": text[:4000]}).encode()
req = urllib.request.Request(LLM + "/api/embeddings", data=body, method="POST")
req.add_header("Content-Type", "application/json")
for attempt in range(3):
try:
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read().decode()).get("embedding") or []
except Exception:
if attempt == 2:
return []
time.sleep(2)
return []
def pack(vec):
return struct.pack("%df" % len(vec), *vec)
def unpack(blob):
n = len(blob) // 4
return list(struct.unpack("%df" % n, blob))
def collect_files():
files = []
for d in DIRS:
base = os.path.join(ROOT, d)
if not os.path.isdir(base):
continue
for dirpath, dirnames, filenames in os.walk(base):
# исключаем только служебное; отчёты гигиены/развития/доверия — это ЭПИЗОДИЧЕСКАЯ память
dirnames[:] = [x for x in dirnames if x not in
(".git", "node_modules", "__pycache__", "index.db")]
for fn in filenames:
if fn.endswith((".md", ".txt", ".yml", ".yaml")) and not fn.startswith("."):
p = os.path.join(dirpath, fn)
if os.path.getsize(p) < 200:
continue
files.append(p)
# эпизодическая память: журнал действий агента
journal = "/var/log/hermes/actions.jsonl"
if os.path.isfile(journal):
try:
eps = []
lines = open(journal, encoding="utf-8", errors="ignore").readlines()[-400:]
for ln in lines:
try:
e = json.loads(ln)
except Exception:
continue
if e.get("event") not in ("done", "failed", "denied"):
continue
eps.append("%s | %s | %s | %s" % (
e.get("ts", ""), e.get("rule", ""), e.get("event", ""),
str(e.get("reason") or e.get("detail") or "")[:200]))
if eps:
ep_path = ROOT + "/ops/rag/episodes-actions.txt"
open(ep_path, "w", encoding="utf-8").write("\n".join(eps))
files.append(ep_path)
except Exception:
pass
for f in EXTRA_FILES:
p = os.path.join(ROOT, f)
if os.path.isfile(p):
files.append(p)
return sorted(set(files))
def chunk_text(text):
text = text.replace("\r\n", "\n")
out, i = [], 0
while i < len(text):
piece = text[i:i + CHUNK]
if piece.strip():
out.append(piece)
i += CHUNK - OVERLAP
return out
def main():
rebuild = "--rebuild" in sys.argv
os.makedirs(os.path.dirname(DB), exist_ok=True)
import sqlite3
con = sqlite3.connect(DB)
cur = con.cursor()
cur.execute("""CREATE TABLE IF NOT EXISTS docs(
path TEXT PRIMARY KEY, mtime REAL, size INTEGER, hash TEXT)""")
cur.execute("""CREATE TABLE IF NOT EXISTS chunks(
id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT, idx INTEGER,
text TEXT, vec BLOB)""")
cur.execute("""CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT)""")
if rebuild:
cur.execute("DELETE FROM chunks")
cur.execute("DELETE FROM docs")
con.commit()
files = collect_files()
known = {r[0]: r for r in cur.execute("SELECT path, mtime, size, hash FROM docs")}
added = skipped = failed = 0
ch_total = 0
for p in files:
rel = os.path.relpath(p, ROOT)
try:
st = os.stat(p)
raw = open(p, encoding="utf-8", errors="ignore").read()
except Exception:
failed += 1
continue
h = hashlib.sha1(raw.encode("utf-8", "ignore")).hexdigest()
old = known.get(rel)
if old and old[3] == h and not rebuild:
skipped += 1
continue
cur.execute("DELETE FROM chunks WHERE path=?", (rel,))
n = 0
for idx, chunk in enumerate(chunk_text(raw)):
vec = embed(chunk)
if not vec:
failed += 1
continue
cur.execute("INSERT INTO chunks(path, idx, text, vec) VALUES(?,?,?,?)",
(rel, idx, chunk, pack(vec)))
n += 1
ch_total += 1
cur.execute("INSERT OR REPLACE INTO docs(path, mtime, size, hash) VALUES(?,?,?,?)",
(rel, st.st_mtime, st.st_size, h))
con.commit()
added += 1
print(" + %-60s чанков: %d" % (rel[:60], n), flush=True)
cur.execute("INSERT OR REPLACE INTO meta(k, v) VALUES('updated', ?)",
(time.strftime("%Y-%m-%dT%H:%M:%S"),))
con.commit()
total_chunks = cur.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
con.close()
print("\nфайлов: обновлено %d, пропущено %d, ошибок %d | чанков всего: %d"
% (added, skipped, failed, total_chunks))
if __name__ == "__main__":
main()