174 lines
6.0 KiB
Python
Executable File
174 lines
6.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""RAG-индекс по документации Buryans Hub (память мозга).
|
|
|
|
Собирает markdown/text из ops/, docs/, deploy/, корня репозитория,
|
|
разбивает на чанки, считает эмбеддинги локальной моделью nomic-embed-text
|
|
и складывает в SQLite.
|
|
|
|
Использование:
|
|
rag_index.py # обновить индекс (инкрементально)
|
|
rag_index.py --rebuild # пересобрать с нуля
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
LLM = "http://192.168.1.229:11434"
|
|
EMB_MODEL = "nomic-embed-text"
|
|
ROOT = "/root/buryan-infra"
|
|
DB = ROOT + "/ops/rag/index.db"
|
|
DIRS = ["ops", "docs", "deploy"]
|
|
EXTRA_FILES = ["AGENTS.md", "README.md"]
|
|
CHUNK = 900
|
|
OVERLAP = 120
|
|
|
|
|
|
def embed(text):
|
|
body = json.dumps({"model": EMB_MODEL, "prompt": text[:4000]}).encode()
|
|
req = urllib.request.Request(LLM + "/api/embeddings", data=body, method="POST")
|
|
req.add_header("Content-Type", "application/json")
|
|
for attempt in range(3):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=120) as r:
|
|
return json.loads(r.read().decode()).get("embedding") or []
|
|
except Exception:
|
|
if attempt == 2:
|
|
return []
|
|
time.sleep(2)
|
|
return []
|
|
|
|
|
|
def pack(vec):
|
|
return struct.pack("%df" % len(vec), *vec)
|
|
|
|
|
|
def unpack(blob):
|
|
n = len(blob) // 4
|
|
return list(struct.unpack("%df" % n, blob))
|
|
|
|
|
|
def collect_files():
|
|
files = []
|
|
for d in DIRS:
|
|
base = os.path.join(ROOT, d)
|
|
if not os.path.isdir(base):
|
|
continue
|
|
for dirpath, dirnames, filenames in os.walk(base):
|
|
# исключаем только служебное; отчёты гигиены/развития/доверия — это ЭПИЗОДИЧЕСКАЯ память
|
|
dirnames[:] = [x for x in dirnames if x not in
|
|
(".git", "node_modules", "__pycache__", "index.db")]
|
|
for fn in filenames:
|
|
if fn.endswith((".md", ".txt", ".yml", ".yaml")) and not fn.startswith("."):
|
|
p = os.path.join(dirpath, fn)
|
|
if os.path.getsize(p) < 200:
|
|
continue
|
|
files.append(p)
|
|
# эпизодическая память: журнал действий агента
|
|
journal = "/var/log/hermes/actions.jsonl"
|
|
if os.path.isfile(journal):
|
|
try:
|
|
eps = []
|
|
lines = open(journal, encoding="utf-8", errors="ignore").readlines()[-400:]
|
|
for ln in lines:
|
|
try:
|
|
e = json.loads(ln)
|
|
except Exception:
|
|
continue
|
|
if e.get("event") not in ("done", "failed", "denied"):
|
|
continue
|
|
eps.append("%s | %s | %s | %s" % (
|
|
e.get("ts", ""), e.get("rule", ""), e.get("event", ""),
|
|
str(e.get("reason") or e.get("detail") or "")[:200]))
|
|
if eps:
|
|
ep_path = ROOT + "/ops/rag/episodes-actions.txt"
|
|
open(ep_path, "w", encoding="utf-8").write("\n".join(eps))
|
|
files.append(ep_path)
|
|
except Exception:
|
|
pass
|
|
for f in EXTRA_FILES:
|
|
p = os.path.join(ROOT, f)
|
|
if os.path.isfile(p):
|
|
files.append(p)
|
|
return sorted(set(files))
|
|
|
|
|
|
def chunk_text(text):
|
|
text = text.replace("\r\n", "\n")
|
|
out, i = [], 0
|
|
while i < len(text):
|
|
piece = text[i:i + CHUNK]
|
|
if piece.strip():
|
|
out.append(piece)
|
|
i += CHUNK - OVERLAP
|
|
return out
|
|
|
|
|
|
def main():
|
|
rebuild = "--rebuild" in sys.argv
|
|
os.makedirs(os.path.dirname(DB), exist_ok=True)
|
|
import sqlite3
|
|
con = sqlite3.connect(DB)
|
|
cur = con.cursor()
|
|
cur.execute("""CREATE TABLE IF NOT EXISTS docs(
|
|
path TEXT PRIMARY KEY, mtime REAL, size INTEGER, hash TEXT)""")
|
|
cur.execute("""CREATE TABLE IF NOT EXISTS chunks(
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT, idx INTEGER,
|
|
text TEXT, vec BLOB)""")
|
|
cur.execute("""CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT)""")
|
|
if rebuild:
|
|
cur.execute("DELETE FROM chunks")
|
|
cur.execute("DELETE FROM docs")
|
|
con.commit()
|
|
|
|
files = collect_files()
|
|
known = {r[0]: r for r in cur.execute("SELECT path, mtime, size, hash FROM docs")}
|
|
added = skipped = failed = 0
|
|
ch_total = 0
|
|
|
|
for p in files:
|
|
rel = os.path.relpath(p, ROOT)
|
|
try:
|
|
st = os.stat(p)
|
|
raw = open(p, encoding="utf-8", errors="ignore").read()
|
|
except Exception:
|
|
failed += 1
|
|
continue
|
|
h = hashlib.sha1(raw.encode("utf-8", "ignore")).hexdigest()
|
|
old = known.get(rel)
|
|
if old and old[3] == h and not rebuild:
|
|
skipped += 1
|
|
continue
|
|
|
|
cur.execute("DELETE FROM chunks WHERE path=?", (rel,))
|
|
n = 0
|
|
for idx, chunk in enumerate(chunk_text(raw)):
|
|
vec = embed(chunk)
|
|
if not vec:
|
|
failed += 1
|
|
continue
|
|
cur.execute("INSERT INTO chunks(path, idx, text, vec) VALUES(?,?,?,?)",
|
|
(rel, idx, chunk, pack(vec)))
|
|
n += 1
|
|
ch_total += 1
|
|
cur.execute("INSERT OR REPLACE INTO docs(path, mtime, size, hash) VALUES(?,?,?,?)",
|
|
(rel, st.st_mtime, st.st_size, h))
|
|
con.commit()
|
|
added += 1
|
|
print(" + %-60s чанков: %d" % (rel[:60], n), flush=True)
|
|
|
|
cur.execute("INSERT OR REPLACE INTO meta(k, v) VALUES('updated', ?)",
|
|
(time.strftime("%Y-%m-%dT%H:%M:%S"),))
|
|
con.commit()
|
|
total_chunks = cur.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
|
|
con.close()
|
|
print("\nфайлов: обновлено %d, пропущено %d, ошибок %d | чанков всего: %d"
|
|
% (added, skipped, failed, total_chunks))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|