227 lines
8.4 KiB
Python
227 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
"""RAG-индекс Buryans Hub в PostgreSQL (pgvector).
|
|
|
|
Единственный индексатор: раньше их было два (SQLite и Postgres), и агент
|
|
искал в пустой базе. Теперь одна база — Postgres на 192.168.1.151.
|
|
|
|
Хранит:
|
|
rag_docs(path, mtime, size, hash) — что уже проиндексировано
|
|
chunks(source, ref, text, embedding) — фрагменты с векторами (pgvector)
|
|
|
|
Использование:
|
|
rag_index_pg.py # инкрементально
|
|
rag_index_pg.py --rebuild # пересобрать
|
|
rag_index_pg.py --seed # только запомнить хеши (без эмбеддингов)
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
sys.path.insert(0, "/opt/hub-core")
|
|
import rag # noqa: E402
|
|
import psycopg2 # noqa: E402
|
|
|
|
LLM = "http://192.168.1.229:11434"
|
|
EMB_MODEL = "nomic-embed-text"
|
|
ROOT = "/root/buryan-infra"
|
|
DIRS = ["ops", "docs", "deploy"]
|
|
EXTRA_FILES = ["AGENTS.md", "README.md"]
|
|
CHUNK = 900
|
|
OVERLAP = 120
|
|
SOURCE = "ops-rag"
|
|
|
|
|
|
def embed(text):
|
|
body = json.dumps({"model": EMB_MODEL, "prompt": text[:4000]}).encode()
|
|
req = urllib.request.Request(LLM + "/api/embeddings", data=body, method="POST")
|
|
req.add_header("Content-Type", "application/json")
|
|
for attempt in range(3):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=180) as r:
|
|
return json.loads(r.read().decode()).get("embedding") or []
|
|
except Exception:
|
|
if attempt == 2:
|
|
return []
|
|
time.sleep(3)
|
|
return []
|
|
|
|
|
|
|
|
def embed_batch(texts, keep_alive="5m", batch=32):
|
|
"""Батч через /api/embed: 16 текстов ≈ 1 с после загрузки модели.
|
|
|
|
Раньше каждый чанк шёл отдельным /api/embeddings — на 5-6 тыс. чанков
|
|
это часы. При отсутствии /api/embed (старый Ollama) — по одному.
|
|
"""
|
|
out = []
|
|
for i in range(0, len(texts), batch):
|
|
part = [t[:4000] for t in texts[i:i + batch]]
|
|
body = json.dumps({"model": EMB_MODEL, "input": part,
|
|
"keep_alive": keep_alive}).encode()
|
|
req = urllib.request.Request(LLM + "/api/embed", data=body, method="POST")
|
|
req.add_header("Content-Type", "application/json")
|
|
vecs = []
|
|
for attempt in range(2):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=600) as r:
|
|
vecs = json.loads(r.read().decode()).get("embeddings") or []
|
|
break
|
|
except Exception:
|
|
if attempt == 1:
|
|
vecs = []
|
|
else:
|
|
time.sleep(3)
|
|
if not vecs:
|
|
vecs = [embed(t) for t in part]
|
|
out.extend(vecs + [[]] * (len(part) - len(vecs)))
|
|
return out
|
|
|
|
|
|
def collect_files():
|
|
files = []
|
|
for d in DIRS:
|
|
base = os.path.join(ROOT, d)
|
|
if not os.path.isdir(base):
|
|
continue
|
|
for dirpath, dirnames, filenames in os.walk(base):
|
|
dirnames[:] = [x for x in dirnames if x not in
|
|
(".git", "node_modules", "__pycache__", "rag")]
|
|
for fn in filenames:
|
|
if fn.endswith((".md", ".txt", ".yml", ".yaml")) and not fn.startswith("."):
|
|
p = os.path.join(dirpath, fn)
|
|
if os.path.getsize(p) < 200:
|
|
continue
|
|
files.append(p)
|
|
journal = "/var/log/hermes/actions.jsonl"
|
|
if os.path.isfile(journal):
|
|
try:
|
|
eps = []
|
|
lines = open(journal, encoding="utf-8", errors="ignore").readlines()[-400:]
|
|
for ln in lines:
|
|
try:
|
|
e = json.loads(ln)
|
|
except Exception:
|
|
continue
|
|
if e.get("event") not in ("done", "failed", "denied"):
|
|
continue
|
|
eps.append("%s | %s | %s | %s" % (
|
|
e.get("ts", ""), e.get("rule", ""), e.get("event", ""),
|
|
str(e.get("reason") or e.get("detail") or "")[:200]))
|
|
if eps:
|
|
ep = ROOT + "/ops/rag/episodes-actions.txt"
|
|
os.makedirs(os.path.dirname(ep), exist_ok=True)
|
|
open(ep, "w", encoding="utf-8").write("\n".join(eps))
|
|
files.append(ep)
|
|
except Exception:
|
|
pass
|
|
for f in EXTRA_FILES:
|
|
p = os.path.join(ROOT, f)
|
|
if os.path.isfile(p):
|
|
files.append(p)
|
|
return sorted(set(files))
|
|
|
|
|
|
def chunk_text(text):
|
|
text = text.replace("\r\n", "\n")
|
|
out, i = [], 0
|
|
while i < len(text):
|
|
piece = text[i:i + CHUNK]
|
|
if piece.strip():
|
|
out.append(piece)
|
|
i += CHUNK - OVERLAP
|
|
return out
|
|
|
|
|
|
def vec_lit(v):
|
|
return "[" + ",".join("%.6f" % x for x in v) + "]"
|
|
|
|
|
|
|
|
def unload_other_models():
|
|
"""Ollama на 229 не может держать qwen и эмбеддинги одновременно:
|
|
освобождаем память перед индексацией (иначе embeddings висят минутами)."""
|
|
import json as _j
|
|
import urllib.request as _u
|
|
for m in ("qwen2.5:3b", "qwen2.5:7b"):
|
|
try:
|
|
body = _j.dumps({"model": m, "keep_alive": 0}).encode()
|
|
req = _u.Request(LLM + "/api/generate", data=body, method="POST",
|
|
headers={"Content-Type": "application/json"})
|
|
_u.urlopen(req, timeout=20).read()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def main():
|
|
unload_other_models()
|
|
rebuild = "--rebuild" in sys.argv
|
|
seed = "--seed" in sys.argv
|
|
pg = psycopg2.connect(rag.DSN)
|
|
cur = pg.cursor()
|
|
cur.execute("""CREATE TABLE IF NOT EXISTS rag_docs(
|
|
path TEXT PRIMARY KEY, mtime DOUBLE PRECISION, size BIGINT, hash TEXT)""")
|
|
pg.commit()
|
|
if rebuild:
|
|
cur.execute("DELETE FROM chunks WHERE source=%s", (SOURCE,))
|
|
cur.execute("DELETE FROM rag_docs")
|
|
pg.commit()
|
|
|
|
files = collect_files()
|
|
cur.execute("SELECT path, mtime, size, hash FROM rag_docs")
|
|
known = {r[0]: r for r in cur.fetchall()}
|
|
added = skipped = failed = chunks = 0
|
|
for p in files:
|
|
rel = os.path.relpath(p, ROOT)
|
|
try:
|
|
st = os.stat(p)
|
|
raw = open(p, encoding="utf-8", errors="ignore").read()
|
|
except Exception:
|
|
failed += 1
|
|
continue
|
|
h = hashlib.sha1(raw.encode("utf-8", "ignore")).hexdigest()
|
|
old = known.get(rel)
|
|
if old and old[3] == h and not rebuild:
|
|
skipped += 1
|
|
continue
|
|
if seed:
|
|
cur.execute("INSERT INTO rag_docs(path,mtime,size,hash) VALUES(%s,%s,%s,%s) "
|
|
"ON CONFLICT (path) DO UPDATE SET mtime=EXCLUDED.mtime, "
|
|
"size=EXCLUDED.size, hash=EXCLUDED.hash",
|
|
(rel, st.st_mtime, st.st_size, h))
|
|
pg.commit()
|
|
skipped += 1
|
|
continue
|
|
cur.execute("DELETE FROM chunks WHERE source=%s AND ref LIKE %s", (SOURCE, rel + "#%"))
|
|
pieces = list(chunk_text(raw))
|
|
vectors = embed_batch(pieces)
|
|
for idx, (piece, v) in enumerate(zip(pieces, vectors)):
|
|
if not v:
|
|
failed += 1
|
|
continue
|
|
cur.execute("INSERT INTO chunks(source,ref,text,embedding) VALUES(%s,%s,%s,%s) "
|
|
"ON CONFLICT (source,ref) DO UPDATE SET text=EXCLUDED.text, "
|
|
"embedding=EXCLUDED.embedding, updated=now()",
|
|
(SOURCE, "%s#%s" % (rel, idx), piece, vec_lit(v)))
|
|
chunks += 1
|
|
cur.execute("INSERT INTO rag_docs(path,mtime,size,hash) VALUES(%s,%s,%s,%s) "
|
|
"ON CONFLICT (path) DO UPDATE SET mtime=EXCLUDED.mtime, "
|
|
"size=EXCLUDED.size, hash=EXCLUDED.hash",
|
|
(rel, st.st_mtime, st.st_size, h))
|
|
pg.commit()
|
|
added += 1
|
|
if added % 10 == 0:
|
|
print(" файлов обновлено: %d (чанков %d)" % (added, chunks))
|
|
cur.execute("SELECT count(*) FROM chunks WHERE source=%s", (SOURCE,))
|
|
total = cur.fetchone()[0]
|
|
pg.close()
|
|
print("итог: обновлено %d, пропущено %d, ошибок %d | чанков: %d"
|
|
% (added, skipped, failed, total))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|