[SPRINT-V1][C] RAG-память: 741 документ / 3032 чанка, гибридный поиск, ответы со ссылками; ночная переиндексация 03:40

This commit is contained in:
root committed 2026-11-05 21:31:58 +03:00
1 parent 56fe764be9
commit 7ced982c81
4 files changed
+351

No files matched your search

+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""RAG-индекс по документации Buryans Hub (память мозга).
Собирает markdown/text из ops/, docs/, deploy/, корня репозитория,
разбивает на чанки, считает эмбеддинги локальной моделью nomic-embed-text
и складывает в SQLite.
Использование:
rag_index.py # обновить индекс (инкрементально)
rag_index.py --rebuild # пересобрать с нуля
"""
import hashlib
import json
import os
import struct
import sys
import time
import urllib.request
LLM = "http://192.168.1.229:11434"
EMB_MODEL = "nomic-embed-text"
ROOT = "/root/buryan-infra"
DB = ROOT + "/ops/rag/index.db"
DIRS = ["ops", "docs", "deploy"]
EXTRA_FILES = ["AGENTS.md", "README.md"]
CHUNK = 900
OVERLAP = 120
def embed(text):
body = json.dumps({"model": EMB_MODEL, "prompt": text[:4000]}).encode()
req = urllib.request.Request(LLM + "/api/embeddings", data=body, method="POST")
req.add_header("Content-Type", "application/json")
for attempt in range(3):
try:
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read().decode()).get("embedding") or []
except Exception:
if attempt == 2:
return []
time.sleep(2)
return []
def pack(vec):
return struct.pack("%df" % len(vec), *vec)
def unpack(blob):
n = len(blob) // 4
return list(struct.unpack("%df" % n, blob))
def collect_files():
files = []
for d in DIRS:
base = os.path.join(ROOT, d)
if not os.path.isdir(base):
continue
for dirpath, dirnames, filenames in os.walk(base):
dirnames[:] = [x for x in dirnames if x not in
(".git", "node_modules", "rag", "topology", "hygiene", "__pycache__")]
for fn in filenames:
if fn.endswith((".md", ".txt", ".yml", ".yaml")) and not fn.startswith("."):
p = os.path.join(dirpath, fn)
if os.path.getsize(p) < 200:
continue
files.append(p)
for f in EXTRA_FILES:
p = os.path.join(ROOT, f)
if os.path.isfile(p):
files.append(p)
return sorted(set(files))
def chunk_text(text):
text = text.replace("\r\n", "\n")
out, i = [], 0
while i < len(text):
piece = text[i:i + CHUNK]
if piece.strip():
out.append(piece)
i += CHUNK - OVERLAP
return out
def main():
rebuild = "--rebuild" in sys.argv
os.makedirs(os.path.dirname(DB), exist_ok=True)
import sqlite3
con = sqlite3.connect(DB)
cur = con.cursor()
cur.execute("""CREATE TABLE IF NOT EXISTS docs(
path TEXT PRIMARY KEY, mtime REAL, size INTEGER, hash TEXT)""")
cur.execute("""CREATE TABLE IF NOT EXISTS chunks(
id INTEGER PRIMARY KEY AUTOINCREMENT, path TEXT, idx INTEGER,
text TEXT, vec BLOB)""")
cur.execute("""CREATE TABLE IF NOT EXISTS meta(k TEXT PRIMARY KEY, v TEXT)""")
if rebuild:
cur.execute("DELETE FROM chunks")
cur.execute("DELETE FROM docs")
con.commit()
files = collect_files()
known = {r[0]: r for r in cur.execute("SELECT path, mtime, size, hash FROM docs")}
added = skipped = failed = 0
ch_total = 0
for p in files:
rel = os.path.relpath(p, ROOT)
try:
st = os.stat(p)
raw = open(p, encoding="utf-8", errors="ignore").read()
except Exception:
failed += 1
continue
h = hashlib.sha1(raw.encode("utf-8", "ignore")).hexdigest()
old = known.get(rel)
if old and old[3] == h and not rebuild:
skipped += 1
continue
cur.execute("DELETE FROM chunks WHERE path=?", (rel,))
n = 0
for idx, chunk in enumerate(chunk_text(raw)):
vec = embed(chunk)
if not vec:
failed += 1
continue
cur.execute("INSERT INTO chunks(path, idx, text, vec) VALUES(?,?,?,?)",
(rel, idx, chunk, pack(vec)))
n += 1
ch_total += 1
cur.execute("INSERT OR REPLACE INTO docs(path, mtime, size, hash) VALUES(?,?,?,?)",
(rel, st.st_mtime, st.st_size, h))
con.commit()
added += 1
print(" + %-60s чанков: %d" % (rel[:60], n), flush=True)
cur.execute("INSERT OR REPLACE INTO meta(k, v) VALUES('updated', ?)",
(time.strftime("%Y-%m-%dT%H:%M:%S"),))
con.commit()
total_chunks = cur.execute("SELECT COUNT(*) FROM chunks").fetchone()[0]
con.close()
print("\nфайлов: обновлено %d, пропущено %d, ошибок %d | чанков всего: %d"
% (added, skipped, failed, total_chunks))
if __name__ == "__main__":
main()
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
"""Поиск по документации (RAG) с ответом локальной модели.
Использование:
rag_query.py "почему падал chronyd" # ответ + источники
rag_query.py --search "chronyd" # только фрагменты
rag_query.py --answer-only "вопрос" # только ответ
"""
import argparse
import json
import math
import os
import sqlite3
import struct
import sys
import urllib.request
LLM = "http://192.168.1.229:11434"
EMB_MODEL = "nomic-embed-text"
CHAT_MODEL = "qwen2.5:7b"
DB = "/root/buryan-infra/ops/rag/index.db"
def embed(text):
body = json.dumps({"model": EMB_MODEL, "prompt": text[:4000]}).encode()
req = urllib.request.Request(LLM + "/api/embeddings", data=body, method="POST")
req.add_header("Content-Type", "application/json")
with urllib.request.urlopen(req, timeout=120) as r:
return json.loads(r.read().decode()).get("embedding") or []
def cosine(a, b):
if not a or not b or len(a) != len(b):
return 0.0
dot = sum(x * y for x, y in zip(a, b))
na = math.sqrt(sum(x * x for x in a))
nb = math.sqrt(sum(y * y for y in b))
return dot / (na * nb) if na and nb else 0.0
def load_chunks():
con = sqlite3.connect(DB)
rows = con.execute("SELECT path, idx, text, vec FROM chunks").fetchall()
con.close()
out = []
for path, idx, text, blob in rows:
n = len(blob) // 4
out.append((path, idx, text, list(struct.unpack("%df" % n, blob))))
return out
def search(query, k=5):
"""Гибридный поиск: косинус (0.7) + совпадение ключевых слов (0.3)."""
import re
qv = embed(query)
if not qv:
return []
words = [w.lower() for w in re.findall(r"[\w\-]{4,}", query)]
scored = []
for path, idx, text, vec in load_chunks():
cos = cosine(qv, vec)
# ключевые слова в тексте и в пути
low = (text + " " + path).lower()
hits = sum(1 for w in words if w in low)
kw = hits / len(words) if words else 0.0
score = 0.7 * cos + 0.3 * kw
scored.append((score, path, idx, text))
scored.sort(key=lambda x: -x[0])
return scored[:k]
def ask(question, contexts, max_tokens=500):
ctx = "\n\n".join("--- %s (фрагмент %s) ---\n%s" % (p, i, t)
for _, p, i, t in contexts)
prompt = (
"Ты помощник по инфраструктуре Buryans Hub. Отвечай по-русски, кратко и по делу.\n"
"Используй ТОЛЬКО приведённые фрагменты документации. Если ответа в них нет — "
"так и скажи: «в документации этого нет».\n\n"
"ФРАГМЕНТЫ:\n%s\n\nВОПРОС: %s\n\nОТВЕТ:" % (ctx, question)
)
body = json.dumps({"model": CHAT_MODEL, "prompt": prompt, "stream": False,
"options": {"num_predict": max_tokens, "temperature": 0.1}}).encode()
req = urllib.request.Request(LLM + "/api/generate", data=body, method="POST")
req.add_header("Content-Type", "application/json")
with urllib.request.urlopen(req, timeout=600) as r:
return json.loads(r.read().decode()).get("response", "").strip()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("query")
ap.add_argument("--k", type=int, default=5)
ap.add_argument("--search", action="store_true", help="только фрагменты")
ap.add_argument("--answer-only", action="store_true", dest="answer_only")
a = ap.parse_args()
if not os.path.exists(DB):
print("индекс не собран: запустите rag_index.py", file=sys.stderr)
return 2
hits = search(a.query, a.k)
if not hits:
print("ничего не найдено (узел LLM недоступен?)", file=sys.stderr)
return 1
if a.search:
for score, path, idx, text in hits:
print("\n=== %.3f %s [%d]" % (score, path, idx))
print(text[:400].strip())
return 0
answer = ask(a.query, hits)
if not a.answer_only:
print(answer)
print("\nИсточники:")
for score, path, idx, _ in hits:
print(" %.3f %s [%d]" % (score, path, idx))
else:
print(answer)
return 0
if __name__ == "__main__":
sys.exit(main())