[PRISM] Ingest LLM-extraction в staging: diff, report, review, без прода
lint / hygiene (push) Failing after 46s

This commit is contained in:
root committed 2026-11-18 18:54:47 +03:00
1 parent 536a8fcc63
commit c389c858ff
1 file changed
+100
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""Приём LLM-extraction JSON в staging. Прод не меняет."""
import argparse
import glob
import json
import os
import re
import time
import yaml
BASE = "/opt/work-hub/compat"
def norm(s):
return re.sub(r"[^a-z0-9]", "", str(s).lower())
def load_existing():
out = {}
for p in [BASE + "/catalog.yaml"] + sorted(glob.glob(BASE + "/catalog-*.yaml")):
try:
d = yaml.safe_load(open(p, encoding="utf-8")) or {}
except Exception:
continue
for x in d.get("devices") or []:
n = x.get("name") or x.get("model")
if n:
out[norm(n)] = (x, os.path.basename(p))
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--in", dest="inp", required=True)
ap.add_argument("--out", default="/root/buryan-infra/ops/combine/staging/ingest-" + time.strftime("%Y%m%d-%H%M"))
a = ap.parse_args()
os.makedirs(a.out, exist_ok=True)
existing = load_existing()
records, docs, warnings = [], [], []
for p in sorted(glob.glob(os.path.join(a.inp, "*.json"))):
try:
d = json.load(open(p, encoding="utf-8"))
except Exception as e:
warnings.append({"doc": os.path.basename(p), "error": str(e)})
continue
src = d.get("source") or {}
if not src.get("url"):
warnings.append({"doc": os.path.basename(p), "warning": "source.url отсутствует (local doc)"})
if not src.get("quote"):
warnings.append({"doc": os.path.basename(p), "warning": "source.quote отсутствует"})
models = d.get("models") or []
docs.append({"file": os.path.basename(p), "doc_id": d.get("doc_id"),
"vendor": d.get("vendor"), "models": len(models),
"source": src.get("url")})
for m in models:
model = m.get("model")
if not model:
continue
key = norm(model)
old = existing.get(key)
specs = m.get("specs") or {}
protocols = specs.get("protocols") or m.get("protocols") or []
rec = {
"vendor": d.get("vendor"), "model": model, "family": m.get("family"),
"role": m.get("role"), "form_factor": m.get("form_factor"),
"market": m.get("market"), "variant_of": m.get("variant_of"),
"protocols": protocols, "specs": specs,
"accessories": m.get("accessories") or [], "kits": m.get("kits") or [],
"restrictions": m.get("restrictions") or [],
"lifecycle": m.get("lifecycle"),
"source": {"doc_id": d.get("doc_id"), "url": src.get("url"),
"quote": src.get("quote"), "level": "L1"},
"review": "needs_review", "existing": bool(old),
"existing_file": old[1] if old else None,
}
records.append(rec)
yaml.safe_dump({"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
"status": "staging", "models": records},
open(os.path.join(a.out, "models.yaml"), "w", encoding="utf-8"),
allow_unicode=True, sort_keys=False)
new = sum(1 for x in records if not x["existing"])
old = len(records) - new
lines = ["# Ingest report", "", "Извлечено документов: **%d**" % len(docs),
"Моделей: **%d**, новых: **%d**, уже в каталоге: **%d**" % (len(records), new, old),
"Статус: **staging, прод не изменён**.", "", "## Документы"]
for x in docs:
lines.append("- `%s` — %s, моделей %d" % (x["file"], x["vendor"], x["models"]))
if warnings:
lines += ["", "## Предупреждения"] + ["- %s: %s" % (w.get("doc"), w.get("warning") or w.get("error")) for w in warnings]
lines += ["", "## Модели"]
for x in records:
lines.append("- %s %s — %s%s" % (x["vendor"], x["model"],
"новая" if not x["existing"] else "существует",
" · требует review"))
open(os.path.join(a.out, "REPORT.md"), "w", encoding="utf-8").write("\n".join(lines) + "\n")
print("out", a.out, "docs", len(docs), "models", len(records), "new", new, "existing", old)
if __name__ == "__main__":
main()