From c389c858ff0ff97d28d7c20ea144fb5e46319dbf Mon Sep 17 00:00:00 2001 From: Roman Buryan Date: Wed, 18 Nov 2026 18:54:47 +0300 Subject: [PATCH] =?UTF-8?q?[PRISM]=20Ingest=20LLM-extraction=20=D0=B2=20st?= =?UTF-8?q?aging:=20diff,=20report,=20review,=20=D0=B1=D0=B5=D0=B7=20?= =?UTF-8?q?=D0=BF=D1=80=D0=BE=D0=B4=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ops/combine/research-pack/ingest_extracted.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 ops/combine/research-pack/ingest_extracted.py diff --git a/ops/combine/research-pack/ingest_extracted.py b/ops/combine/research-pack/ingest_extracted.py new file mode 100644 index 0000000..03ee41c --- /dev/null +++ b/ops/combine/research-pack/ingest_extracted.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Приём LLM-extraction JSON в staging. Прод не меняет.""" +import argparse +import glob +import json +import os +import re +import time +import yaml + +BASE = "/opt/work-hub/compat" + + +def norm(s): + return re.sub(r"[^a-z0-9]", "", str(s).lower()) + + +def load_existing(): + out = {} + for p in [BASE + "/catalog.yaml"] + sorted(glob.glob(BASE + "/catalog-*.yaml")): + try: + d = yaml.safe_load(open(p, encoding="utf-8")) or {} + except Exception: + continue + for x in d.get("devices") or []: + n = x.get("name") or x.get("model") + if n: + out[norm(n)] = (x, os.path.basename(p)) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--in", dest="inp", required=True) + ap.add_argument("--out", default="/root/buryan-infra/ops/combine/staging/ingest-" + time.strftime("%Y%m%d-%H%M")) + a = ap.parse_args() + os.makedirs(a.out, exist_ok=True) + existing = load_existing() + records, docs, warnings = [], [], [] + for p in sorted(glob.glob(os.path.join(a.inp, "*.json"))): + try: + d = json.load(open(p, encoding="utf-8")) + except Exception as e: + warnings.append({"doc": os.path.basename(p), "error": str(e)}) + continue + src = d.get("source") or {} + if not src.get("url"): + warnings.append({"doc": os.path.basename(p), "warning": "source.url отсутствует (local doc)"}) + if not src.get("quote"): + warnings.append({"doc": os.path.basename(p), "warning": "source.quote отсутствует"}) + models = d.get("models") or [] + docs.append({"file": os.path.basename(p), "doc_id": d.get("doc_id"), + "vendor": d.get("vendor"), "models": len(models), + "source": src.get("url")}) + for m in models: + model = m.get("model") + if not model: + continue + key = norm(model) + old = existing.get(key) + specs = m.get("specs") or {} + protocols = specs.get("protocols") or m.get("protocols") or [] + rec = { + "vendor": d.get("vendor"), "model": model, "family": m.get("family"), + "role": m.get("role"), "form_factor": m.get("form_factor"), + "market": m.get("market"), "variant_of": m.get("variant_of"), + "protocols": protocols, "specs": specs, + "accessories": m.get("accessories") or [], "kits": m.get("kits") or [], + "restrictions": m.get("restrictions") or [], + "lifecycle": m.get("lifecycle"), + "source": {"doc_id": d.get("doc_id"), "url": src.get("url"), + "quote": src.get("quote"), "level": "L1"}, + "review": "needs_review", "existing": bool(old), + "existing_file": old[1] if old else None, + } + records.append(rec) + yaml.safe_dump({"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "status": "staging", "models": records}, + open(os.path.join(a.out, "models.yaml"), "w", encoding="utf-8"), + allow_unicode=True, sort_keys=False) + new = sum(1 for x in records if not x["existing"]) + old = len(records) - new + lines = ["# Ingest report", "", "Извлечено документов: **%d**" % len(docs), + "Моделей: **%d**, новых: **%d**, уже в каталоге: **%d**" % (len(records), new, old), + "Статус: **staging, прод не изменён**.", "", "## Документы"] + for x in docs: + lines.append("- `%s` — %s, моделей %d" % (x["file"], x["vendor"], x["models"])) + if warnings: + lines += ["", "## Предупреждения"] + ["- %s: %s" % (w.get("doc"), w.get("warning") or w.get("error")) for w in warnings] + lines += ["", "## Модели"] + for x in records: + lines.append("- %s %s — %s%s" % (x["vendor"], x["model"], + "новая" if not x["existing"] else "существует", + " · требует review")) + open(os.path.join(a.out, "REPORT.md"), "w", encoding="utf-8").write("\n".join(lines) + "\n") + print("out", a.out, "docs", len(docs), "models", len(records), "new", new, "existing", old) + + +if __name__ == "__main__": + main()