[PRISM] Ingest LLM-extraction в staging: diff, report, review, без прода
lint / hygiene (push) Failing after 46s
lint / hygiene (push) Failing after 46s
This commit is contained in:
1 parent
536a8fcc63
commit
c389c858ff
1 file changed
+100
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Приём LLM-extraction JSON в staging. Прод не меняет."""
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import yaml
|
||||
|
||||
BASE = "/opt/work-hub/compat"
|
||||
|
||||
|
||||
def norm(s):
|
||||
return re.sub(r"[^a-z0-9]", "", str(s).lower())
|
||||
|
||||
|
||||
def load_existing():
|
||||
out = {}
|
||||
for p in [BASE + "/catalog.yaml"] + sorted(glob.glob(BASE + "/catalog-*.yaml")):
|
||||
try:
|
||||
d = yaml.safe_load(open(p, encoding="utf-8")) or {}
|
||||
except Exception:
|
||||
continue
|
||||
for x in d.get("devices") or []:
|
||||
n = x.get("name") or x.get("model")
|
||||
if n:
|
||||
out[norm(n)] = (x, os.path.basename(p))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--in", dest="inp", required=True)
|
||||
ap.add_argument("--out", default="/root/buryan-infra/ops/combine/staging/ingest-" + time.strftime("%Y%m%d-%H%M"))
|
||||
a = ap.parse_args()
|
||||
os.makedirs(a.out, exist_ok=True)
|
||||
existing = load_existing()
|
||||
records, docs, warnings = [], [], []
|
||||
for p in sorted(glob.glob(os.path.join(a.inp, "*.json"))):
|
||||
try:
|
||||
d = json.load(open(p, encoding="utf-8"))
|
||||
except Exception as e:
|
||||
warnings.append({"doc": os.path.basename(p), "error": str(e)})
|
||||
continue
|
||||
src = d.get("source") or {}
|
||||
if not src.get("url"):
|
||||
warnings.append({"doc": os.path.basename(p), "warning": "source.url отсутствует (local doc)"})
|
||||
if not src.get("quote"):
|
||||
warnings.append({"doc": os.path.basename(p), "warning": "source.quote отсутствует"})
|
||||
models = d.get("models") or []
|
||||
docs.append({"file": os.path.basename(p), "doc_id": d.get("doc_id"),
|
||||
"vendor": d.get("vendor"), "models": len(models),
|
||||
"source": src.get("url")})
|
||||
for m in models:
|
||||
model = m.get("model")
|
||||
if not model:
|
||||
continue
|
||||
key = norm(model)
|
||||
old = existing.get(key)
|
||||
specs = m.get("specs") or {}
|
||||
protocols = specs.get("protocols") or m.get("protocols") or []
|
||||
rec = {
|
||||
"vendor": d.get("vendor"), "model": model, "family": m.get("family"),
|
||||
"role": m.get("role"), "form_factor": m.get("form_factor"),
|
||||
"market": m.get("market"), "variant_of": m.get("variant_of"),
|
||||
"protocols": protocols, "specs": specs,
|
||||
"accessories": m.get("accessories") or [], "kits": m.get("kits") or [],
|
||||
"restrictions": m.get("restrictions") or [],
|
||||
"lifecycle": m.get("lifecycle"),
|
||||
"source": {"doc_id": d.get("doc_id"), "url": src.get("url"),
|
||||
"quote": src.get("quote"), "level": "L1"},
|
||||
"review": "needs_review", "existing": bool(old),
|
||||
"existing_file": old[1] if old else None,
|
||||
}
|
||||
records.append(rec)
|
||||
yaml.safe_dump({"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||
"status": "staging", "models": records},
|
||||
open(os.path.join(a.out, "models.yaml"), "w", encoding="utf-8"),
|
||||
allow_unicode=True, sort_keys=False)
|
||||
new = sum(1 for x in records if not x["existing"])
|
||||
old = len(records) - new
|
||||
lines = ["# Ingest report", "", "Извлечено документов: **%d**" % len(docs),
|
||||
"Моделей: **%d**, новых: **%d**, уже в каталоге: **%d**" % (len(records), new, old),
|
||||
"Статус: **staging, прод не изменён**.", "", "## Документы"]
|
||||
for x in docs:
|
||||
lines.append("- `%s` — %s, моделей %d" % (x["file"], x["vendor"], x["models"]))
|
||||
if warnings:
|
||||
lines += ["", "## Предупреждения"] + ["- %s: %s" % (w.get("doc"), w.get("warning") or w.get("error")) for w in warnings]
|
||||
lines += ["", "## Модели"]
|
||||
for x in records:
|
||||
lines.append("- %s %s — %s%s" % (x["vendor"], x["model"],
|
||||
"новая" if not x["existing"] else "существует",
|
||||
" · требует review"))
|
||||
open(os.path.join(a.out, "REPORT.md"), "w", encoding="utf-8").write("\n".join(lines) + "\n")
|
||||
print("out", a.out, "docs", len(docs), "models", len(records), "new", new, "existing", old)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user