119 lines
4.8 KiB
Python
119 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Приём LLM-extraction JSON в staging. Прод не меняет."""
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
import yaml
|
|
|
|
BASE = "/opt/work-hub/compat"
|
|
|
|
|
|
def norm(s):
|
|
return re.sub(r"[^a-z0-9]", "", str(s).lower())
|
|
|
|
|
|
def vendor_key(v):
|
|
s = str(v or "").lower()
|
|
if "poly" in s or s in ("hp", "hp inc.", "hp inc. (poly)"):
|
|
return "poly"
|
|
if "yealink" in s:
|
|
return "yealink"
|
|
if "logitech" in s:
|
|
return "logitech"
|
|
return s
|
|
|
|
|
|
def model_key(m):
|
|
s = str(m or "").strip()
|
|
s = re.sub(r"^HP\s+", "", s, flags=re.I)
|
|
s = re.sub(r"^HP Inc\. \(Poly\)\s+", "", s, flags=re.I)
|
|
return s.strip()
|
|
|
|
|
|
def load_existing():
|
|
out = {}
|
|
for p in [BASE + "/catalog.yaml"] + sorted(glob.glob(BASE + "/catalog-*.yaml")):
|
|
try:
|
|
d = yaml.safe_load(open(p, encoding="utf-8")) or {}
|
|
except Exception:
|
|
continue
|
|
for x in d.get("devices") or []:
|
|
n = x.get("name") or x.get("model")
|
|
if n:
|
|
out[norm(n)] = (x, os.path.basename(p))
|
|
return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--in", dest="inp", required=True)
|
|
ap.add_argument("--out", default="/root/buryan-infra/ops/combine/staging/ingest-" + time.strftime("%Y%m%d-%H%M"))
|
|
a = ap.parse_args()
|
|
os.makedirs(a.out, exist_ok=True)
|
|
existing = load_existing()
|
|
records, docs, warnings = [], [], []
|
|
for p in sorted(glob.glob(os.path.join(a.inp, "*.json"))):
|
|
try:
|
|
d = json.load(open(p, encoding="utf-8"))
|
|
except Exception as e:
|
|
warnings.append({"doc": os.path.basename(p), "error": str(e)})
|
|
continue
|
|
src = d.get("source") or {}
|
|
if not src.get("url"):
|
|
warnings.append({"doc": os.path.basename(p), "warning": "source.url отсутствует (local doc)"})
|
|
if not src.get("quote"):
|
|
warnings.append({"doc": os.path.basename(p), "warning": "source.quote отсутствует"})
|
|
models = d.get("models") or []
|
|
docs.append({"file": os.path.basename(p), "doc_id": d.get("doc_id"),
|
|
"vendor": d.get("vendor"), "models": len(models),
|
|
"source": src.get("url")})
|
|
for m in models:
|
|
model = model_key(m.get("model"))
|
|
if not model:
|
|
continue
|
|
key = norm(model)
|
|
old = existing.get(key)
|
|
specs = m.get("specs") or {}
|
|
protocols = specs.get("protocols") or m.get("protocols") or []
|
|
rec = {
|
|
"vendor": vendor_key(d.get("vendor")), "model": model, "family": m.get("family"),
|
|
"role": m.get("role"), "form_factor": m.get("form_factor"),
|
|
"market": m.get("market"), "variant_of": m.get("variant_of"),
|
|
"protocols": protocols, "specs": specs,
|
|
"accessories": m.get("accessories") or [], "kits": m.get("kits") or [],
|
|
"restrictions": m.get("restrictions") or [],
|
|
"lifecycle": m.get("lifecycle"),
|
|
"source": {"doc_id": d.get("doc_id"), "url": src.get("url"),
|
|
"quote": src.get("quote"), "level": "L1"},
|
|
"review": "needs_review", "existing": bool(old),
|
|
"existing_file": old[1] if old else None,
|
|
}
|
|
records.append(rec)
|
|
yaml.safe_dump({"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
|
"status": "staging", "models": records},
|
|
open(os.path.join(a.out, "models.yaml"), "w", encoding="utf-8"),
|
|
allow_unicode=True, sort_keys=False)
|
|
new = sum(1 for x in records if not x["existing"])
|
|
old = len(records) - new
|
|
lines = ["# Ingest report", "", "Извлечено документов: **%d**" % len(docs),
|
|
"Моделей: **%d**, новых: **%d**, уже в каталоге: **%d**" % (len(records), new, old),
|
|
"Статус: **staging, прод не изменён**.", "", "## Документы"]
|
|
for x in docs:
|
|
lines.append("- `%s` — %s, моделей %d" % (x["file"], x["vendor"], x["models"]))
|
|
if warnings:
|
|
lines += ["", "## Предупреждения"] + ["- %s: %s" % (w.get("doc"), w.get("warning") or w.get("error")) for w in warnings]
|
|
lines += ["", "## Модели"]
|
|
for x in records:
|
|
lines.append("- %s %s — %s%s" % (x["vendor"], x["model"],
|
|
"новая" if not x["existing"] else "существует",
|
|
" · требует review"))
|
|
open(os.path.join(a.out, "REPORT.md"), "w", encoding="utf-8").write("\n".join(lines) + "\n")
|
|
print("out", a.out, "docs", len(docs), "models", len(records), "new", new, "existing", old)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|