#!/usr/bin/env python3 """P0.7: отчёт по мусорным фактам (точные ключи).""" import glob import json import collections JUNK = {"pageviews", "downloads", "date", "support document type"} counts = collections.Counter() examples = [] for f in glob.glob("/opt/work-hub/research/results-*.jsonl"): if "retry" in f: continue for line in open(f, encoding="utf-8", errors="ignore"): try: r = json.loads(line) except Exception: continue for x in (r.get("data") or []): if not isinstance(x, dict): continue for fact in (x.get("facts") or []): k = fact.get("key") if isinstance(fact, dict) else None if k and str(k).strip().lower() in JUNK: counts[str(k)] += 1 if len(examples) < 10: examples.append((r.get("vendor"), str(k), fact.get("value", ""))) out = "/root/buryan-infra/ops/combine/P0-JUNK-REPORT-2026-11-18.md" lines = ["# P0.7 — мусорные факты", "", "Точные мусорные ключи: **%d**, вхождений: **%d**." % (len(counts), sum(counts.values())), "", "| Ключ | Вхождений |", "|---|---|"] for k, n in counts.most_common(): lines.append("| %s | %d |" % (k, n)) lines += ["", "## Примеры", "| Вендор | Ключ | Значение |", "|---|---|---|"] for v, k, val in examples: lines.append("| %s | %s | %s |" % (v, k, str(val)[:80])) open(out, "w", encoding="utf-8").write("\n".join(lines) + "\n") print("junk", dict(counts)) print(out)