Files

39 lines
1.6 KiB
Python

#!/usr/bin/env python3
"""P0.7: отчёт по мусорным фактам (точные ключи)."""
import glob
import json
import collections
JUNK = {"pageviews", "downloads", "date", "support document type"}
counts = collections.Counter()
examples = []
for f in glob.glob("/opt/work-hub/research/results-*.jsonl"):
if "retry" in f:
continue
for line in open(f, encoding="utf-8", errors="ignore"):
try:
r = json.loads(line)
except Exception:
continue
for x in (r.get("data") or []):
if not isinstance(x, dict):
continue
for fact in (x.get("facts") or []):
k = fact.get("key") if isinstance(fact, dict) else None
if k and str(k).strip().lower() in JUNK:
counts[str(k)] += 1
if len(examples) < 10:
examples.append((r.get("vendor"), str(k), fact.get("value", "")))
out = "/root/buryan-infra/ops/combine/P0-JUNK-REPORT-2026-11-18.md"
lines = ["# P0.7 — мусорные факты", "",
"Точные мусорные ключи: **%d**, вхождений: **%d**." % (len(counts), sum(counts.values())), "",
"| Ключ | Вхождений |", "|---|---|"]
for k, n in counts.most_common():
lines.append("| %s | %d |" % (k, n))
lines += ["", "## Примеры", "| Вендор | Ключ | Значение |", "|---|---|---|"]
for v, k, val in examples:
lines.append("| %s | %s | %s |" % (v, k, str(val)[:80]))
open(out, "w", encoding="utf-8").write("\n".join(lines) + "\n")
print("junk", dict(counts))
print(out)