39 lines
1.6 KiB
Python
39 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
"""P0.7: отчёт по мусорным фактам (точные ключи)."""
|
|
import glob
|
|
import json
|
|
import collections
|
|
|
|
JUNK = {"pageviews", "downloads", "date", "support document type"}
|
|
counts = collections.Counter()
|
|
examples = []
|
|
for f in glob.glob("/opt/work-hub/research/results-*.jsonl"):
|
|
if "retry" in f:
|
|
continue
|
|
for line in open(f, encoding="utf-8", errors="ignore"):
|
|
try:
|
|
r = json.loads(line)
|
|
except Exception:
|
|
continue
|
|
for x in (r.get("data") or []):
|
|
if not isinstance(x, dict):
|
|
continue
|
|
for fact in (x.get("facts") or []):
|
|
k = fact.get("key") if isinstance(fact, dict) else None
|
|
if k and str(k).strip().lower() in JUNK:
|
|
counts[str(k)] += 1
|
|
if len(examples) < 10:
|
|
examples.append((r.get("vendor"), str(k), fact.get("value", "")))
|
|
out = "/root/buryan-infra/ops/combine/P0-JUNK-REPORT-2026-11-18.md"
|
|
lines = ["# P0.7 — мусорные факты", "",
|
|
"Точные мусорные ключи: **%d**, вхождений: **%d**." % (len(counts), sum(counts.values())), "",
|
|
"| Ключ | Вхождений |", "|---|---|"]
|
|
for k, n in counts.most_common():
|
|
lines.append("| %s | %d |" % (k, n))
|
|
lines += ["", "## Примеры", "| Вендор | Ключ | Значение |", "|---|---|---|"]
|
|
for v, k, val in examples:
|
|
lines.append("| %s | %s | %s |" % (v, k, str(val)[:80]))
|
|
open(out, "w", encoding="utf-8").write("\n".join(lines) + "\n")
|
|
print("junk", dict(counts))
|
|
print(out)
|