Files

39 lines
1.2 KiB
Python

#!/usr/bin/env python3
"""Собрать queue-retry.jsonl из пустых результатов LLM-сбора."""
import glob
import json
import os
OUT = "/opt/work-hub/research"
QUEUE = os.path.join(OUT, "queue-retry.jsonl")
seen = set()
rows = []
for f in glob.glob(os.path.join(OUT, "results-*.jsonl")):
if "retry" in f:
continue
for line in open(f, encoding="utf-8", errors="ignore"):
try:
r = json.loads(line)
except Exception:
continue
data = r.get("data") or []
empty = True
for x in data:
if not isinstance(x, dict):
continue
if x.get("device") or x.get("facts") or x.get("dependencies") or x.get("package"):
empty = False
break
if not empty:
continue
url = r.get("url")
if not url or url in seen:
continue
seen.add(url)
rows.append({"url": url, "product": r.get("product") or "",
"slug": r.get("slug") or "", "retry": True})
with open(QUEUE, "w", encoding="utf-8") as f:
for x in rows:
f.write(json.dumps(x, ensure_ascii=False) + "\n")
print("retry urls:", len(rows), "->", QUEUE)