39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Собрать queue-retry.jsonl из пустых результатов LLM-сбора."""
|
|
import glob
|
|
import json
|
|
import os
|
|
|
|
OUT = "/opt/work-hub/research"
|
|
QUEUE = os.path.join(OUT, "queue-retry.jsonl")
|
|
seen = set()
|
|
rows = []
|
|
for f in glob.glob(os.path.join(OUT, "results-*.jsonl")):
|
|
if "retry" in f:
|
|
continue
|
|
for line in open(f, encoding="utf-8", errors="ignore"):
|
|
try:
|
|
r = json.loads(line)
|
|
except Exception:
|
|
continue
|
|
data = r.get("data") or []
|
|
empty = True
|
|
for x in data:
|
|
if not isinstance(x, dict):
|
|
continue
|
|
if x.get("device") or x.get("facts") or x.get("dependencies") or x.get("package"):
|
|
empty = False
|
|
break
|
|
if not empty:
|
|
continue
|
|
url = r.get("url")
|
|
if not url or url in seen:
|
|
continue
|
|
seen.add(url)
|
|
rows.append({"url": url, "product": r.get("product") or "",
|
|
"slug": r.get("slug") or "", "retry": True})
|
|
with open(QUEUE, "w", encoding="utf-8") as f:
|
|
for x in rows:
|
|
f.write(json.dumps(x, ensure_ascii=False) + "\n")
|
|
print("retry urls:", len(rows), "->", QUEUE)
|