Files
buryan-infra/ops/combine/research-pack/extract_docs.py
T
2026-11-18 18:58:04 +03:00

170 lines
6.3 KiB
Python

#!/usr/bin/env python3
"""Document -> JSON extraction for official vendor docs.
Manifest JSONL:
{"id":"yealink-a40-datasheet","vendor":"yealink","source":"https://.../a40.pdf"}
{"id":"part","vendor":"poly","source":"/path/to/file.pdf"}
"""
import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import urllib.request
ROOT = os.path.dirname(os.path.abspath(__file__))
def env(k, d=""):
return os.environ.get(k, d)
def text_from_pdf(path):
try:
r = subprocess.run(["pdftotext", "-layout", path, "-"], capture_output=True, text=True, timeout=120)
if r.returncode == 0 and r.stdout.strip():
return r.stdout
except Exception:
pass
try:
import pypdf
rd = pypdf.PdfReader(path)
return "\n".join((p.extract_text() or "") for p in rd.pages[:30])
except Exception as e:
return ""
def text_from_html(raw):
raw = re.sub(r"(?is)<script.*?</script>|<style.*?</style>", " ", raw)
raw = re.sub(r"(?s)<[^>]+>", " ", raw)
return re.sub(r"\s+", " ", raw)
def load_source(src):
if src.startswith("http://") or src.startswith("https://"):
req = urllib.request.Request(src, headers={"User-Agent": "Mozilla/5.0 PrismResearch/1.0"})
with urllib.request.urlopen(req, timeout=60) as r:
data = r.read()
ctype = r.headers.get("Content-Type", "")
if "pdf" in ctype.lower() or src.lower().split("?")[0].endswith(".pdf"):
fd, p = tempfile.mkstemp(suffix=".pdf")
os.close(fd)
open(p, "wb").write(data)
try:
return text_from_pdf(p)
finally:
try:
os.unlink(p)
except Exception:
pass
return text_from_html(data.decode("utf-8", errors="ignore"))
return text_from_pdf(src) if src.lower().endswith(".pdf") else open(src, encoding="utf-8", errors="ignore").read()
def call_llm(prompt):
provider = env("LLM_PROVIDER", "openai").lower()
base = env("LLM_URL", "https://api.deepseek.com").rstrip("/")
model = env("LLM_MODEL", "deepseek-chat")
key = env("LLM_API_KEY", "")
if not key and os.path.exists("/root/.deepseek-key"):
key = open("/root/.deepseek-key").read().strip()
if provider == "ollama":
url = base + "/api/generate"
body = {"model": model, "prompt": prompt, "stream": False, "options": {"temperature": 0.1}}
headers = {"Content-Type": "application/json"}
else:
url = base if base.endswith("/v1") else base + "/v1"
url = url + "/chat/completions"
body = {"model": model, "temperature": 0.1, "messages": [{"role": "user", "content": prompt}]}
headers = {"Content-Type": "application/json"}
if key:
headers["Authorization"] = "Bearer " + key
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=int(env("LLM_TIMEOUT", "300"))) as r:
d = json.loads(r.read().decode())
return d["choices"][0]["message"]["content"] if provider != "ollama" else d.get("response", "")
def prompt_for(doc, text):
return """Ты извлекаешь только официальные технические данные производителя.
Верни ОДИН JSON без markdown. Не придумывай. Если нет — null.
Источник: %s
Документ: %s
Схема:
{
"doc_id": "...", "vendor": "...", "source": {"url": "...", "quote": "..."},
"models": [{
"model": "...", "family": "...", "role": "...", "form_factor": "...",
"market": "ru|global|null", "variant_of": null,
"specs": {"4k": null, "zoom_optical": null, "fov_h": null,
"pickup_m": null, "mic_count": null, "protocols": []},
"accessories": [], "kits": [], "restrictions": [],
"lifecycle": null
}],
"compatibility": [{"from": "...", "to": "...", "kind": "...", "condition": "...", "source": "..."}],
"unknowns": []
}
Правила: specs только из документа; market restrictions — если явно указаны;
каждый факт — с цитатой в source.quote; PVT140/A40 — variant_of, если документ говорит.
Текст:
%s
""" % (doc.get("source"), doc.get("id"), text[:60000])
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--manifest", required=True)
ap.add_argument("--out", default=os.path.join(ROOT, "results-docs"))
ap.add_argument("--limit", type=int, default=0)
ap.add_argument("--dry-run", action="store_true")
a = ap.parse_args()
os.makedirs(a.out, exist_ok=True)
docs = [json.loads(l) for l in open(a.manifest, encoding="utf-8") if l.strip()]
if a.limit:
docs = docs[:a.limit]
for doc in docs:
out = os.path.join(a.out, doc["id"] + ".json")
if os.path.exists(out):
print("skip", doc["id"])
continue
print("read", doc["id"], doc["source"], flush=True)
try:
text = load_source(doc["source"])
except Exception as e:
print(" read error", e)
continue
if len(text.strip()) < 200:
print(" too little text")
continue
p = prompt_for(doc, text)
if a.dry_run:
print(" dry chars", len(p))
continue
try:
raw = call_llm(p)
m = re.search(r"\{.*\}", raw, re.S)
obj = json.loads(m.group(0) if m else raw)
obj.setdefault("doc_id", doc["id"])
obj.setdefault("vendor", doc.get("vendor"))
if not isinstance(obj.get("source"), dict):
obj["source"] = {}
if doc.get("url"):
obj["source"]["url"] = doc["url"]
elif str(doc.get("source", "")).startswith("http"):
obj["source"]["url"] = doc["source"]
else:
obj["source"]["file"] = doc.get("source")
open(out, "w", encoding="utf-8").write(json.dumps(obj, ensure_ascii=False, indent=2))
print(" ok")
except Exception as e:
print(" llm/json error", e)
open(os.path.join(a.out, doc["id"] + ".raw.txt"), "w", encoding="utf-8").write(raw if 'raw' in locals() else "")
print("done")
if __name__ == "__main__":
sys.exit(main())