Files
buryan-infra/ops/combine/research-pack/extract_docs.py
T
2026-11-18 20:29:05 +03:00

205 lines
7.6 KiB
Python

#!/usr/bin/env python3
"""Document -> JSON extraction for official vendor docs.
Manifest JSONL:
{"id":"yealink-a40-datasheet","vendor":"yealink","source":"https://.../a40.pdf"}
{"id":"part","vendor":"poly","source":"/path/to/file.pdf"}
"""
import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import urllib.request
ROOT = os.path.dirname(os.path.abspath(__file__))
def env(k, d=""):
return os.environ.get(k, d)
def text_from_pdf(path):
try:
r = subprocess.run(["pdftotext", "-layout", path, "-"], capture_output=True, text=True, timeout=120)
if r.returncode == 0 and r.stdout.strip():
return r.stdout
except Exception:
pass
try:
import pypdf
rd = pypdf.PdfReader(path)
return "\n".join((p.extract_text() or "") for p in rd.pages[:30])
except Exception as e:
return ""
def text_from_html(raw):
raw = re.sub(r"(?is)<script.*?</script>|<style.*?</style>", " ", raw)
raw = re.sub(r"(?s)<[^>]+>", " ", raw)
return re.sub(r"\s+", " ", raw)
def load_source(src):
if src.startswith("http://") or src.startswith("https://"):
req = urllib.request.Request(src, headers={"User-Agent": "Mozilla/5.0 PrismResearch/1.0"})
with urllib.request.urlopen(req, timeout=60) as r:
data = r.read()
ctype = r.headers.get("Content-Type", "")
if "pdf" in ctype.lower() or src.lower().split("?")[0].endswith(".pdf"):
fd, p = tempfile.mkstemp(suffix=".pdf")
os.close(fd)
open(p, "wb").write(data)
try:
return text_from_pdf(p)
finally:
try:
os.unlink(p)
except Exception:
pass
return text_from_html(data.decode("utf-8", errors="ignore"))
if src.lower().endswith(".pdf"):
return text_from_pdf(src)
raw = open(src, "rb").read()
if src.lower().endswith((".html", ".htm")):
head = raw[:4000].lower()
if b"windows-1251" in head or b"cp1251" in head:
txt = raw.decode("cp1251", errors="ignore")
else:
txt = raw.decode("utf-8", errors="ignore")
return text_from_html(txt)
return raw.decode("utf-8", errors="ignore")
def call_llm(prompt):
provider = env("LLM_PROVIDER", "openai").lower()
base = env("LLM_URL", "https://api.deepseek.com").rstrip("/")
model = env("LLM_MODEL", "deepseek-chat")
key = env("LLM_API_KEY", "")
if not key and os.path.exists("/root/.deepseek-key"):
key = open("/root/.deepseek-key").read().strip()
if provider == "ollama":
url = base + "/api/generate"
body = {"model": model, "prompt": prompt, "stream": False, "format": "json", "options": {"temperature": 0.1, "num_ctx": 16384, "num_predict": 4096}}
headers = {"Content-Type": "application/json"}
else:
url = base if base.endswith("/v1") else base + "/v1"
url = url + "/chat/completions"
body = {"model": model, "temperature": 0.1, "messages": [{"role": "user", "content": prompt}]}
headers = {"Content-Type": "application/json"}
if key:
headers["Authorization"] = "Bearer " + key
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=int(env("LLM_TIMEOUT", "300"))) as r:
d = json.loads(r.read().decode())
return d["choices"][0]["message"]["content"] if provider != "ollama" else d.get("response", "")
def focus_text(text, focus):
if not focus:
return text[:60000]
low = text.lower()
f = str(focus).lower()
pos = low.find(f)
if pos < 0:
# try significant token
for tok in sorted(set(re.findall(r"[a-z0-9]{3,}", f)), key=len, reverse=True):
pos = low.find(tok)
if pos >= 0:
break
if pos < 0:
return text[:60000]
start = max(0, pos - 6000)
end = min(len(text), pos + 34000)
return text[start:end]
def prompt_for(doc, text):
level = doc.get("level") or "L1"
focus = doc.get("focus") or ""
return """Ты извлекаешь только официальные технические данные производителя.
Верни ОДИН JSON без markdown. Не придумывай. Если нет — null.
Источник: %s
Уровень источника: %s (L1 — specs; L2 — контекст, помечай source.level=L2)
Фокус: %s
Документ: %s
Схема:
{
"doc_id": "...", "vendor": "...", "source": {"url": "...", "quote": "...", "level": "%s"},
"models": [{
"model": "...", "family": "...", "role": "...", "form_factor": "...",
"market": "ru|global|null", "variant_of": null,
"specs": {"4k": null, "zoom_optical": null, "fov_h": null,
"pickup_m": null, "mic_count": null, "protocols": []},
"accessories": [], "kits": [], "restrictions": [],
"lifecycle": null
}],
"compatibility": [{"from": "...", "to": "...", "kind": "...", "condition": "...", "source": "..."}],
"unknowns": []
}
Правила: specs только из документа; market restrictions — если явно указаны;
каждый факт — с цитатой в source.quote; PVT140/A40 — variant_of, если документ говорит.
Не пересказывай статью. Если фокус задан — извлекай только эту модель.
Текст:
%s
""" % (doc.get("source"), level, focus or "—", doc.get("id"), level, text[:40000])
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--manifest", required=True)
ap.add_argument("--out", default=os.path.join(ROOT, "results-docs"))
ap.add_argument("--limit", type=int, default=0)
ap.add_argument("--dry-run", action="store_true")
a = ap.parse_args()
os.makedirs(a.out, exist_ok=True)
docs = [json.loads(l) for l in open(a.manifest, encoding="utf-8-sig") if l.strip()]
if a.limit:
docs = docs[:a.limit]
for doc in docs:
out = os.path.join(a.out, doc["id"] + ".json")
if os.path.exists(out):
print("skip", doc["id"])
continue
print("read", doc["id"], doc["source"], flush=True)
try:
text = load_source(doc["source"])
except Exception as e:
print(" read error", e)
continue
if len(text.strip()) < 200:
print(" too little text")
continue
p = prompt_for(doc, focus_text(text, doc.get("focus")))
if a.dry_run:
print(" dry chars", len(p))
continue
try:
raw = call_llm(p)
m = re.search(r"\{.*\}", raw, re.S)
obj = json.loads(m.group(0) if m else raw)
obj.setdefault("doc_id", doc["id"])
obj.setdefault("vendor", doc.get("vendor"))
if not isinstance(obj.get("source"), dict):
obj["source"] = {}
obj["source"]["level"] = doc.get("level") or "L1"
if doc.get("url"):
obj["source"]["url"] = doc["url"]
elif str(doc.get("source", "")).startswith("http"):
obj["source"]["url"] = doc["source"]
else:
obj["source"]["file"] = doc.get("source")
open(out, "w", encoding="utf-8").write(json.dumps(obj, ensure_ascii=False, indent=2))
print(" ok")
except Exception as e:
print(" llm/json error", e)
open(os.path.join(a.out, doc["id"] + ".raw.txt"), "w", encoding="utf-8").write(raw if 'raw' in locals() else "")
print("done")
if __name__ == "__main__":
sys.exit(main())