170 lines
6.3 KiB
Python
170 lines
6.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Document -> JSON extraction for official vendor docs.
|
|
|
|
Manifest JSONL:
|
|
{"id":"yealink-a40-datasheet","vendor":"yealink","source":"https://.../a40.pdf"}
|
|
{"id":"part","vendor":"poly","source":"/path/to/file.pdf"}
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.request
|
|
|
|
ROOT = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
|
|
def env(k, d=""):
|
|
return os.environ.get(k, d)
|
|
|
|
|
|
def text_from_pdf(path):
|
|
try:
|
|
r = subprocess.run(["pdftotext", "-layout", path, "-"], capture_output=True, text=True, timeout=120)
|
|
if r.returncode == 0 and r.stdout.strip():
|
|
return r.stdout
|
|
except Exception:
|
|
pass
|
|
try:
|
|
import pypdf
|
|
rd = pypdf.PdfReader(path)
|
|
return "\n".join((p.extract_text() or "") for p in rd.pages[:30])
|
|
except Exception as e:
|
|
return ""
|
|
|
|
|
|
def text_from_html(raw):
|
|
raw = re.sub(r"(?is)<script.*?</script>|<style.*?</style>", " ", raw)
|
|
raw = re.sub(r"(?s)<[^>]+>", " ", raw)
|
|
return re.sub(r"\s+", " ", raw)
|
|
|
|
|
|
def load_source(src):
|
|
if src.startswith("http://") or src.startswith("https://"):
|
|
req = urllib.request.Request(src, headers={"User-Agent": "Mozilla/5.0 PrismResearch/1.0"})
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
data = r.read()
|
|
ctype = r.headers.get("Content-Type", "")
|
|
if "pdf" in ctype.lower() or src.lower().split("?")[0].endswith(".pdf"):
|
|
fd, p = tempfile.mkstemp(suffix=".pdf")
|
|
os.close(fd)
|
|
open(p, "wb").write(data)
|
|
try:
|
|
return text_from_pdf(p)
|
|
finally:
|
|
try:
|
|
os.unlink(p)
|
|
except Exception:
|
|
pass
|
|
return text_from_html(data.decode("utf-8", errors="ignore"))
|
|
return text_from_pdf(src) if src.lower().endswith(".pdf") else open(src, encoding="utf-8", errors="ignore").read()
|
|
|
|
|
|
def call_llm(prompt):
|
|
provider = env("LLM_PROVIDER", "openai").lower()
|
|
base = env("LLM_URL", "https://api.deepseek.com").rstrip("/")
|
|
model = env("LLM_MODEL", "deepseek-chat")
|
|
key = env("LLM_API_KEY", "")
|
|
if not key and os.path.exists("/root/.deepseek-key"):
|
|
key = open("/root/.deepseek-key").read().strip()
|
|
if provider == "ollama":
|
|
url = base + "/api/generate"
|
|
body = {"model": model, "prompt": prompt, "stream": False, "options": {"temperature": 0.1}}
|
|
headers = {"Content-Type": "application/json"}
|
|
else:
|
|
url = base if base.endswith("/v1") else base + "/v1"
|
|
url = url + "/chat/completions"
|
|
body = {"model": model, "temperature": 0.1, "messages": [{"role": "user", "content": prompt}]}
|
|
headers = {"Content-Type": "application/json"}
|
|
if key:
|
|
headers["Authorization"] = "Bearer " + key
|
|
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST")
|
|
with urllib.request.urlopen(req, timeout=int(env("LLM_TIMEOUT", "300"))) as r:
|
|
d = json.loads(r.read().decode())
|
|
return d["choices"][0]["message"]["content"] if provider != "ollama" else d.get("response", "")
|
|
|
|
|
|
def prompt_for(doc, text):
|
|
return """Ты извлекаешь только официальные технические данные производителя.
|
|
Верни ОДИН JSON без markdown. Не придумывай. Если нет — null.
|
|
Источник: %s
|
|
Документ: %s
|
|
|
|
Схема:
|
|
{
|
|
"doc_id": "...", "vendor": "...", "source": {"url": "...", "quote": "..."},
|
|
"models": [{
|
|
"model": "...", "family": "...", "role": "...", "form_factor": "...",
|
|
"market": "ru|global|null", "variant_of": null,
|
|
"specs": {"4k": null, "zoom_optical": null, "fov_h": null,
|
|
"pickup_m": null, "mic_count": null, "protocols": []},
|
|
"accessories": [], "kits": [], "restrictions": [],
|
|
"lifecycle": null
|
|
}],
|
|
"compatibility": [{"from": "...", "to": "...", "kind": "...", "condition": "...", "source": "..."}],
|
|
"unknowns": []
|
|
}
|
|
Правила: specs только из документа; market restrictions — если явно указаны;
|
|
каждый факт — с цитатой в source.quote; PVT140/A40 — variant_of, если документ говорит.
|
|
Текст:
|
|
%s
|
|
""" % (doc.get("source"), doc.get("id"), text[:60000])
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--manifest", required=True)
|
|
ap.add_argument("--out", default=os.path.join(ROOT, "results-docs"))
|
|
ap.add_argument("--limit", type=int, default=0)
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
a = ap.parse_args()
|
|
os.makedirs(a.out, exist_ok=True)
|
|
docs = [json.loads(l) for l in open(a.manifest, encoding="utf-8") if l.strip()]
|
|
if a.limit:
|
|
docs = docs[:a.limit]
|
|
for doc in docs:
|
|
out = os.path.join(a.out, doc["id"] + ".json")
|
|
if os.path.exists(out):
|
|
print("skip", doc["id"])
|
|
continue
|
|
print("read", doc["id"], doc["source"], flush=True)
|
|
try:
|
|
text = load_source(doc["source"])
|
|
except Exception as e:
|
|
print(" read error", e)
|
|
continue
|
|
if len(text.strip()) < 200:
|
|
print(" too little text")
|
|
continue
|
|
p = prompt_for(doc, text)
|
|
if a.dry_run:
|
|
print(" dry chars", len(p))
|
|
continue
|
|
try:
|
|
raw = call_llm(p)
|
|
m = re.search(r"\{.*\}", raw, re.S)
|
|
obj = json.loads(m.group(0) if m else raw)
|
|
obj.setdefault("doc_id", doc["id"])
|
|
obj.setdefault("vendor", doc.get("vendor"))
|
|
if not isinstance(obj.get("source"), dict):
|
|
obj["source"] = {}
|
|
if doc.get("url"):
|
|
obj["source"]["url"] = doc["url"]
|
|
elif str(doc.get("source", "")).startswith("http"):
|
|
obj["source"]["url"] = doc["source"]
|
|
else:
|
|
obj["source"]["file"] = doc.get("source")
|
|
open(out, "w", encoding="utf-8").write(json.dumps(obj, ensure_ascii=False, indent=2))
|
|
print(" ok")
|
|
except Exception as e:
|
|
print(" llm/json error", e)
|
|
open(os.path.join(a.out, doc["id"] + ".raw.txt"), "w", encoding="utf-8").write(raw if 'raw' in locals() else "")
|
|
print("done")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|