[PRISM] extract_docs: официальные PDF/HTML -> JSON через LLM (проверено)
lint / hygiene (push) Failing after 45s

This commit is contained in:
root committed 2026-11-18 18:49:56 +03:00
1 parent e57ded216a
commit 536a8fcc63
3 files changed
+186 -1

No files matched your search

+24 -1
View File
@@ -20,11 +20,34 @@ research-pack/
schema-practice.json # схема practice audit
prompt-vendor.md
prompt-practice.md
run_research.py # запуск на ПК, resume, без зависимостей
run_research.py # задачи по вендорам/практикам
extract_docs.py # официальный PDF/HTML -> JSON через LLM
docs-manifest-test.jsonl # пример манифеста документов
sources/ # сюда можно положить официальные PDF/HTML (опционально)
results/ # сюда пишутся ответы
```
## Извлечение из документов (главный путь)
```bash
python extract_docs.py --manifest docs-manifest.jsonl --out results-docs
```
Манифест — JSONL:
```json
{"id":"yealink-meetingbar-a40","vendor":"yealink","source":"https://.../a40-datasheet.pdf"}
{"id":"yealink-compat","vendor":"yealink","source":"C:/docs/yealink-compatibility.pdf"}
```
`extract_docs.py`:
- скачивает PDF/HTML или читает локальный PDF;
- извлекает текст (`pdftotext`/pypdf);
- отправляет в LLM по строгой схеме;
- пишет `results-docs/<id>.json`.
Проверено на официальном Poly G62 PDF: DeepSeek вернул `models/specs/source.quote`.
## Запуск
```bash
@@ -0,0 +1 @@
{"id":"poly-g62","vendor":"poly","source":"/opt/work-hub/research/pdf/40f1e8547e28.pdf"}
+161
View File
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
"""Document -> JSON extraction for official vendor docs.
Manifest JSONL:
{"id":"yealink-a40-datasheet","vendor":"yealink","source":"https://.../a40.pdf"}
{"id":"part","vendor":"poly","source":"/path/to/file.pdf"}
"""
import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import urllib.request
ROOT = os.path.dirname(os.path.abspath(__file__))
def env(k, d=""):
return os.environ.get(k, d)
def text_from_pdf(path):
try:
r = subprocess.run(["pdftotext", "-layout", path, "-"], capture_output=True, text=True, timeout=120)
if r.returncode == 0 and r.stdout.strip():
return r.stdout
except Exception:
pass
try:
import pypdf
rd = pypdf.PdfReader(path)
return "\n".join((p.extract_text() or "") for p in rd.pages[:30])
except Exception as e:
return ""
def text_from_html(raw):
raw = re.sub(r"(?is)<script.*?</script>|<style.*?</style>", " ", raw)
raw = re.sub(r"(?s)<[^>]+>", " ", raw)
return re.sub(r"\s+", " ", raw)
def load_source(src):
if src.startswith("http://") or src.startswith("https://"):
req = urllib.request.Request(src, headers={"User-Agent": "Mozilla/5.0 PrismResearch/1.0"})
with urllib.request.urlopen(req, timeout=60) as r:
data = r.read()
ctype = r.headers.get("Content-Type", "")
if "pdf" in ctype.lower() or src.lower().split("?")[0].endswith(".pdf"):
fd, p = tempfile.mkstemp(suffix=".pdf")
os.close(fd)
open(p, "wb").write(data)
try:
return text_from_pdf(p)
finally:
try:
os.unlink(p)
except Exception:
pass
return text_from_html(data.decode("utf-8", errors="ignore"))
return text_from_pdf(src) if src.lower().endswith(".pdf") else open(src, encoding="utf-8", errors="ignore").read()
def call_llm(prompt):
provider = env("LLM_PROVIDER", "openai").lower()
base = env("LLM_URL", "https://api.deepseek.com").rstrip("/")
model = env("LLM_MODEL", "deepseek-chat")
key = env("LLM_API_KEY", "")
if not key and os.path.exists("/root/.deepseek-key"):
key = open("/root/.deepseek-key").read().strip()
if provider == "ollama":
url = base + "/api/generate"
body = {"model": model, "prompt": prompt, "stream": False, "options": {"temperature": 0.1}}
headers = {"Content-Type": "application/json"}
else:
url = base if base.endswith("/v1") else base + "/v1"
url = url + "/chat/completions"
body = {"model": model, "temperature": 0.1, "messages": [{"role": "user", "content": prompt}]}
headers = {"Content-Type": "application/json"}
if key:
headers["Authorization"] = "Bearer " + key
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=int(env("LLM_TIMEOUT", "300"))) as r:
d = json.loads(r.read().decode())
return d["choices"][0]["message"]["content"] if provider != "ollama" else d.get("response", "")
def prompt_for(doc, text):
return """Ты извлекаешь только официальные технические данные производителя.
Верни ОДИН JSON без markdown. Не придумывай. Если нет — null.
Источник: %s
Документ: %s
Схема:
{
"doc_id": "...", "vendor": "...", "source": {"url": "...", "quote": "..."},
"models": [{
"model": "...", "family": "...", "role": "...", "form_factor": "...",
"market": "ru|global|null", "variant_of": null,
"specs": {"4k": null, "zoom_optical": null, "fov_h": null,
"pickup_m": null, "mic_count": null, "protocols": []},
"accessories": [], "kits": [], "restrictions": [],
"lifecycle": null
}],
"compatibility": [{"from": "...", "to": "...", "kind": "...", "condition": "...", "source": "..."}],
"unknowns": []
}
Правила: specs только из документа; market restrictions — если явно указаны;
каждый факт — с цитатой в source.quote; PVT140/A40 — variant_of, если документ говорит.
Текст:
%s
""" % (doc.get("source"), doc.get("id"), text[:60000])
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--manifest", required=True)
ap.add_argument("--out", default=os.path.join(ROOT, "results-docs"))
ap.add_argument("--limit", type=int, default=0)
ap.add_argument("--dry-run", action="store_true")
a = ap.parse_args()
os.makedirs(a.out, exist_ok=True)
docs = [json.loads(l) for l in open(a.manifest, encoding="utf-8") if l.strip()]
if a.limit:
docs = docs[:a.limit]
for doc in docs:
out = os.path.join(a.out, doc["id"] + ".json")
if os.path.exists(out):
print("skip", doc["id"])
continue
print("read", doc["id"], doc["source"], flush=True)
try:
text = load_source(doc["source"])
except Exception as e:
print(" read error", e)
continue
if len(text.strip()) < 200:
print(" too little text")
continue
p = prompt_for(doc, text)
if a.dry_run:
print(" dry chars", len(p))
continue
try:
raw = call_llm(p)
m = re.search(r"\{.*\}", raw, re.S)
obj = json.loads(m.group(0) if m else raw)
obj.setdefault("doc_id", doc["id"])
obj.setdefault("vendor", doc.get("vendor"))
open(out, "w", encoding="utf-8").write(json.dumps(obj, ensure_ascii=False, indent=2))
print(" ok")
except Exception as e:
print(" llm/json error", e)
open(os.path.join(a.out, doc["id"] + ".raw.txt"), "w", encoding="utf-8").write(raw if 'raw' in locals() else "")
print("done")
if __name__ == "__main__":
sys.exit(main())