[PRISM] extract_docs: официальные PDF/HTML -> JSON через LLM (проверено)
lint / hygiene (push) Failing after 45s
lint / hygiene (push) Failing after 45s
This commit is contained in:
1 parent
e57ded216a
commit
536a8fcc63
3 files changed
+186
-1
No files matched your search
@@ -20,11 +20,34 @@ research-pack/
|
||||
schema-practice.json # схема practice audit
|
||||
prompt-vendor.md
|
||||
prompt-practice.md
|
||||
run_research.py # запуск на ПК, resume, без зависимостей
|
||||
run_research.py # задачи по вендорам/практикам
|
||||
extract_docs.py # официальный PDF/HTML -> JSON через LLM
|
||||
docs-manifest-test.jsonl # пример манифеста документов
|
||||
sources/ # сюда можно положить официальные PDF/HTML (опционально)
|
||||
results/ # сюда пишутся ответы
|
||||
```
|
||||
|
||||
## Извлечение из документов (главный путь)
|
||||
|
||||
```bash
|
||||
python extract_docs.py --manifest docs-manifest.jsonl --out results-docs
|
||||
```
|
||||
|
||||
Манифест — JSONL:
|
||||
|
||||
```json
|
||||
{"id":"yealink-meetingbar-a40","vendor":"yealink","source":"https://.../a40-datasheet.pdf"}
|
||||
{"id":"yealink-compat","vendor":"yealink","source":"C:/docs/yealink-compatibility.pdf"}
|
||||
```
|
||||
|
||||
`extract_docs.py`:
|
||||
- скачивает PDF/HTML или читает локальный PDF;
|
||||
- извлекает текст (`pdftotext`/pypdf);
|
||||
- отправляет в LLM по строгой схеме;
|
||||
- пишет `results-docs/<id>.json`.
|
||||
|
||||
Проверено на официальном Poly G62 PDF: DeepSeek вернул `models/specs/source.quote`.
|
||||
|
||||
## Запуск
|
||||
|
||||
```bash
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"id":"poly-g62","vendor":"poly","source":"/opt/work-hub/research/pdf/40f1e8547e28.pdf"}
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Document -> JSON extraction for official vendor docs.
|
||||
|
||||
Manifest JSONL:
|
||||
{"id":"yealink-a40-datasheet","vendor":"yealink","source":"https://.../a40.pdf"}
|
||||
{"id":"part","vendor":"poly","source":"/path/to/file.pdf"}
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
|
||||
ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def env(k, d=""):
|
||||
return os.environ.get(k, d)
|
||||
|
||||
|
||||
def text_from_pdf(path):
|
||||
try:
|
||||
r = subprocess.run(["pdftotext", "-layout", path, "-"], capture_output=True, text=True, timeout=120)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
return r.stdout
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
import pypdf
|
||||
rd = pypdf.PdfReader(path)
|
||||
return "\n".join((p.extract_text() or "") for p in rd.pages[:30])
|
||||
except Exception as e:
|
||||
return ""
|
||||
|
||||
|
||||
def text_from_html(raw):
|
||||
raw = re.sub(r"(?is)<script.*?</script>|<style.*?</style>", " ", raw)
|
||||
raw = re.sub(r"(?s)<[^>]+>", " ", raw)
|
||||
return re.sub(r"\s+", " ", raw)
|
||||
|
||||
|
||||
def load_source(src):
|
||||
if src.startswith("http://") or src.startswith("https://"):
|
||||
req = urllib.request.Request(src, headers={"User-Agent": "Mozilla/5.0 PrismResearch/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
data = r.read()
|
||||
ctype = r.headers.get("Content-Type", "")
|
||||
if "pdf" in ctype.lower() or src.lower().split("?")[0].endswith(".pdf"):
|
||||
fd, p = tempfile.mkstemp(suffix=".pdf")
|
||||
os.close(fd)
|
||||
open(p, "wb").write(data)
|
||||
try:
|
||||
return text_from_pdf(p)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(p)
|
||||
except Exception:
|
||||
pass
|
||||
return text_from_html(data.decode("utf-8", errors="ignore"))
|
||||
return text_from_pdf(src) if src.lower().endswith(".pdf") else open(src, encoding="utf-8", errors="ignore").read()
|
||||
|
||||
|
||||
def call_llm(prompt):
|
||||
provider = env("LLM_PROVIDER", "openai").lower()
|
||||
base = env("LLM_URL", "https://api.deepseek.com").rstrip("/")
|
||||
model = env("LLM_MODEL", "deepseek-chat")
|
||||
key = env("LLM_API_KEY", "")
|
||||
if not key and os.path.exists("/root/.deepseek-key"):
|
||||
key = open("/root/.deepseek-key").read().strip()
|
||||
if provider == "ollama":
|
||||
url = base + "/api/generate"
|
||||
body = {"model": model, "prompt": prompt, "stream": False, "options": {"temperature": 0.1}}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
else:
|
||||
url = base if base.endswith("/v1") else base + "/v1"
|
||||
url = url + "/chat/completions"
|
||||
body = {"model": model, "temperature": 0.1, "messages": [{"role": "user", "content": prompt}]}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key:
|
||||
headers["Authorization"] = "Bearer " + key
|
||||
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST")
|
||||
with urllib.request.urlopen(req, timeout=int(env("LLM_TIMEOUT", "300"))) as r:
|
||||
d = json.loads(r.read().decode())
|
||||
return d["choices"][0]["message"]["content"] if provider != "ollama" else d.get("response", "")
|
||||
|
||||
|
||||
def prompt_for(doc, text):
|
||||
return """Ты извлекаешь только официальные технические данные производителя.
|
||||
Верни ОДИН JSON без markdown. Не придумывай. Если нет — null.
|
||||
Источник: %s
|
||||
Документ: %s
|
||||
|
||||
Схема:
|
||||
{
|
||||
"doc_id": "...", "vendor": "...", "source": {"url": "...", "quote": "..."},
|
||||
"models": [{
|
||||
"model": "...", "family": "...", "role": "...", "form_factor": "...",
|
||||
"market": "ru|global|null", "variant_of": null,
|
||||
"specs": {"4k": null, "zoom_optical": null, "fov_h": null,
|
||||
"pickup_m": null, "mic_count": null, "protocols": []},
|
||||
"accessories": [], "kits": [], "restrictions": [],
|
||||
"lifecycle": null
|
||||
}],
|
||||
"compatibility": [{"from": "...", "to": "...", "kind": "...", "condition": "...", "source": "..."}],
|
||||
"unknowns": []
|
||||
}
|
||||
Правила: specs только из документа; market restrictions — если явно указаны;
|
||||
каждый факт — с цитатой в source.quote; PVT140/A40 — variant_of, если документ говорит.
|
||||
Текст:
|
||||
%s
|
||||
""" % (doc.get("source"), doc.get("id"), text[:60000])
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--manifest", required=True)
|
||||
ap.add_argument("--out", default=os.path.join(ROOT, "results-docs"))
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
a = ap.parse_args()
|
||||
os.makedirs(a.out, exist_ok=True)
|
||||
docs = [json.loads(l) for l in open(a.manifest, encoding="utf-8") if l.strip()]
|
||||
if a.limit:
|
||||
docs = docs[:a.limit]
|
||||
for doc in docs:
|
||||
out = os.path.join(a.out, doc["id"] + ".json")
|
||||
if os.path.exists(out):
|
||||
print("skip", doc["id"])
|
||||
continue
|
||||
print("read", doc["id"], doc["source"], flush=True)
|
||||
try:
|
||||
text = load_source(doc["source"])
|
||||
except Exception as e:
|
||||
print(" read error", e)
|
||||
continue
|
||||
if len(text.strip()) < 200:
|
||||
print(" too little text")
|
||||
continue
|
||||
p = prompt_for(doc, text)
|
||||
if a.dry_run:
|
||||
print(" dry chars", len(p))
|
||||
continue
|
||||
try:
|
||||
raw = call_llm(p)
|
||||
m = re.search(r"\{.*\}", raw, re.S)
|
||||
obj = json.loads(m.group(0) if m else raw)
|
||||
obj.setdefault("doc_id", doc["id"])
|
||||
obj.setdefault("vendor", doc.get("vendor"))
|
||||
open(out, "w", encoding="utf-8").write(json.dumps(obj, ensure_ascii=False, indent=2))
|
||||
print(" ok")
|
||||
except Exception as e:
|
||||
print(" llm/json error", e)
|
||||
open(os.path.join(a.out, doc["id"] + ".raw.txt"), "w", encoding="utf-8").write(raw if 'raw' in locals() else "")
|
||||
print("done")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user