From 536a8fcc63b4b82f7d4604c43f23b8ba5ef989dd Mon Sep 17 00:00:00 2001 From: Roman Buryan Date: Wed, 18 Nov 2026 18:49:56 +0300 Subject: [PATCH] =?UTF-8?q?[PRISM]=20extract=5Fdocs:=20=D0=BE=D1=84=D0=B8?= =?UTF-8?q?=D1=86=D0=B8=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D0=B5=20PDF/HTML=20-?= =?UTF-8?q?>=20JSON=20=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20LLM=20(=D0=BF=D1=80?= =?UTF-8?q?=D0=BE=D0=B2=D0=B5=D1=80=D0=B5=D0=BD=D0=BE)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ops/combine/research-pack/README.md | 25 ++- .../research-pack/docs-manifest-test.jsonl | 1 + ops/combine/research-pack/extract_docs.py | 161 ++++++++++++++++++ 3 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 ops/combine/research-pack/docs-manifest-test.jsonl create mode 100644 ops/combine/research-pack/extract_docs.py diff --git a/ops/combine/research-pack/README.md b/ops/combine/research-pack/README.md index 1ebcf5e..4558634 100644 --- a/ops/combine/research-pack/README.md +++ b/ops/combine/research-pack/README.md @@ -20,11 +20,34 @@ research-pack/ schema-practice.json # схема practice audit prompt-vendor.md prompt-practice.md - run_research.py # запуск на ПК, resume, без зависимостей + run_research.py # задачи по вендорам/практикам + extract_docs.py # официальный PDF/HTML -> JSON через LLM + docs-manifest-test.jsonl # пример манифеста документов sources/ # сюда можно положить официальные PDF/HTML (опционально) results/ # сюда пишутся ответы ``` +## Извлечение из документов (главный путь) + +```bash +python extract_docs.py --manifest docs-manifest.jsonl --out results-docs +``` + +Манифест — JSONL: + +```json +{"id":"yealink-meetingbar-a40","vendor":"yealink","source":"https://.../a40-datasheet.pdf"} +{"id":"yealink-compat","vendor":"yealink","source":"C:/docs/yealink-compatibility.pdf"} +``` + +`extract_docs.py`: +- скачивает PDF/HTML или читает локальный PDF; +- извлекает текст (`pdftotext`/pypdf); +- отправляет в LLM по строгой схеме; +- пишет `results-docs/.json`. + +Проверено на официальном Poly G62 PDF: DeepSeek вернул `models/specs/source.quote`. + ## Запуск ```bash diff --git a/ops/combine/research-pack/docs-manifest-test.jsonl b/ops/combine/research-pack/docs-manifest-test.jsonl new file mode 100644 index 0000000..7a1f986 --- /dev/null +++ b/ops/combine/research-pack/docs-manifest-test.jsonl @@ -0,0 +1 @@ +{"id":"poly-g62","vendor":"poly","source":"/opt/work-hub/research/pdf/40f1e8547e28.pdf"} diff --git a/ops/combine/research-pack/extract_docs.py b/ops/combine/research-pack/extract_docs.py new file mode 100644 index 0000000..a540ef3 --- /dev/null +++ b/ops/combine/research-pack/extract_docs.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Document -> JSON extraction for official vendor docs. + +Manifest JSONL: + {"id":"yealink-a40-datasheet","vendor":"yealink","source":"https://.../a40.pdf"} + {"id":"part","vendor":"poly","source":"/path/to/file.pdf"} +""" +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import urllib.request + +ROOT = os.path.dirname(os.path.abspath(__file__)) + + +def env(k, d=""): + return os.environ.get(k, d) + + +def text_from_pdf(path): + try: + r = subprocess.run(["pdftotext", "-layout", path, "-"], capture_output=True, text=True, timeout=120) + if r.returncode == 0 and r.stdout.strip(): + return r.stdout + except Exception: + pass + try: + import pypdf + rd = pypdf.PdfReader(path) + return "\n".join((p.extract_text() or "") for p in rd.pages[:30]) + except Exception as e: + return "" + + +def text_from_html(raw): + raw = re.sub(r"(?is)|", " ", raw) + raw = re.sub(r"(?s)<[^>]+>", " ", raw) + return re.sub(r"\s+", " ", raw) + + +def load_source(src): + if src.startswith("http://") or src.startswith("https://"): + req = urllib.request.Request(src, headers={"User-Agent": "Mozilla/5.0 PrismResearch/1.0"}) + with urllib.request.urlopen(req, timeout=60) as r: + data = r.read() + ctype = r.headers.get("Content-Type", "") + if "pdf" in ctype.lower() or src.lower().split("?")[0].endswith(".pdf"): + fd, p = tempfile.mkstemp(suffix=".pdf") + os.close(fd) + open(p, "wb").write(data) + try: + return text_from_pdf(p) + finally: + try: + os.unlink(p) + except Exception: + pass + return text_from_html(data.decode("utf-8", errors="ignore")) + return text_from_pdf(src) if src.lower().endswith(".pdf") else open(src, encoding="utf-8", errors="ignore").read() + + +def call_llm(prompt): + provider = env("LLM_PROVIDER", "openai").lower() + base = env("LLM_URL", "https://api.deepseek.com").rstrip("/") + model = env("LLM_MODEL", "deepseek-chat") + key = env("LLM_API_KEY", "") + if not key and os.path.exists("/root/.deepseek-key"): + key = open("/root/.deepseek-key").read().strip() + if provider == "ollama": + url = base + "/api/generate" + body = {"model": model, "prompt": prompt, "stream": False, "options": {"temperature": 0.1}} + headers = {"Content-Type": "application/json"} + else: + url = base if base.endswith("/v1") else base + "/v1" + url = url + "/chat/completions" + body = {"model": model, "temperature": 0.1, "messages": [{"role": "user", "content": prompt}]} + headers = {"Content-Type": "application/json"} + if key: + headers["Authorization"] = "Bearer " + key + req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=int(env("LLM_TIMEOUT", "300"))) as r: + d = json.loads(r.read().decode()) + return d["choices"][0]["message"]["content"] if provider != "ollama" else d.get("response", "") + + +def prompt_for(doc, text): + return """Ты извлекаешь только официальные технические данные производителя. +Верни ОДИН JSON без markdown. Не придумывай. Если нет — null. +Источник: %s +Документ: %s + +Схема: +{ + "doc_id": "...", "vendor": "...", "source": {"url": "...", "quote": "..."}, + "models": [{ + "model": "...", "family": "...", "role": "...", "form_factor": "...", + "market": "ru|global|null", "variant_of": null, + "specs": {"4k": null, "zoom_optical": null, "fov_h": null, + "pickup_m": null, "mic_count": null, "protocols": []}, + "accessories": [], "kits": [], "restrictions": [], + "lifecycle": null + }], + "compatibility": [{"from": "...", "to": "...", "kind": "...", "condition": "...", "source": "..."}], + "unknowns": [] +} +Правила: specs только из документа; market restrictions — если явно указаны; +каждый факт — с цитатой в source.quote; PVT140/A40 — variant_of, если документ говорит. +Текст: +%s +""" % (doc.get("source"), doc.get("id"), text[:60000]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--manifest", required=True) + ap.add_argument("--out", default=os.path.join(ROOT, "results-docs")) + ap.add_argument("--limit", type=int, default=0) + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args() + os.makedirs(a.out, exist_ok=True) + docs = [json.loads(l) for l in open(a.manifest, encoding="utf-8") if l.strip()] + if a.limit: + docs = docs[:a.limit] + for doc in docs: + out = os.path.join(a.out, doc["id"] + ".json") + if os.path.exists(out): + print("skip", doc["id"]) + continue + print("read", doc["id"], doc["source"], flush=True) + try: + text = load_source(doc["source"]) + except Exception as e: + print(" read error", e) + continue + if len(text.strip()) < 200: + print(" too little text") + continue + p = prompt_for(doc, text) + if a.dry_run: + print(" dry chars", len(p)) + continue + try: + raw = call_llm(p) + m = re.search(r"\{.*\}", raw, re.S) + obj = json.loads(m.group(0) if m else raw) + obj.setdefault("doc_id", doc["id"]) + obj.setdefault("vendor", doc.get("vendor")) + open(out, "w", encoding="utf-8").write(json.dumps(obj, ensure_ascii=False, indent=2)) + print(" ok") + except Exception as e: + print(" llm/json error", e) + open(os.path.join(a.out, doc["id"] + ".raw.txt"), "w", encoding="utf-8").write(raw if 'raw' in locals() else "") + print("done") + + +if __name__ == "__main__": + sys.exit(main())