125 lines
5.1 KiB
Python
125 lines
5.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Парсер даташитов на pdfplumber: таблицы по координатам колонок.
|
|
|
|
Версия 2 — заменяет pdftotext-подход, который приклеивал строки не к тем параметрам.
|
|
"""
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
BASE = os.environ.get("DS_BASE", "/opt/work-hub/datasheets-src/Datasheet")
|
|
OUT = os.environ.get("DS_OUT", "/opt/work-hub/devices")
|
|
|
|
MODEL_RE = re.compile(r"\b(UVC\d{2,3}|CTP\d{2,3}|VCM\d{2,3}|CP\d{3,4}|CPE\d{2,3}|"
|
|
r"MeetingEye\s?\d{3}|MeetingBar\s?A\d{2}|MSpeaker\s?(?:II|Pro)|"
|
|
r"VCH\d{2}|WPP\d{2}|MVC\s?\w+|CM\d{2}|CS\d{2})\b", re.I)
|
|
|
|
|
|
def clean(v):
|
|
if v is None:
|
|
return ""
|
|
v = re.sub(r"\s*\n\s*", " ", str(v))
|
|
v = re.sub(r"[ \t]{2,}", " ", v)
|
|
return v.strip(" •-—\t")
|
|
|
|
|
|
def parse_pdf(path, pdfplumber):
|
|
"""Возвращает {параметр: [строки]} из таблиц PDF."""
|
|
specs = {}
|
|
try:
|
|
with pdfplumber.open(path) as pdf:
|
|
for page in pdf.pages:
|
|
for table in page.extract_tables() or []:
|
|
cur = None
|
|
for row in table:
|
|
cells = [clean(c) for c in (row or [])]
|
|
cells = [c for c in cells if c]
|
|
if len(cells) < 2:
|
|
# одна колонка — продолжение предыдущего параметра
|
|
if cur and cells:
|
|
specs[cur].append(cells[0])
|
|
continue
|
|
param, desc = cells[0], " ".join(cells[1:])
|
|
# мусорные заголовки: "PPaarraammeetteerr", "DDeessccrriippttiioonn"
|
|
flat = re.sub(r"(.)\1", r"\1", param).lower().strip()
|
|
if flat.startswith(("parameter", "description", "param")):
|
|
continue
|
|
if param.lower().startswith(("parameter", "param")):
|
|
continue
|
|
# заголовок параметра короткий, описание длиннее
|
|
if len(param) <= 40 and not param.endswith("."):
|
|
cur = param
|
|
specs.setdefault(cur, [])
|
|
if desc:
|
|
for part in re.split(r"\s*[•]\s*", desc):
|
|
for sub in re.split(r"(?<=[.;])\s+(?=[A-Z])", part):
|
|
sub = sub.strip(" •-—")
|
|
if len(sub) > 1:
|
|
specs[cur].append(sub)
|
|
elif cur:
|
|
specs[cur].append(param + (" " + desc if desc else ""))
|
|
except Exception as e:
|
|
return {}, str(e)
|
|
return specs, None
|
|
|
|
|
|
def models_from(specs):
|
|
found = set()
|
|
for rows in specs.values():
|
|
for r in rows:
|
|
for m in MODEL_RE.findall(r):
|
|
found.add(re.sub(r"\s+", " ", m).strip())
|
|
return sorted(found, key=str.lower)
|
|
|
|
|
|
def main():
|
|
try:
|
|
import pdfplumber
|
|
except ImportError:
|
|
print("pdfplumber не установлен"); return 1
|
|
import yaml
|
|
if not os.path.isdir(BASE):
|
|
print("нет каталога:", BASE); return 1
|
|
|
|
pdfs = []
|
|
for root, _, files in os.walk(BASE):
|
|
for f in files:
|
|
if f.lower().endswith(".pdf") and "datasheet" in f.lower():
|
|
pdfs.append(os.path.join(root, f))
|
|
print("даташитов:", len(pdfs))
|
|
|
|
ok = partial = fail = 0
|
|
for p in sorted(pdfs):
|
|
specs, err = parse_pdf(p, pdfplumber)
|
|
n = len(specs)
|
|
if n == 0:
|
|
fail += 1
|
|
continue
|
|
model = re.sub(r"(?i)yealink\s*", "", os.path.basename(p).replace(".pdf", ""))
|
|
model = re.sub(r"(?i)\s*datasheet\s*", " ", model).strip(" -()_")
|
|
data = {"device": {
|
|
"vendor": "yealink", "model": model,
|
|
"source": {"file": os.path.relpath(p, BASE), "extracted": "pdfplumber",
|
|
"verified": False},
|
|
"specs": specs,
|
|
"compatible_models": models_from(specs)}}
|
|
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", model)[:60] or "unnamed"
|
|
path = os.path.join(OUT, "yealink", safe + ".yaml")
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
yaml.safe_dump(data, f, allow_unicode=True, sort_keys=False, width=150)
|
|
if n >= 3:
|
|
ok += 1
|
|
if ok <= 5:
|
|
print(" + %-38s параметров: %2d, совместимых: %d"
|
|
% (model[:38], n, len(data["device"]["compatible_models"])))
|
|
else:
|
|
partial += 1
|
|
print("\nполноценно (3+ параметров): %d | частично: %d | не разобрано: %d"
|
|
% (ok, partial, fail))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|