Files
buryan-infra/ops/work/tools/harvest_v2.py

125 lines
5.1 KiB
Python

#!/usr/bin/env python3
"""Парсер даташитов на pdfplumber: таблицы по координатам колонок.
Версия 2 — заменяет pdftotext-подход, который приклеивал строки не к тем параметрам.
"""
import os
import re
import sys
BASE = os.environ.get("DS_BASE", "/opt/work-hub/datasheets-src/Datasheet")
OUT = os.environ.get("DS_OUT", "/opt/work-hub/devices")
MODEL_RE = re.compile(r"\b(UVC\d{2,3}|CTP\d{2,3}|VCM\d{2,3}|CP\d{3,4}|CPE\d{2,3}|"
r"MeetingEye\s?\d{3}|MeetingBar\s?A\d{2}|MSpeaker\s?(?:II|Pro)|"
r"VCH\d{2}|WPP\d{2}|MVC\s?\w+|CM\d{2}|CS\d{2})\b", re.I)
def clean(v):
if v is None:
return ""
v = re.sub(r"\s*\n\s*", " ", str(v))
v = re.sub(r"[ \t]{2,}", " ", v)
return v.strip(" •-—\t")
def parse_pdf(path, pdfplumber):
"""Возвращает {параметр: [строки]} из таблиц PDF."""
specs = {}
try:
with pdfplumber.open(path) as pdf:
for page in pdf.pages:
for table in page.extract_tables() or []:
cur = None
for row in table:
cells = [clean(c) for c in (row or [])]
cells = [c for c in cells if c]
if len(cells) < 2:
# одна колонка — продолжение предыдущего параметра
if cur and cells:
specs[cur].append(cells[0])
continue
param, desc = cells[0], " ".join(cells[1:])
# мусорные заголовки: "PPaarraammeetteerr", "DDeessccrriippttiioonn"
flat = re.sub(r"(.)\1", r"\1", param).lower().strip()
if flat.startswith(("parameter", "description", "param")):
continue
if param.lower().startswith(("parameter", "param")):
continue
# заголовок параметра короткий, описание длиннее
if len(param) <= 40 and not param.endswith("."):
cur = param
specs.setdefault(cur, [])
if desc:
for part in re.split(r"\s*[•]\s*", desc):
for sub in re.split(r"(?<=[.;])\s+(?=[A-Z])", part):
sub = sub.strip(" •-—")
if len(sub) > 1:
specs[cur].append(sub)
elif cur:
specs[cur].append(param + (" " + desc if desc else ""))
except Exception as e:
return {}, str(e)
return specs, None
def models_from(specs):
found = set()
for rows in specs.values():
for r in rows:
for m in MODEL_RE.findall(r):
found.add(re.sub(r"\s+", " ", m).strip())
return sorted(found, key=str.lower)
def main():
try:
import pdfplumber
except ImportError:
print("pdfplumber не установлен"); return 1
import yaml
if not os.path.isdir(BASE):
print("нет каталога:", BASE); return 1
pdfs = []
for root, _, files in os.walk(BASE):
for f in files:
if f.lower().endswith(".pdf") and "datasheet" in f.lower():
pdfs.append(os.path.join(root, f))
print("даташитов:", len(pdfs))
ok = partial = fail = 0
for p in sorted(pdfs):
specs, err = parse_pdf(p, pdfplumber)
n = len(specs)
if n == 0:
fail += 1
continue
model = re.sub(r"(?i)yealink\s*", "", os.path.basename(p).replace(".pdf", ""))
model = re.sub(r"(?i)\s*datasheet\s*", " ", model).strip(" -()_")
data = {"device": {
"vendor": "yealink", "model": model,
"source": {"file": os.path.relpath(p, BASE), "extracted": "pdfplumber",
"verified": False},
"specs": specs,
"compatible_models": models_from(specs)}}
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", model)[:60] or "unnamed"
path = os.path.join(OUT, "yealink", safe + ".yaml")
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
yaml.safe_dump(data, f, allow_unicode=True, sort_keys=False, width=150)
if n >= 3:
ok += 1
if ok <= 5:
print(" + %-38s параметров: %2d, совместимых: %d"
% (model[:38], n, len(data["device"]["compatible_models"])))
else:
partial += 1
print("\nполноценно (3+ параметров): %d | частично: %d | не разобрано: %d"
% (ok, partial, fail))
return 0
if __name__ == "__main__":
sys.exit(main())