Files
buryan-infra/ops/work/tools/harvest_datasheets.py

139 lines
5.1 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Парсер даташитов Yealink: PDF -> YAML со спецификациями и провенансом.
Основано на реальной структуре: таблица «Parameter -> Description»,
заголовки бывают разорваны по буквам (Chips and System s).
"""
import os
import re
import subprocess
import sys
BASE = os.environ.get("DS_BASE", "/opt/work-hub/datasheets-src")
OUT = "/opt/work-hub/devices"
# заголовки параметров, которые встречаются в даташитах Yealink
PARAMS = [
"Chips and Systems", "Camera", "Audio Device", "Accessories", "Controller",
"Content Sharing", "NAT and Firewall Traversal", "Network and Security Features",
"Physical", "Interface", "Video", "Audio", "Power", "Environment",
]
MODEL_RE = re.compile(r"\b(UVC\d{2,3}|CTP\d{2,3}|VCM\d{2,3}|CP\d{3,4}|CPE\d{2,3}|"
r"MeetingEye\s?\d{3}|MeetingBar\s?A\d{2}|MSpeaker\s?(?:II|Pro)|"
r"VCH\d{2}|WPP\d{2}|MVC\d{3}|CM\d{2}|CS\d{2})\b", re.I)
def pdf_text(path):
r = subprocess.run(["pdftotext", "-layout", path, "-"],
capture_output=True, text=True, timeout=120)
return r.stdout or ""
def normalize(t):
"""Склеиваем слова, разорванные переносом: 'Chips and System s' -> 'Chips and Systems'."""
t = re.sub(r"(\w)\s*\n\s*([a-z])\b", r"\1\2", t) # System + s
t = re.sub(r"[ \t]{2,}", " ", t)
return t
def find_param_header(line):
"""Определяет, какой параметр начинается в строке (с учётом разрывов)."""
flat = re.sub(r"\s+", " ", line).strip().lower()
for p in PARAMS:
key = p.lower()
# допускаем разрывы: сравниваем только буквы
if re.sub(r"[^a-z]", "", flat).startswith(re.sub(r"[^a-z]", "", key)):
return p
return None
def parse_specs(text):
"""Извлекает {параметр: [строки описания]} из раздела Specification."""
lines = normalize(text).splitlines()
start = None
for i, ln in enumerate(lines):
if re.search(r"^\s*Specification\s*$", ln, re.I):
start = i
break
if start is None:
# ищем строку с "Parameter" и "Description"
for i, ln in enumerate(lines):
if re.search(r"Parameter.*Description", ln, re.I):
start = i
break
if start is None:
return {}, False
specs, cur = {}, None
for ln in lines[start:start + 200]:
if not ln.strip():
continue
p = find_param_header(ln)
if p:
cur = p
specs.setdefault(cur, [])
rest = re.sub(r"^\s*\S+(\s+\S+)*?\s{2,}", "", ln).strip()
if rest:
specs[cur].append(rest)
continue
if cur:
specs[cur].append(ln.strip())
return specs, True
def models_from(specs):
found = set()
for rows in specs.values():
for r in rows:
for m in MODEL_RE.findall(r):
found.add(re.sub(r"\s+", " ", m).strip())
return sorted(found, key=str.lower)
def main():
os.makedirs(OUT, exist_ok=True)
vendor_dir = os.path.join(BASE, "Yealink")
pdfs = []
for root, _, files in os.walk(vendor_dir):
for f in files:
if f.lower().endswith(".pdf") and "datasheet" in f.lower():
pdfs.append(os.path.join(root, f))
print("даташитов найдено:", len(pdfs))
ok = fail = 0
import yaml
for p in sorted(pdfs):
text = pdf_text(p)
if not text.strip():
fail += 1
continue
specs, found = parse_specs(text)
if not found:
fail += 1
continue
model = os.path.basename(p).replace(".pdf", "")
model = re.sub(r"(?i)yealink\s*", "", model).replace("Datasheet", "").strip(" -()")
rel = os.path.relpath(p, BASE)
data = {"device": {
"vendor": "yealink",
"model": model,
"source": {"file": rel, "extracted": "pdftotext -layout", "verified": False},
"specs": {k: [re.sub(r"^\s*[•\-]\s*", "", x) for x in v if x.strip()]
for k, v in specs.items()},
"compatible_models": models_from(specs),
}}
safe = re.sub(r"[^A-Za-z0-9._-]+", "_", model)[:60] or "unnamed"
path = os.path.join(OUT, "yealink", safe + ".yaml")
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
yaml.safe_dump(data, f, allow_unicode=True, sort_keys=False, width=150)
ok += 1
if ok <= 5:
print(" + %-42s параметров: %2d, совместимых: %d"
% (model[:42], len(specs), len(data["device"]["compatible_models"])))
print("\nразобрано: %d | не удалось: %d" % (ok, fail))
if __name__ == "__main__":
sys.exit(main())