#!/usr/bin/env python3 """Извлечение таблиц совместимости из ProseMirror-JSON страниц support.yealink.com. Страница — Nuxt SPA; контент лежит в __NUXT_DATA__ как JSON-строка {"type":"doc","content":[...]}. Таблицы — узлы type=table. """ import json import os import re import subprocess import sys UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120" OUT = os.environ.get("COMPAT_OUT", "/opt/work-hub/compat") def fetch(url, dest="/tmp/page.html"): subprocess.run(["curl", "-sL", "-m", "60", "-A", UA, "-o", dest, url], capture_output=True, text=True, timeout=120) return open(dest, encoding="utf-8", errors="ignore").read() def prosemirror_docs(html): """Все ProseMirror-документы со страницы.""" m = re.search(r']*id="__NUXT_DATA__"[^>]*>(.*?)', html, re.S) if not m: return [] try: data = json.loads(m.group(1).strip()) except Exception: return [] docs = [] def walk(o): if isinstance(o, str): s = o.strip() if s.startswith('{"type":"doc"'): try: docs.append(json.loads(s)) except Exception: pass elif isinstance(o, list): for v in o: walk(v) elif isinstance(o, dict): for v in o.values(): walk(v) walk(data) return docs def text_of(node): if not isinstance(node, dict): return "" if node.get("type") == "text": return node.get("text", "") parts = [text_of(c) for c in node.get("content") or []] return "".join(parts) def tables_of(doc): """Возвращает список таблиц: [[cell, cell, ...], ...].""" out = [] def walk(n): if not isinstance(n, dict): return if n.get("type") == "table": rows = [] for row in n.get("content") or []: if row.get("type") != "tableRow": continue cells = [] for cell in row.get("content") or []: cells.append(re.sub(r"\s+", " ", text_of(cell)).strip()) if cells: rows.append(cells) if rows: out.append(rows) return for c in n.get("content") or []: walk(c) walk(doc) return out def main(): urls = sys.argv[1:] if not urls: print("использование: fetch_doc_tables.py [...]") return 1 import yaml os.makedirs(OUT, exist_ok=True) for url in urls: html = fetch(url) docs = prosemirror_docs(html) parts = [p for p in url.rstrip("/").split("/") if p] doc_hash = parts[-1][:8] if parts else "unknown" slug = (parts[-2] if len(parts) > 1 else "doc") print("\n=== %s ===" % slug) print(" документов ProseMirror: %d" % len(docs)) all_tables = [] for d in docs: all_tables += tables_of(d) print(" таблиц: %d" % len(all_tables)) for i, t in enumerate(all_tables[:3]): print(" таблица %d: %d строк" % (i + 1, len(t))) for row in t[:4]: print(" ", " | ".join(c[:22] for c in row[:5])) path = os.path.join(OUT, "raw", "%s-%s.yaml" % (re.sub(r"[^a-z0-9-]", "_", slug.lower()), doc_hash)) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: yaml.safe_dump({"source": url, "tables": all_tables}, f, allow_unicode=True, sort_keys=False, width=200) print(" сохранено:", path) return 0 if __name__ == "__main__": sys.exit(main())