121 lines
3.8 KiB
Python
121 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Извлечение таблиц совместимости из ProseMirror-JSON страниц support.yealink.com.
|
|
|
|
Страница — Nuxt SPA; контент лежит в __NUXT_DATA__ как JSON-строка
|
|
{"type":"doc","content":[...]}. Таблицы — узлы type=table.
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120"
|
|
OUT = os.environ.get("COMPAT_OUT", "/opt/work-hub/compat")
|
|
|
|
|
|
def fetch(url, dest="/tmp/page.html"):
|
|
subprocess.run(["curl", "-sL", "-m", "60", "-A", UA, "-o", dest, url],
|
|
capture_output=True, text=True, timeout=120)
|
|
return open(dest, encoding="utf-8", errors="ignore").read()
|
|
|
|
|
|
def prosemirror_docs(html):
|
|
"""Все ProseMirror-документы со страницы."""
|
|
m = re.search(r'<script[^>]*id="__NUXT_DATA__"[^>]*>(.*?)</script>', html, re.S)
|
|
if not m:
|
|
return []
|
|
try:
|
|
data = json.loads(m.group(1).strip())
|
|
except Exception:
|
|
return []
|
|
docs = []
|
|
|
|
def walk(o):
|
|
if isinstance(o, str):
|
|
s = o.strip()
|
|
if s.startswith('{"type":"doc"'):
|
|
try:
|
|
docs.append(json.loads(s))
|
|
except Exception:
|
|
pass
|
|
elif isinstance(o, list):
|
|
for v in o:
|
|
walk(v)
|
|
elif isinstance(o, dict):
|
|
for v in o.values():
|
|
walk(v)
|
|
walk(data)
|
|
return docs
|
|
|
|
|
|
def text_of(node):
|
|
if not isinstance(node, dict):
|
|
return ""
|
|
if node.get("type") == "text":
|
|
return node.get("text", "")
|
|
parts = [text_of(c) for c in node.get("content") or []]
|
|
return "".join(parts)
|
|
|
|
|
|
def tables_of(doc):
|
|
"""Возвращает список таблиц: [[cell, cell, ...], ...]."""
|
|
out = []
|
|
|
|
def walk(n):
|
|
if not isinstance(n, dict):
|
|
return
|
|
if n.get("type") == "table":
|
|
rows = []
|
|
for row in n.get("content") or []:
|
|
if row.get("type") != "tableRow":
|
|
continue
|
|
cells = []
|
|
for cell in row.get("content") or []:
|
|
cells.append(re.sub(r"\s+", " ", text_of(cell)).strip())
|
|
if cells:
|
|
rows.append(cells)
|
|
if rows:
|
|
out.append(rows)
|
|
return
|
|
for c in n.get("content") or []:
|
|
walk(c)
|
|
walk(doc)
|
|
return out
|
|
|
|
|
|
def main():
|
|
urls = sys.argv[1:]
|
|
if not urls:
|
|
print("использование: fetch_doc_tables.py <url> [<url>...]")
|
|
return 1
|
|
import yaml
|
|
os.makedirs(OUT, exist_ok=True)
|
|
for url in urls:
|
|
html = fetch(url)
|
|
docs = prosemirror_docs(html)
|
|
parts = [p for p in url.rstrip("/").split("/") if p]
|
|
doc_hash = parts[-1][:8] if parts else "unknown"
|
|
slug = (parts[-2] if len(parts) > 1 else "doc")
|
|
print("\n=== %s ===" % slug)
|
|
print(" документов ProseMirror: %d" % len(docs))
|
|
all_tables = []
|
|
for d in docs:
|
|
all_tables += tables_of(d)
|
|
print(" таблиц: %d" % len(all_tables))
|
|
for i, t in enumerate(all_tables[:3]):
|
|
print(" таблица %d: %d строк" % (i + 1, len(t)))
|
|
for row in t[:4]:
|
|
print(" ", " | ".join(c[:22] for c in row[:5]))
|
|
path = os.path.join(OUT, "raw", "%s-%s.yaml" % (re.sub(r"[^a-z0-9-]", "_", slug.lower()), doc_hash))
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
yaml.safe_dump({"source": url, "tables": all_tables}, f,
|
|
allow_unicode=True, sort_keys=False, width=200)
|
|
print(" сохранено:", path)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|