Files
buryan-infra/ops/work/tools/fetch_doc_tables.py
T

119 lines
3.7 KiB
Python

#!/usr/bin/env python3
"""Извлечение таблиц совместимости из ProseMirror-JSON страниц support.yealink.com.
Страница — Nuxt SPA; контент лежит в __NUXT_DATA__ как JSON-строка
{"type":"doc","content":[...]}. Таблицы — узлы type=table.
"""
import json
import os
import re
import subprocess
import sys
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120"
OUT = os.environ.get("COMPAT_OUT", "/opt/work-hub/compat")
def fetch(url, dest="/tmp/page.html"):
subprocess.run(["curl", "-sL", "-m", "60", "-A", UA, "-o", dest, url],
capture_output=True, text=True, timeout=120)
return open(dest, encoding="utf-8", errors="ignore").read()
def prosemirror_docs(html):
"""Все ProseMirror-документы со страницы."""
m = re.search(r'<script[^>]*id="__NUXT_DATA__"[^>]*>(.*?)</script>', html, re.S)
if not m:
return []
try:
data = json.loads(m.group(1).strip())
except Exception:
return []
docs = []
def walk(o):
if isinstance(o, str):
s = o.strip()
if s.startswith('{"type":"doc"'):
try:
docs.append(json.loads(s))
except Exception:
pass
elif isinstance(o, list):
for v in o:
walk(v)
elif isinstance(o, dict):
for v in o.values():
walk(v)
walk(data)
return docs
def text_of(node):
if not isinstance(node, dict):
return ""
if node.get("type") == "text":
return node.get("text", "")
parts = [text_of(c) for c in node.get("content") or []]
return "".join(parts)
def tables_of(doc):
"""Возвращает список таблиц: [[cell, cell, ...], ...]."""
out = []
def walk(n):
if not isinstance(n, dict):
return
if n.get("type") == "table":
rows = []
for row in n.get("content") or []:
if row.get("type") != "tableRow":
continue
cells = []
for cell in row.get("content") or []:
cells.append(re.sub(r"\s+", " ", text_of(cell)).strip())
if cells:
rows.append(cells)
if rows:
out.append(rows)
return
for c in n.get("content") or []:
walk(c)
walk(doc)
return out
def main():
urls = sys.argv[1:]
if not urls:
print("использование: fetch_doc_tables.py <url> [<url>...]")
return 1
import yaml
os.makedirs(OUT, exist_ok=True)
for url in urls:
html = fetch(url)
docs = prosemirror_docs(html)
slug = url.rstrip("/").split("/")[-2] or "doc"
print("\n=== %s ===" % slug)
print(" документов ProseMirror: %d" % len(docs))
all_tables = []
for d in docs:
all_tables += tables_of(d)
print(" таблиц: %d" % len(all_tables))
for i, t in enumerate(all_tables[:3]):
print(" таблица %d: %d строк" % (i + 1, len(t)))
for row in t[:4]:
print(" ", " | ".join(c[:22] for c in row[:5]))
path = os.path.join(OUT, "raw", "%s.yaml" % re.sub(r"[^a-z0-9-]", "_", slug.lower()))
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
yaml.safe_dump({"source": url, "tables": all_tables}, f,
allow_unicode=True, sort_keys=False, width=200)
print(" сохранено:", path)
return 0
if __name__ == "__main__":
sys.exit(main())