Files
buryan-infra/ops/work/tools/collect_docs.py
T

88 lines
3.1 KiB
Python

#!/usr/bin/env python3
"""Собирает документы «Compatibility» из категорий support.yealink.com."""
import json
import re
import subprocess
import sys
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120"
BASE = "https://support.yealink.com"
CATEGORIES = [
("video-conferencing-product", "01985e29c13c770bbc41c9e7fed61696"),
("pro-av-solution", "019d9b4d60207393a60b9f3c380e3693"),
("personal-usb-endpoint", "01985eceaedd75a39a3cf2b0b33f2b58"),
("phone", "01985ebe3602732a994346b12966d2e3"),
("management-platform", "01985e27ee157da1a0ed1aaaf367ca0a"),
]
def get(url, timeout=60):
r = subprocess.run(["curl", "-sL", "-m", str(timeout), "-A", UA, url],
capture_output=True, text=True, timeout=timeout + 30)
return r.stdout or ""
def nuxt_data(html):
m = re.search(r'<script[^>]*id="__NUXT_DATA__"[^>]*>(.*?)</script>', html, re.S)
if not m:
return None
try:
return json.loads(m.group(1).strip())
except Exception:
return None
def find_docs(obj, out, path=""):
"""Ищем объекты, похожие на документ: есть title/name и 32-символьный хеш."""
if isinstance(obj, dict):
vals = {k: v for k, v in obj.items() if isinstance(v, (str, int))}
title = None
for k in ("title", "name", "docTitle", "articleTitle"):
if k in vals and isinstance(vals[k], str):
title = vals[k]
break
hid = None
for v in vals.values():
if isinstance(v, str) and re.fullmatch(r"[0-9a-f]{32}", v):
hid = v
break
if title and hid:
out.append({"id": hid, "title": title, "path": path})
for k, v in obj.items():
find_docs(v, out, "%s.%s" % (path, k))
elif isinstance(obj, list):
for i, v in enumerate(obj):
find_docs(v, out, "%s[%d]" % (path, i))
def main():
all_docs = []
for name, cid in CATEGORIES:
url = "%s/categories/%s/%s" % (BASE, name, cid)
html = get(url)
data = nuxt_data(html)
if data is None:
print(" %-28s NUXT не найден (%d Б)" % (name, len(html)))
continue
docs = []
find_docs(data, docs)
uniq = {d["id"]: d for d in docs}
compat = [d for d in uniq.values() if re.search(r"compatib", d["title"], re.I)]
print(" %-28s документов: %3d | «compatibility»: %2d" % (name, len(uniq), len(compat)))
for d in compat[:6]:
print(" %s" % d["title"][:70])
all_docs += compat
uniq = {d["id"]: d for d in all_docs}
with open("/opt/work-hub/compat-src/compat_docs.json", "w", encoding="utf-8") as f:
json.dump(sorted(uniq.values(), key=lambda x: x["title"]), f,
ensure_ascii=False, indent=1)
print("\nвсего уникальных «compatibility»-документов: %d" % len(uniq))
for d in sorted(uniq.values(), key=lambda x: x["title"])[:15]:
print(" %-70s %s" % (d["title"][:70], d["id"][:8]))
if __name__ == "__main__":
sys.exit(main())