Files

96 lines
3.6 KiB
Python

#!/usr/bin/env python3
"""Собирает очередь документов Yealink Support из sitemap.
1) читает sitemapindex
2) качает вложенные sitemap
3) вытаскивает URL вида /docs/<product>/<slug>/<hash>
4) фильтрует по интересным продуктам
5) пишет очередь queue.jsonl
"""
import json
import os
import re
import subprocess
import sys
OUT = "/opt/work-hub/research"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120"
# продукты, которые нам нужны (ВКС + аудио + инфраструктура + телефония)
PRODUCTS = set("""
a10 a20 a25 a30 a40 a50 a40-zoom a50-zoom meetingbar meetingboard meetingeye
mcore mcore4 mc-pro mvc pvb pvt roomcast
uvc20 uvc30 uvc34 uvc40 uvc40-e2 uvc42 uvc44 uvc84 uvc85 uvc86 uvc88
smartvision sv40 sv60 sv80
vcm34 vcm35 vcm36 vcm38 cm20 cm50 cs10 cs10-d mspeaker mspeaker-pro
cp50 cp920 cp960 cp965 cpe40 cpe90 cpe95 pa20 ap08
vch50 vch51 vch51-pro wpp20 wpp30 rch40 rch40-e2 avbridge avhub
ctp18 ctp25 mtouch vcr11 vcr20
meetingdisplay etv65 etv86
t31 t33 t34 t43 t46 t48 t53 t54 t57 t58 t73 t74 t77 t78
w70 w73 w76 w78 w80 w90 wh62 wh63 wh66 wh67 yhs34 uh34 bh70
yealink-room-designer yealink-room-device-calculator
""".split())
def fetch(url):
return subprocess.run(["curl", "-sL", "-m", "60", "-A", UA, url],
capture_output=True, text=True, timeout=120).stdout
def main():
os.makedirs(OUT, exist_ok=True)
idx = fetch("https://support.yealink.com/sitemap.xml")
subs = re.findall(r"<loc>([^<]+)</loc>", idx)
print("вложенных sitemap:", len(subs))
urls = set()
for i, s in enumerate(subs, 1):
if not s.endswith(".xml"):
continue
body = fetch(s)
found = re.findall(r"<loc>(https://support\.yealink\.com/docs/[^<]+)</loc>", body)
if found:
print(" %-45s +%d" % (s.rsplit('/', 1)[-1], len(found)))
urls.update(found)
print("\nвсего doc-URL:", len(urls))
# продукт = первый сегмент после /docs/
by_product = {}
for u in urls:
m = re.match(r"https://support\.yealink\.com/docs/([^/]+)/", u)
if not m:
continue
by_product.setdefault(m.group(1), []).append(u)
print("продуктов в sitemap:", len(by_product))
chosen = []
for p, lst in by_product.items():
if p in PRODUCTS:
chosen.extend(lst)
print("URL по нашим продуктам:", len(chosen))
if not chosen: # fallback: взять всё, что похоже на ВКС
chosen = [u for u in urls if any(k in u for k in
("meetingbar", "meetingeye", "meetingboard", "uvc", "vcm", "mvc"))]
print("fallback-выборка:", len(chosen))
path = os.path.join(OUT, "queue.jsonl")
with open(path, "w", encoding="utf-8") as f:
for u in sorted(chosen):
m = re.match(r"https://support\.yealink\.com/docs/([^/]+)/([^/]+)/", u)
f.write(json.dumps({"url": u,
"product": m.group(1) if m else "",
"slug": m.group(2) if m else "",
"status": "pending"}, ensure_ascii=False) + "\n")
print("\nочередь:", path, "| записей:", len(chosen))
# сводка по продуктам
top = sorted(((p, len(v)) for p, v in by_product.items() if p in PRODUCTS),
key=lambda x: -x[1])[:15]
for p, n in top:
print(" %-22s %d док." % (p, n))
return 0
if __name__ == "__main__":
sys.exit(main())