96 lines
3.6 KiB
Python
96 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Собирает очередь документов Yealink Support из sitemap.
|
|
|
|
1) читает sitemapindex
|
|
2) качает вложенные sitemap
|
|
3) вытаскивает URL вида /docs/<product>/<slug>/<hash>
|
|
4) фильтрует по интересным продуктам
|
|
5) пишет очередь queue.jsonl
|
|
"""
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
OUT = "/opt/work-hub/research"
|
|
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120"
|
|
|
|
# продукты, которые нам нужны (ВКС + аудио + инфраструктура + телефония)
|
|
PRODUCTS = set("""
|
|
a10 a20 a25 a30 a40 a50 a40-zoom a50-zoom meetingbar meetingboard meetingeye
|
|
mcore mcore4 mc-pro mvc pvb pvt roomcast
|
|
uvc20 uvc30 uvc34 uvc40 uvc40-e2 uvc42 uvc44 uvc84 uvc85 uvc86 uvc88
|
|
smartvision sv40 sv60 sv80
|
|
vcm34 vcm35 vcm36 vcm38 cm20 cm50 cs10 cs10-d mspeaker mspeaker-pro
|
|
cp50 cp920 cp960 cp965 cpe40 cpe90 cpe95 pa20 ap08
|
|
vch50 vch51 vch51-pro wpp20 wpp30 rch40 rch40-e2 avbridge avhub
|
|
ctp18 ctp25 mtouch vcr11 vcr20
|
|
meetingdisplay etv65 etv86
|
|
t31 t33 t34 t43 t46 t48 t53 t54 t57 t58 t73 t74 t77 t78
|
|
w70 w73 w76 w78 w80 w90 wh62 wh63 wh66 wh67 yhs34 uh34 bh70
|
|
yealink-room-designer yealink-room-device-calculator
|
|
""".split())
|
|
|
|
|
|
def fetch(url):
|
|
return subprocess.run(["curl", "-sL", "-m", "60", "-A", UA, url],
|
|
capture_output=True, text=True, timeout=120).stdout
|
|
|
|
|
|
def main():
|
|
os.makedirs(OUT, exist_ok=True)
|
|
idx = fetch("https://support.yealink.com/sitemap.xml")
|
|
subs = re.findall(r"<loc>([^<]+)</loc>", idx)
|
|
print("вложенных sitemap:", len(subs))
|
|
urls = set()
|
|
for i, s in enumerate(subs, 1):
|
|
if not s.endswith(".xml"):
|
|
continue
|
|
body = fetch(s)
|
|
found = re.findall(r"<loc>(https://support\.yealink\.com/docs/[^<]+)</loc>", body)
|
|
if found:
|
|
print(" %-45s +%d" % (s.rsplit('/', 1)[-1], len(found)))
|
|
urls.update(found)
|
|
|
|
print("\nвсего doc-URL:", len(urls))
|
|
# продукт = первый сегмент после /docs/
|
|
by_product = {}
|
|
for u in urls:
|
|
m = re.match(r"https://support\.yealink\.com/docs/([^/]+)/", u)
|
|
if not m:
|
|
continue
|
|
by_product.setdefault(m.group(1), []).append(u)
|
|
print("продуктов в sitemap:", len(by_product))
|
|
|
|
chosen = []
|
|
for p, lst in by_product.items():
|
|
if p in PRODUCTS:
|
|
chosen.extend(lst)
|
|
print("URL по нашим продуктам:", len(chosen))
|
|
|
|
if not chosen: # fallback: взять всё, что похоже на ВКС
|
|
chosen = [u for u in urls if any(k in u for k in
|
|
("meetingbar", "meetingeye", "meetingboard", "uvc", "vcm", "mvc"))]
|
|
print("fallback-выборка:", len(chosen))
|
|
|
|
path = os.path.join(OUT, "queue.jsonl")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
for u in sorted(chosen):
|
|
m = re.match(r"https://support\.yealink\.com/docs/([^/]+)/([^/]+)/", u)
|
|
f.write(json.dumps({"url": u,
|
|
"product": m.group(1) if m else "",
|
|
"slug": m.group(2) if m else "",
|
|
"status": "pending"}, ensure_ascii=False) + "\n")
|
|
print("\nочередь:", path, "| записей:", len(chosen))
|
|
# сводка по продуктам
|
|
top = sorted(((p, len(v)) for p, v in by_product.items() if p in PRODUCTS),
|
|
key=lambda x: -x[1])[:15]
|
|
for p, n in top:
|
|
print(" %-22s %d док." % (p, n))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|