Files
buryan-infra/ops/work/tools/collect_urls.py

53 lines
1.7 KiB
Python

#!/usr/bin/env python3
"""Собирает все страницы списков совместимости Yealink из sitemap."""
import re
import subprocess
import sys
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120"
BASE = "https://support.yealink.com"
def get(url):
r = subprocess.run(["curl", "-sL", "-m", "40", "-A", UA, url],
capture_output=True, text=True, timeout=90)
return r.stdout or ""
def locs(xml):
return re.findall(r"<loc>([^<]+)</loc>", xml)
def main():
idx = get(BASE + "/sitemap.xml")
subs = locs(idx)
print("подкарт в индексе:", len(subs))
urls = set()
for s in subs:
xml = get(s)
found = locs(xml)
for u in found:
if "/docs/" in u:
urls.add(u.split("?")[0])
print(" %-52s %d URL" % (s.replace(BASE, ""), len(found)))
print("\nвсего /docs/ URL:", len(urls))
compat = sorted(u for u in urls if re.search(r"compatib|compatible", u, re.I))
print("со 'compatib' в адресе:", len(compat))
# выделяем английские версии (без локали в пути)
en = [u for u in compat if re.search(r"/docs/", u) and not re.search(r"/(de-de|fr-fr|es-es|it-it|zh-cn|ru-ru|ja-jp|ko-kr|pt-br)/", u)]
print("английских:", len(en))
with open("/opt/work-hub/compat-src/urls.txt", "w", encoding="utf-8") as f:
f.write("\n".join(en) + "\n")
print("\nпервые 25:")
for u in en[:25]:
print(" ", u.replace(BASE, ""))
print("\nсохранено: /opt/work-hub/compat-src/urls.txt")
return 0
if __name__ == "__main__":
sys.exit(main())