#!/usr/bin/env python3 """Собирает все страницы списков совместимости Yealink из sitemap.""" import re import subprocess import sys UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120" BASE = "https://support.yealink.com" def get(url): r = subprocess.run(["curl", "-sL", "-m", "40", "-A", UA, url], capture_output=True, text=True, timeout=90) return r.stdout or "" def locs(xml): return re.findall(r"([^<]+)", xml) def main(): idx = get(BASE + "/sitemap.xml") subs = locs(idx) print("подкарт в индексе:", len(subs)) urls = set() for s in subs: xml = get(s) found = locs(xml) for u in found: if "/docs/" in u: urls.add(u.split("?")[0]) print(" %-52s %d URL" % (s.replace(BASE, ""), len(found))) print("\nвсего /docs/ URL:", len(urls)) compat = sorted(u for u in urls if re.search(r"compatib|compatible", u, re.I)) print("со 'compatib' в адресе:", len(compat)) # выделяем английские версии (без локали в пути) en = [u for u in compat if re.search(r"/docs/", u) and not re.search(r"/(de-de|fr-fr|es-es|it-it|zh-cn|ru-ru|ja-jp|ko-kr|pt-br)/", u)] print("английских:", len(en)) with open("/opt/work-hub/compat-src/urls.txt", "w", encoding="utf-8") as f: f.write("\n".join(en) + "\n") print("\nпервые 25:") for u in en[:25]: print(" ", u.replace(BASE, "")) print("\nсохранено: /opt/work-hub/compat-src/urls.txt") return 0 if __name__ == "__main__": sys.exit(main())