#!/usr/bin/env python3 """Сбор очереди документов вендоров ТОЛЬКО с официальных источников. Каждый вендор: официальный домен + карта сайта. Скрипт забирает URL, фильтрует по признакам документации (pdf, docs, datasheet, guide, manual, support, product) и пишет отдельную очередь: research/queue-<вендор>.jsonl Никаких сторонних сайтов, магазинов и форумов — только домены вендора. """ import json import re import subprocess import sys import time PROXY = "http://192.168.1.161:7890" OUT = "/opt/work-hub/research" VENDORS = { "vinteo": { "proxy": False, "hosts": ["vinteo.ru"], "sitemaps": ["https://vinteo.ru/sitemap.xml"], "follow_index": True, }, "logitech": { "proxy": True, "hosts": ["logitech.com", "logi.com"], "sitemaps": ["https://www.logitech.com/sitemap.xml"], "follow_index": True, }, "iva": { "proxy": True, "hosts": ["iva.ru", "iva-tech.ru"], "sitemaps": ["https://iva.ru/sitemap.xml"], "follow_index": True, }, "poly": { "proxy": True, "hosts": ["poly.com", "polycom.com", "hp.com"], "sitemaps": ["https://www.poly.com/sitemap.xml"], "follow_index": True, }, } DOC_HINT = re.compile( r"(?i)(\.pdf|/docs?/|/documentation|datasheet|data-sheet|guide|manual|" r"/hc/|firmware|release-notes|admin-guide|user-guide|specification|" r"room-designer|/support/|/products?/|/catalog/|/katalog/)") # мусор: ошибки, магазин, новости, вакансии, логин SKIP_HINT = re.compile( r"(?i)(404|500\.html|/shop/|/news/|/blog/|/press/|/career|/login|" r"/cart|/checkout|/privacy|/cookie|/terms|/sitemap|\.xml$|\.php$)") VIDEO_HINT = re.compile( r"(?i)(rally|meetup|sight|roommate|tap|cc\d|eptz|conferenc|video|" r"vinteo|trueconf|terminal|mcu|камер|микрофон|конференц)") def fetch(url, proxy, timeout=25, attempts=3): """С повторами: logitech периодически отдаёт 000 через прокси.""" p = ("-x %s " % PROXY) if proxy else "" for i in range(attempts): r = subprocess.run( "curl -sL -m %d %s-A 'Mozilla/5.0' -o /tmp/q.out -w '%%{http_code}' '%s'" % (timeout, p, url), shell=True, capture_output=True, text=True, timeout=timeout + 10) code = (r.stdout or "").strip()[:3] try: body = open("/tmp/q.out", encoding="utf-8", errors="replace").read() except Exception: body = "" if code == "200" and len(body) > 500: return code, body time.sleep(3) return code, body def locs(body): return re.findall(r"\s*([^<\s]+)\s*", body) def build(vendor, cfg, limit=2000): seen, urls = set(), [] queue = list(cfg["sitemaps"]) done = set() while queue and len(urls) < limit: sm = queue.pop(0) if sm in done: continue done.add(sm) code, body = fetch(sm, cfg["proxy"]) if code != "200": print(" %-10s карта %s → %s" % (vendor, sm[:60], code)) continue found = locs(body) # индекс карт: ссылки на другие sitemap-*.xml subs = [u for u in found if "sitemap" in u.lower() and u.endswith(".xml")] pages = [u for u in found if u not in subs] if cfg.get("follow_index") and subs: queue.extend(subs[:60]) for u in pages: host = u.split("/")[2] if "//" in u else "" if not any(h in host for h in cfg["hosts"]): continue if SKIP_HINT.search(u): continue # у logitech в карте много бытовухи — берём только ВКС-линейку if vendor == "logitech" and not VIDEO_HINT.search(u): continue if not (DOC_HINT.search(u) or VIDEO_HINT.search(u)): continue if u in seen: continue seen.add(u) urls.append(u) if len(urls) >= limit: break print(" %-10s карт: %-3d адресов: %d" % (vendor, len(done), len(urls))) return urls def main(): only = sys.argv[1] if len(sys.argv) > 1 else None total = 0 for vendor, cfg in VENDORS.items(): if only and vendor != only: continue urls = build(vendor, cfg) path = "%s/queue-%s.jsonl" % (OUT, vendor) with open(path, "w", encoding="utf-8") as f: for u in urls: f.write(json.dumps({"url": u, "vendor": vendor, "product": "", "slug": ""}, ensure_ascii=False) + "\n") print(" записано: %s" % path) total += len(urls) print("всего новых адресов:", total) if __name__ == "__main__": main()