150 lines
5.3 KiB
Python
150 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
||
"""Сбор очереди документов вендоров ТОЛЬКО с официальных источников.
|
||
|
||
Каждый вендор: официальный домен + карта сайта. Скрипт забирает URL,
|
||
фильтрует по признакам документации (pdf, docs, datasheet, guide, manual,
|
||
support, product) и пишет отдельную очередь: research/queue-<вендор>.jsonl
|
||
|
||
Никаких сторонних сайтов, магазинов и форумов — только домены вендора.
|
||
"""
|
||
import json
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
PROXY = "http://192.168.1.161:7890"
|
||
OUT = "/opt/work-hub/research"
|
||
|
||
VENDORS = {
|
||
"vinteo": {
|
||
"proxy": False,
|
||
"hosts": ["vinteo.ru"],
|
||
"sitemaps": ["https://vinteo.ru/sitemap.xml"],
|
||
"follow_index": True,
|
||
},
|
||
"logitech": {
|
||
"proxy": True,
|
||
"hosts": ["logitech.com", "logi.com"],
|
||
"sitemaps": ["https://www.logitech.com/sitemap.xml"],
|
||
"follow_index": True,
|
||
},
|
||
"iva": {
|
||
"proxy": True,
|
||
"hosts": ["iva.ru", "iva-tech.ru"],
|
||
"sitemaps": ["https://iva.ru/sitemap.xml"],
|
||
"follow_index": True,
|
||
},
|
||
"poly": {
|
||
"proxy": True,
|
||
"hosts": ["poly.com", "polycom.com", "hp.com"],
|
||
"sitemaps": ["https://www.poly.com/sitemap.xml"],
|
||
"follow_index": True,
|
||
},
|
||
"hitrolink": {
|
||
"proxy": False,
|
||
"hosts": ["hitrolink.tech", "hitrolink.com.ru", "hitrolink.com"],
|
||
"sitemaps": ["https://hitrolink.tech/sitemap_index.xml",
|
||
"https://www.hitrolink.com.ru/sitemap.xml"],
|
||
"follow_index": True,
|
||
},
|
||
}
|
||
|
||
DOC_HINT = re.compile(
|
||
r"(?i)(\.pdf|/docs?/|/documentation|datasheet|data-sheet|guide|manual|"
|
||
r"/hc/|firmware|release-notes|admin-guide|user-guide|specification|"
|
||
r"room-designer|/support/|/products?/|/catalog/|/katalog/|hitrolink_)")
|
||
|
||
# мусор: ошибки, магазин, новости, вакансии, логин
|
||
SKIP_HINT = re.compile(
|
||
r"(?i)(404|500\.html|/shop/|/news/|/blog/|/press/|/career|/login|"
|
||
r"/cart|/checkout|/privacy|/cookie|/terms|/sitemap|\.xml$|\.php$)")
|
||
|
||
VIDEO_HINT = re.compile(
|
||
r"(?i)(rally|meetup|sight|roommate|tap|cc\d|eptz|conferenc|video|"
|
||
r"vinteo|trueconf|terminal|mcu|камер|микрофон|конференц)")
|
||
|
||
|
||
def fetch(url, proxy, timeout=25, attempts=3):
|
||
"""С повторами: logitech периодически отдаёт 000 через прокси."""
|
||
p = ("-x %s " % PROXY) if proxy else ""
|
||
for i in range(attempts):
|
||
r = subprocess.run(
|
||
"curl -sL -m %d %s-A 'Mozilla/5.0' -o /tmp/q.out -w '%%{http_code}' '%s'"
|
||
% (timeout, p, url), shell=True, capture_output=True, text=True,
|
||
timeout=timeout + 10)
|
||
code = (r.stdout or "").strip()[:3]
|
||
try:
|
||
body = open("/tmp/q.out", encoding="utf-8", errors="replace").read()
|
||
except Exception:
|
||
body = ""
|
||
if code == "200" and len(body) > 500:
|
||
return code, body
|
||
time.sleep(3)
|
||
return code, body
|
||
|
||
|
||
def locs(body):
|
||
return re.findall(r"<loc>\s*([^<\s]+)\s*</loc>", body)
|
||
|
||
|
||
def build(vendor, cfg, limit=2000):
|
||
seen, urls = set(), []
|
||
queue = list(cfg["sitemaps"])
|
||
done = set()
|
||
while queue and len(urls) < limit:
|
||
sm = queue.pop(0)
|
||
if sm in done:
|
||
continue
|
||
done.add(sm)
|
||
code, body = fetch(sm, cfg["proxy"])
|
||
if code != "200":
|
||
print(" %-10s карта %s → %s" % (vendor, sm[:60], code))
|
||
continue
|
||
found = locs(body)
|
||
# индекс карт: ссылки на другие sitemap-*.xml
|
||
subs = [u for u in found if "sitemap" in u.lower() and u.endswith(".xml")]
|
||
pages = [u for u in found if u not in subs]
|
||
if cfg.get("follow_index") and subs:
|
||
queue.extend(subs[:60])
|
||
for u in pages:
|
||
host = u.split("/")[2] if "//" in u else ""
|
||
if not any(h in host for h in cfg["hosts"]):
|
||
continue
|
||
if SKIP_HINT.search(u):
|
||
continue
|
||
# у logitech в карте много бытовухи — берём только ВКС-линейку
|
||
if vendor == "logitech" and not VIDEO_HINT.search(u):
|
||
continue
|
||
if not (DOC_HINT.search(u) or VIDEO_HINT.search(u)):
|
||
continue
|
||
if u in seen:
|
||
continue
|
||
seen.add(u)
|
||
urls.append(u)
|
||
if len(urls) >= limit:
|
||
break
|
||
print(" %-10s карт: %-3d адресов: %d" % (vendor, len(done), len(urls)))
|
||
return urls
|
||
|
||
|
||
def main():
|
||
only = sys.argv[1] if len(sys.argv) > 1 else None
|
||
total = 0
|
||
for vendor, cfg in VENDORS.items():
|
||
if only and vendor != only:
|
||
continue
|
||
urls = build(vendor, cfg)
|
||
path = "%s/queue-%s.jsonl" % (OUT, vendor)
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
for u in urls:
|
||
f.write(json.dumps({"url": u, "vendor": vendor, "product": "",
|
||
"slug": ""}, ensure_ascii=False) + "\n")
|
||
print(" записано: %s" % path)
|
||
total += len(urls)
|
||
print("всего новых адресов:", total)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|