Files

150 lines
5.3 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Сбор очереди документов вендоров ТОЛЬКО с официальных источников.
Каждый вендор: официальный домен + карта сайта. Скрипт забирает URL,
фильтрует по признакам документации (pdf, docs, datasheet, guide, manual,
support, product) и пишет отдельную очередь: research/queue-<вендор>.jsonl
Никаких сторонних сайтов, магазинов и форумов — только домены вендора.
"""
import json
import re
import subprocess
import sys
import time
PROXY = "http://192.168.1.161:7890"
OUT = "/opt/work-hub/research"
VENDORS = {
"vinteo": {
"proxy": False,
"hosts": ["vinteo.ru"],
"sitemaps": ["https://vinteo.ru/sitemap.xml"],
"follow_index": True,
},
"logitech": {
"proxy": True,
"hosts": ["logitech.com", "logi.com"],
"sitemaps": ["https://www.logitech.com/sitemap.xml"],
"follow_index": True,
},
"iva": {
"proxy": True,
"hosts": ["iva.ru", "iva-tech.ru"],
"sitemaps": ["https://iva.ru/sitemap.xml"],
"follow_index": True,
},
"poly": {
"proxy": True,
"hosts": ["poly.com", "polycom.com", "hp.com"],
"sitemaps": ["https://www.poly.com/sitemap.xml"],
"follow_index": True,
},
"hitrolink": {
"proxy": False,
"hosts": ["hitrolink.tech", "hitrolink.com.ru", "hitrolink.com"],
"sitemaps": ["https://hitrolink.tech/sitemap_index.xml",
"https://www.hitrolink.com.ru/sitemap.xml"],
"follow_index": True,
},
}
DOC_HINT = re.compile(
r"(?i)(\.pdf|/docs?/|/documentation|datasheet|data-sheet|guide|manual|"
r"/hc/|firmware|release-notes|admin-guide|user-guide|specification|"
r"room-designer|/support/|/products?/|/catalog/|/katalog/|hitrolink_)")
# мусор: ошибки, магазин, новости, вакансии, логин
SKIP_HINT = re.compile(
r"(?i)(404|500\.html|/shop/|/news/|/blog/|/press/|/career|/login|"
r"/cart|/checkout|/privacy|/cookie|/terms|/sitemap|\.xml$|\.php$)")
VIDEO_HINT = re.compile(
r"(?i)(rally|meetup|sight|roommate|tap|cc\d|eptz|conferenc|video|"
r"vinteo|trueconf|terminal|mcu|камер|микрофон|конференц)")
def fetch(url, proxy, timeout=25, attempts=3):
"""С повторами: logitech периодически отдаёт 000 через прокси."""
p = ("-x %s " % PROXY) if proxy else ""
for i in range(attempts):
r = subprocess.run(
"curl -sL -m %d %s-A 'Mozilla/5.0' -o /tmp/q.out -w '%%{http_code}' '%s'"
% (timeout, p, url), shell=True, capture_output=True, text=True,
timeout=timeout + 10)
code = (r.stdout or "").strip()[:3]
try:
body = open("/tmp/q.out", encoding="utf-8", errors="replace").read()
except Exception:
body = ""
if code == "200" and len(body) > 500:
return code, body
time.sleep(3)
return code, body
def locs(body):
return re.findall(r"<loc>\s*([^<\s]+)\s*</loc>", body)
def build(vendor, cfg, limit=2000):
seen, urls = set(), []
queue = list(cfg["sitemaps"])
done = set()
while queue and len(urls) < limit:
sm = queue.pop(0)
if sm in done:
continue
done.add(sm)
code, body = fetch(sm, cfg["proxy"])
if code != "200":
print(" %-10s карта %s → %s" % (vendor, sm[:60], code))
continue
found = locs(body)
# индекс карт: ссылки на другие sitemap-*.xml
subs = [u for u in found if "sitemap" in u.lower() and u.endswith(".xml")]
pages = [u for u in found if u not in subs]
if cfg.get("follow_index") and subs:
queue.extend(subs[:60])
for u in pages:
host = u.split("/")[2] if "//" in u else ""
if not any(h in host for h in cfg["hosts"]):
continue
if SKIP_HINT.search(u):
continue
# у logitech в карте много бытовухи — берём только ВКС-линейку
if vendor == "logitech" and not VIDEO_HINT.search(u):
continue
if not (DOC_HINT.search(u) or VIDEO_HINT.search(u)):
continue
if u in seen:
continue
seen.add(u)
urls.append(u)
if len(urls) >= limit:
break
print(" %-10s карт: %-3d адресов: %d" % (vendor, len(done), len(urls)))
return urls
def main():
only = sys.argv[1] if len(sys.argv) > 1 else None
total = 0
for vendor, cfg in VENDORS.items():
if only and vendor != only:
continue
urls = build(vendor, cfg)
path = "%s/queue-%s.jsonl" % (OUT, vendor)
with open(path, "w", encoding="utf-8") as f:
for u in urls:
f.write(json.dumps({"url": u, "vendor": vendor, "product": "",
"slug": ""}, ensure_ascii=False) + "\n")
print(" записано: %s" % path)
total += len(urls)
print("всего новых адресов:", total)
if __name__ == "__main__":
main()