#!/usr/bin/env python3 """Единый диспетчер LLM-сбора: один поток, общая очередь, модели грузятся по нужде. Зачем: раньше каждый вендор — отдельный сервис, и они дрались за Ollama (7B стоял в очереди за 3B, а RAG падал по таймауту). Мини-ПК работает 24/7, значит достаточно одной очереди: берём по одному документу, отдаём модели, пишем результат. Ollama сама загружает/выгружает нужную модель. Конфиг: ops/work/research/jobs.yaml — список источников с весом и моделью. """ import json import os import sys import time import yaml sys.path.insert(0, "/opt/work-hub/research") OUT = "/opt/work-hub/research" JOBS = "/opt/work-hub/research/jobs.yaml" LOG = "/var/log/llm-dispatch.log" LOCK = "/var/lib/llm-dispatcher.lock" os.environ.setdefault("OLLAMA_URL", "http://192.168.1.229:11434") def log(msg): line = "[%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), msg) try: with open(LOG, "a", encoding="utf-8") as f: f.write(line + "\n") except Exception: pass print(line, flush=True) def one_instance(): """Не даём запуститься двум диспетчерам.""" try: import fcntl fh = open(LOCK, "w") fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) return fh except Exception: return None def progress_path(job): tag = ("-" + job["tag"]) if job.get("tag") else "" return os.path.join(OUT, "progress-%s%s-0.json" % (job["vendor"], tag)) def results_path(job): tag = ("-" + job["tag"]) if job.get("tag") else "" return os.path.join(OUT, "results-%s%s-0.jsonl" % (job["vendor"], tag)) def load_progress(job): try: return json.load(open(progress_path(job), encoding="utf-8")) except Exception: return {} def save_progress(job, done): json.dump(done, open(progress_path(job), "w", encoding="utf-8")) def pending(job): q = os.path.join(OUT, job["queue"]) try: items = [json.loads(l) for l in open(q, encoding="utf-8") if l.strip()] except Exception: return [] done = load_progress(job) return [i for i in items if done.get(i["url"]) not in ("ok", "empty", "skip")] def process(job, item, L): """Обрабатывает один документ выбранной моделью.""" L.VENDOR = job["vendor"] L.MODEL = job.get("model", "qwen2.5:3b") L.PROXY = job.get("proxy", "") L.MAX_PREDICT = int(job.get("max_predict", 450)) window = int(job.get("window", 2600)) url = item["url"] done = load_progress(job) txt = L.fetch(url) if len(txt) < 200: done[url] = "empty" save_progress(job, done) return "empty", 0 low = txt.lower() if job["vendor"] == "yealink": hits = sum(1 for t in L.MODEL_TOKENS if t in low) gate = hits > 0 and any(w in low for w in L.SPEC_WORDS) else: hits = sum(1 for t in L.VENDOR_TOKENS.get(job["vendor"], []) if t in low) gate = hits > 0 if not gate: done[url] = "skip" save_progress(job, done) return "skip", 0 chunk = L.pick_excerpt(txt, window) try: res = L.ask(chunk) except Exception as e: log(" ошибка LLM (%s): %s" % (job["vendor"], str(e)[:100])) return "error", 0 with open(results_path(job), "a", encoding="utf-8") as f: f.write(json.dumps({"url": item.get("source_url") or url, "vendor": job["vendor"], "model": L.MODEL, "product": item.get("product"), "slug": item.get("slug"), "chunks": 1, "data": [res] if res else []}, ensure_ascii=False) + "\n") done[url] = "ok" save_progress(job, done) nfacts = len((res or {}).get("facts") or []) ndeps = len((res or {}).get("dependencies") or []) return "ok", nfacts + ndeps def main(): fh = one_instance() if fh is None: log("диспетчер уже запущен — выходим") return import llm_research as L log("=== диспетчер старт ===") empty_rounds = 0 while True: cfg = yaml.safe_load(open(JOBS, encoding="utf-8")) or {} jobs = cfg.get("jobs") or [] did_work = False for job in jobs: if job.get("enabled") is False: continue weight = int(job.get("weight", 1)) for _ in range(weight): items = pending(job) if not items: continue item = items[0] try: status, n = process(job, item, L) except Exception as e: log("сбой обработки %s: %s" % (job["vendor"], str(e)[:120])) continue did_work = True log("%-9s %-5s +%-2d %s" % (job["vendor"], status, n, item["url"][-70:])) time.sleep(1) if not did_work: empty_rounds += 1 if empty_rounds == 1: log("очереди пусты — ждём") time.sleep(300) else: empty_rounds = 0 if __name__ == "__main__": main()