169 lines
5.4 KiB
Python
169 lines
5.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Единый диспетчер LLM-сбора: один поток, общая очередь, модели грузятся по нужде.
|
|
|
|
Зачем: раньше каждый вендор — отдельный сервис, и они дрались за Ollama
|
|
(7B стоял в очереди за 3B, а RAG падал по таймауту). Мини-ПК работает 24/7,
|
|
значит достаточно одной очереди: берём по одному документу, отдаём модели,
|
|
пишем результат. Ollama сама загружает/выгружает нужную модель.
|
|
|
|
Конфиг: ops/work/research/jobs.yaml — список источников с весом и моделью.
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
import yaml
|
|
|
|
sys.path.insert(0, "/opt/work-hub/research")
|
|
OUT = "/opt/work-hub/research"
|
|
JOBS = "/opt/work-hub/research/jobs.yaml"
|
|
LOG = "/var/log/llm-dispatch.log"
|
|
LOCK = "/var/lib/llm-dispatcher.lock"
|
|
|
|
os.environ.setdefault("OLLAMA_URL", "http://192.168.1.229:11434")
|
|
|
|
|
|
def log(msg):
|
|
line = "[%s] %s" % (time.strftime("%Y-%m-%d %H:%M:%S"), msg)
|
|
try:
|
|
with open(LOG, "a", encoding="utf-8") as f:
|
|
f.write(line + "\n")
|
|
except Exception:
|
|
pass
|
|
print(line, flush=True)
|
|
|
|
|
|
def one_instance():
|
|
"""Не даём запуститься двум диспетчерам."""
|
|
try:
|
|
import fcntl
|
|
fh = open(LOCK, "w")
|
|
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
return fh
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def progress_path(job):
|
|
tag = ("-" + job["tag"]) if job.get("tag") else ""
|
|
return os.path.join(OUT, "progress-%s%s-0.json" % (job["vendor"], tag))
|
|
|
|
|
|
def results_path(job):
|
|
tag = ("-" + job["tag"]) if job.get("tag") else ""
|
|
return os.path.join(OUT, "results-%s%s-0.jsonl" % (job["vendor"], tag))
|
|
|
|
|
|
def load_progress(job):
|
|
try:
|
|
return json.load(open(progress_path(job), encoding="utf-8"))
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def save_progress(job, done):
|
|
json.dump(done, open(progress_path(job), "w", encoding="utf-8"))
|
|
|
|
|
|
def pending(job):
|
|
q = os.path.join(OUT, job["queue"])
|
|
try:
|
|
items = [json.loads(l) for l in open(q, encoding="utf-8") if l.strip()]
|
|
except Exception:
|
|
return []
|
|
done = load_progress(job)
|
|
return [i for i in items if done.get(i["url"]) not in ("ok", "empty", "skip")]
|
|
|
|
|
|
def process(job, item, L):
|
|
"""Обрабатывает один документ выбранной моделью."""
|
|
L.VENDOR = job["vendor"]
|
|
L.MODEL = job.get("model", "qwen2.5:3b")
|
|
L.PROXY = job.get("proxy", "")
|
|
L.MAX_PREDICT = int(job.get("max_predict", 450))
|
|
window = int(job.get("window", 2600))
|
|
|
|
url = item["url"]
|
|
done = load_progress(job)
|
|
txt = L.fetch(url)
|
|
if len(txt) < 200:
|
|
done[url] = "empty"
|
|
save_progress(job, done)
|
|
return "empty", 0
|
|
|
|
low = txt.lower()
|
|
if job["vendor"] == "yealink":
|
|
hits = sum(1 for t in L.MODEL_TOKENS if t in low)
|
|
gate = hits > 0 and any(w in low for w in L.SPEC_WORDS)
|
|
else:
|
|
hits = sum(1 for t in L.VENDOR_TOKENS.get(job["vendor"], []) if t in low)
|
|
gate = hits > 0
|
|
if not gate:
|
|
done[url] = "skip"
|
|
save_progress(job, done)
|
|
return "skip", 0
|
|
|
|
chunk = L.pick_excerpt(txt, window)
|
|
try:
|
|
res = L.ask(chunk)
|
|
except Exception as e:
|
|
log(" ошибка LLM (%s): %s" % (job["vendor"], str(e)[:100]))
|
|
return "error", 0
|
|
|
|
with open(results_path(job), "a", encoding="utf-8") as f:
|
|
f.write(json.dumps({"url": item.get("source_url") or url,
|
|
"vendor": job["vendor"], "model": L.MODEL,
|
|
"product": item.get("product"), "slug": item.get("slug"),
|
|
"chunks": 1, "data": [res] if res else []},
|
|
ensure_ascii=False) + "\n")
|
|
done[url] = "ok"
|
|
save_progress(job, done)
|
|
nfacts = len((res or {}).get("facts") or [])
|
|
ndeps = len((res or {}).get("dependencies") or [])
|
|
return "ok", nfacts + ndeps
|
|
|
|
|
|
def main():
|
|
fh = one_instance()
|
|
if fh is None:
|
|
log("диспетчер уже запущен — выходим")
|
|
return
|
|
|
|
import llm_research as L
|
|
|
|
log("=== диспетчер старт ===")
|
|
empty_rounds = 0
|
|
while True:
|
|
cfg = yaml.safe_load(open(JOBS, encoding="utf-8")) or {}
|
|
jobs = cfg.get("jobs") or []
|
|
did_work = False
|
|
for job in jobs:
|
|
if job.get("enabled") is False:
|
|
continue
|
|
weight = int(job.get("weight", 1))
|
|
for _ in range(weight):
|
|
items = pending(job)
|
|
if not items:
|
|
continue
|
|
item = items[0]
|
|
try:
|
|
status, n = process(job, item, L)
|
|
except Exception as e:
|
|
log("сбой обработки %s: %s" % (job["vendor"], str(e)[:120]))
|
|
continue
|
|
did_work = True
|
|
log("%-9s %-5s +%-2d %s" % (job["vendor"], status, n, item["url"][-70:]))
|
|
time.sleep(1)
|
|
if not did_work:
|
|
empty_rounds += 1
|
|
if empty_rounds == 1:
|
|
log("очереди пусты — ждём")
|
|
time.sleep(300)
|
|
else:
|
|
empty_rounds = 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|