Files

165 lines
6.8 KiB
Python
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""LLM Lab — отдельный трек ускорения локальной LLM (read-only).
Замеряет prefill и генерацию Ollama на разных размерах промпта и моделях.
Ничего не меняет в сервисах: только HTTP-вызовы и отчёт.
Запуск:
llm_lab.py --quick # быстрый смоук (1 модель, 3 размера)
llm_lab.py --full # все модели/размеры (долго)
llm_lab.py --models qwen2.5:3b --sizes 500,4000
llm_lab.py --json # только JSONL
Результаты: /var/lib/llm-lab/YYYY-MM-DD.jsonl и summary.md
"""
import argparse
import json
import os
import sys
import time
import urllib.request
from datetime import date
LLM = "http://192.168.1.229:11434"
OUT = "/var/lib/llm-lab"
DEFAULT_MODELS = ["qwen2.5:3b-32k", "qwen2.5:3b", "qwen2.5:7b"]
DEFAULT_SIZES = [300, 2000, 8000, 14000]
QUICK_SIZES = [300, 2000]
def make_prompt(tokens):
# ~4 символа на токен для английского; для русского будет больше токенов
n = max(40, int(tokens * 4))
base = ("The quick brown fox jumps over the lazy dog. ")
return (base * (n // len(base) + 1))[:n]
def post(path, payload, timeout):
req = urllib.request.Request(LLM + path, data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"},
method="POST")
t0 = time.time()
with urllib.request.urlopen(req, timeout=timeout) as r:
body = json.loads(r.read().decode())
return body, time.time() - t0
def test_native(model, tokens, timeout):
prompt = make_prompt(tokens) + "\nОтветь одним словом: ок"
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
"stream": False,
"options": {"num_ctx": 32768, "num_predict": 8}}
body, wall = post("/api/chat", payload, timeout)
pe = body.get("prompt_eval_count") or 0
pd = (body.get("prompt_eval_duration") or 0) / 1e9
ec = body.get("eval_count") or 0
ed = (body.get("eval_duration") or 0) / 1e9
return {
"model": model, "target_tokens": tokens,
"prompt_tokens": pe, "gen_tokens": ec,
"prefill_s": round(pd, 2), "gen_s": round(ed, 2),
"wall_s": round(wall, 2),
"prefill_tps": round(pe / pd, 1) if pd > 0 else None,
"gen_tps": round(ec / ed, 1) if ed > 0 else None,
"reply": (body.get("message") or {}).get("content", "")[:80],
}
def test_openai(model, tokens, timeout):
prompt = make_prompt(tokens) + "\nОтветь одним словом: ок"
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
"max_tokens": 8}
body, wall = post("/v1/chat/completions", payload, timeout)
usage = body.get("usage") or {}
return {
"model": model, "target_tokens": tokens, "endpoint": "openai",
"prompt_tokens": usage.get("prompt_tokens"),
"wall_s": round(wall, 2),
"reply": ((body.get("choices") or [{}])[0].get("message") or {}).get("content", "")[:80],
}
def warm(model, timeout=300):
try:
post("/api/chat", {"model": model, "messages": [{"role": "user", "content": "ок"}],
"stream": False, "options": {"num_predict": 1}}, timeout)
except Exception:
pass
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--quick", action="store_true")
ap.add_argument("--full", action="store_true")
ap.add_argument("--models", default="")
ap.add_argument("--sizes", default="")
ap.add_argument("--timeout", type=int, default=900)
ap.add_argument("--json", action="store_true")
args = ap.parse_args()
models = [m.strip() for m in args.models.split(",") if m.strip()] or DEFAULT_MODELS
sizes = [int(s) for s in args.sizes.split(",") if s.strip()] or \
(QUICK_SIZES if args.quick else DEFAULT_SIZES)
if args.quick:
models = models[:1]
os.makedirs(OUT, exist_ok=True)
out = os.path.join(OUT, date.today().isoformat() + ".jsonl")
rows = []
print("LLM Lab: модели=%s размеры=%s" % (models, sizes))
for m in models:
print(" прогрев", m)
warm(m)
for n in sizes:
try:
r = test_native(m, n, args.timeout)
r["endpoint"] = "native"
except Exception as e:
r = {"model": m, "target_tokens": n, "error": str(e)[:120]}
rows.append(r)
with open(out, "a", encoding="utf-8") as f:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
if not args.json:
if r.get("error"):
print(" %-20s %5d → ошибка: %s" % (m, n, r["error"]))
else:
print(" %-20s %5d → prefill %.1fс (%.0f т/с), gen %.1fс, всего %.1fс"
% (m, n, r["prefill_s"], r["prefill_tps"] or 0,
r["gen_s"], r["wall_s"]))
# один замер через OpenAI-совместимый endpoint для сравнения
try:
r = test_openai(models[0], sizes[0], args.timeout)
rows.append(r)
with open(out, "a", encoding="utf-8") as f:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
if not args.json:
print(" openai endpoint: %.1fс" % r["wall_s"])
except Exception as e:
print(" openai endpoint: ошибка", str(e)[:100])
# оценка: сколько займёт промпт OpenClaw (~15k токенов)
native = [r for r in rows if r.get("endpoint") == "native" and r.get("prefill_s")]
if native:
rate = sum(r["prefill_tps"] for r in native if r["prefill_tps"]) / max(
1, len([r for r in native if r["prefill_tps"]]))
est = 15000 / rate if rate else None
print("\nОценка: при %.0f т/с prefill 15k токенов ≈ %.0f с" % (rate, est or 0))
if not args.json:
path = os.path.join(OUT, "summary.md")
with open(path, "w", encoding="utf-8") as f:
f.write("# LLM Lab — %s\n\n" % date.today().isoformat())
f.write("| модель | токенов | prefill, с | prefill, т/с | gen, с | всего, с |\n")
f.write("|---|---|---|---|---|---|\n")
for r in rows:
if r.get("endpoint") == "native" and not r.get("error"):
f.write("| %s | %d | %.1f | %.0f | %.1f | %.1f |\n" % (
r["model"], r["target_tokens"], r["prefill_s"],
r["prefill_tps"] or 0, r["gen_s"], r["wall_s"]))
print("отчёт:", path)
return 0
if __name__ == "__main__":
sys.exit(main())