165 lines
6.8 KiB
Python
Executable File
165 lines
6.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""LLM Lab — отдельный трек ускорения локальной LLM (read-only).
|
||
|
||
Замеряет prefill и генерацию Ollama на разных размерах промпта и моделях.
|
||
Ничего не меняет в сервисах: только HTTP-вызовы и отчёт.
|
||
|
||
Запуск:
|
||
llm_lab.py --quick # быстрый смоук (1 модель, 3 размера)
|
||
llm_lab.py --full # все модели/размеры (долго)
|
||
llm_lab.py --models qwen2.5:3b --sizes 500,4000
|
||
llm_lab.py --json # только JSONL
|
||
|
||
Результаты: /var/lib/llm-lab/YYYY-MM-DD.jsonl и summary.md
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
from datetime import date
|
||
|
||
LLM = "http://192.168.1.229:11434"
|
||
OUT = "/var/lib/llm-lab"
|
||
DEFAULT_MODELS = ["qwen2.5:3b-32k", "qwen2.5:3b", "qwen2.5:7b"]
|
||
DEFAULT_SIZES = [300, 2000, 8000, 14000]
|
||
QUICK_SIZES = [300, 2000]
|
||
|
||
|
||
def make_prompt(tokens):
|
||
# ~4 символа на токен для английского; для русского будет больше токенов
|
||
n = max(40, int(tokens * 4))
|
||
base = ("The quick brown fox jumps over the lazy dog. ")
|
||
return (base * (n // len(base) + 1))[:n]
|
||
|
||
|
||
def post(path, payload, timeout):
|
||
req = urllib.request.Request(LLM + path, data=json.dumps(payload).encode(),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST")
|
||
t0 = time.time()
|
||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||
body = json.loads(r.read().decode())
|
||
return body, time.time() - t0
|
||
|
||
|
||
def test_native(model, tokens, timeout):
|
||
prompt = make_prompt(tokens) + "\nОтветь одним словом: ок"
|
||
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
|
||
"stream": False,
|
||
"options": {"num_ctx": 32768, "num_predict": 8}}
|
||
body, wall = post("/api/chat", payload, timeout)
|
||
pe = body.get("prompt_eval_count") or 0
|
||
pd = (body.get("prompt_eval_duration") or 0) / 1e9
|
||
ec = body.get("eval_count") or 0
|
||
ed = (body.get("eval_duration") or 0) / 1e9
|
||
return {
|
||
"model": model, "target_tokens": tokens,
|
||
"prompt_tokens": pe, "gen_tokens": ec,
|
||
"prefill_s": round(pd, 2), "gen_s": round(ed, 2),
|
||
"wall_s": round(wall, 2),
|
||
"prefill_tps": round(pe / pd, 1) if pd > 0 else None,
|
||
"gen_tps": round(ec / ed, 1) if ed > 0 else None,
|
||
"reply": (body.get("message") or {}).get("content", "")[:80],
|
||
}
|
||
|
||
|
||
def test_openai(model, tokens, timeout):
|
||
prompt = make_prompt(tokens) + "\nОтветь одним словом: ок"
|
||
payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
|
||
"max_tokens": 8}
|
||
body, wall = post("/v1/chat/completions", payload, timeout)
|
||
usage = body.get("usage") or {}
|
||
return {
|
||
"model": model, "target_tokens": tokens, "endpoint": "openai",
|
||
"prompt_tokens": usage.get("prompt_tokens"),
|
||
"wall_s": round(wall, 2),
|
||
"reply": ((body.get("choices") or [{}])[0].get("message") or {}).get("content", "")[:80],
|
||
}
|
||
|
||
|
||
def warm(model, timeout=300):
|
||
try:
|
||
post("/api/chat", {"model": model, "messages": [{"role": "user", "content": "ок"}],
|
||
"stream": False, "options": {"num_predict": 1}}, timeout)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--quick", action="store_true")
|
||
ap.add_argument("--full", action="store_true")
|
||
ap.add_argument("--models", default="")
|
||
ap.add_argument("--sizes", default="")
|
||
ap.add_argument("--timeout", type=int, default=900)
|
||
ap.add_argument("--json", action="store_true")
|
||
args = ap.parse_args()
|
||
|
||
models = [m.strip() for m in args.models.split(",") if m.strip()] or DEFAULT_MODELS
|
||
sizes = [int(s) for s in args.sizes.split(",") if s.strip()] or \
|
||
(QUICK_SIZES if args.quick else DEFAULT_SIZES)
|
||
if args.quick:
|
||
models = models[:1]
|
||
|
||
os.makedirs(OUT, exist_ok=True)
|
||
out = os.path.join(OUT, date.today().isoformat() + ".jsonl")
|
||
rows = []
|
||
print("LLM Lab: модели=%s размеры=%s" % (models, sizes))
|
||
for m in models:
|
||
print(" прогрев", m)
|
||
warm(m)
|
||
for n in sizes:
|
||
try:
|
||
r = test_native(m, n, args.timeout)
|
||
r["endpoint"] = "native"
|
||
except Exception as e:
|
||
r = {"model": m, "target_tokens": n, "error": str(e)[:120]}
|
||
rows.append(r)
|
||
with open(out, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
if not args.json:
|
||
if r.get("error"):
|
||
print(" %-20s %5d → ошибка: %s" % (m, n, r["error"]))
|
||
else:
|
||
print(" %-20s %5d → prefill %.1fс (%.0f т/с), gen %.1fс, всего %.1fс"
|
||
% (m, n, r["prefill_s"], r["prefill_tps"] or 0,
|
||
r["gen_s"], r["wall_s"]))
|
||
# один замер через OpenAI-совместимый endpoint для сравнения
|
||
try:
|
||
r = test_openai(models[0], sizes[0], args.timeout)
|
||
rows.append(r)
|
||
with open(out, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
if not args.json:
|
||
print(" openai endpoint: %.1fс" % r["wall_s"])
|
||
except Exception as e:
|
||
print(" openai endpoint: ошибка", str(e)[:100])
|
||
|
||
# оценка: сколько займёт промпт OpenClaw (~15k токенов)
|
||
native = [r for r in rows if r.get("endpoint") == "native" and r.get("prefill_s")]
|
||
if native:
|
||
rate = sum(r["prefill_tps"] for r in native if r["prefill_tps"]) / max(
|
||
1, len([r for r in native if r["prefill_tps"]]))
|
||
est = 15000 / rate if rate else None
|
||
print("\nОценка: при %.0f т/с prefill 15k токенов ≈ %.0f с" % (rate, est or 0))
|
||
|
||
if not args.json:
|
||
path = os.path.join(OUT, "summary.md")
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
f.write("# LLM Lab — %s\n\n" % date.today().isoformat())
|
||
f.write("| модель | токенов | prefill, с | prefill, т/с | gen, с | всего, с |\n")
|
||
f.write("|---|---|---|---|---|---|\n")
|
||
for r in rows:
|
||
if r.get("endpoint") == "native" and not r.get("error"):
|
||
f.write("| %s | %d | %.1f | %.0f | %.1f | %.1f |\n" % (
|
||
r["model"], r["target_tokens"], r["prefill_s"],
|
||
r["prefill_tps"] or 0, r["gen_s"], r["wall_s"]))
|
||
print("отчёт:", path)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|