Files
buryan-infra/ops/combine/research-pack/run_research.py
T

139 lines
5.0 KiB
Python
Executable File

#!/usr/bin/env python3
"""Портативный runner аудита для ПК. Без внешних зависимостей."""
import argparse
import json
import os
import re
import sys
import time
import urllib.request
ROOT = os.path.dirname(os.path.abspath(__file__))
def env(name, default=""):
return os.environ.get(name, default)
def load_tasks(path):
out = []
with open(path, encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
out.append(json.loads(line))
return out
def prompt(task):
f = os.path.join(ROOT, "prompt-vendor.md" if task["kind"] == "vendor" else "prompt-practice.md")
schema = os.path.join(ROOT, "schema-vendor.json" if task["kind"] == "vendor" else "schema-practice.json")
p = open(f, encoding="utf-8").read()
body = json.dumps(task, ensure_ascii=False, indent=2)
schema_txt = open(schema, encoding="utf-8").read()[:12000]
src = ""
sd = os.path.join(ROOT, "sources", task["id"])
if os.path.isdir(sd):
chunks = []
for name in sorted(os.listdir(sd))[:10]:
fp = os.path.join(sd, name)
if os.path.isfile(fp) and name.lower().endswith((".txt", ".md", ".html")):
try:
chunks.append("### %s\n%s" % (name, open(fp, encoding="utf-8", errors="ignore").read()[:12000]))
except Exception:
pass
if chunks:
src = "\n\n## Локальные источники\n" + "\n\n".join(chunks)
return p + "\n\n## Задача\n```json\n" + body + "\n```\n\n## Схема ответа\n```json\n" + schema_txt + "\n```" + src
def call_llm(text):
provider = env("LLM_PROVIDER", "ollama").lower()
base = env("LLM_URL", "http://127.0.0.1:11434").rstrip("/")
model = env("LLM_MODEL", "qwen2.5:7b")
key = env("LLM_API_KEY", "")
timeout = int(env("LLM_TIMEOUT", "600"))
if provider == "openai":
url = base if base.endswith("/v1") else base + "/v1"
url = url + "/chat/completions"
body = {"model": model, "temperature": 0.2, "messages": [{"role": "user", "content": text}]}
headers = {"Content-Type": "application/json"}
if key:
headers["Authorization"] = "Bearer " + key
else:
url = base + "/api/generate"
body = {"model": model, "prompt": text, "stream": False, "options": {"temperature": 0.2}}
headers = {"Content-Type": "application/json"}
req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers, method="POST")
with urllib.request.urlopen(req, timeout=timeout) as r:
data = json.loads(r.read().decode())
if provider == "openai":
return data["choices"][0]["message"]["content"]
return data.get("response", "")
def extract_json(text):
text = text.strip()
if text.startswith("```"):
text = re.sub(r"^```[a-zA-Z]*\n?", "", text)
text = re.sub(r"\n?```$", "", text).strip()
try:
return json.loads(text), None
except Exception:
pass
a, b = text.find("{"), text.rfind("}")
if a >= 0 and b > a:
try:
return json.loads(text[a:b + 1]), None
except Exception as e:
return None, str(e)
return None, "no json"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--tasks", default=os.path.join(ROOT, "tasks.jsonl"))
ap.add_argument("--out", default=os.path.join(ROOT, "results"))
ap.add_argument("--limit", type=int, default=0)
ap.add_argument("--only", default="")
ap.add_argument("--dry-run", action="store_true")
a = ap.parse_args()
os.makedirs(a.out, exist_ok=True)
tasks = load_tasks(a.tasks)
if a.only:
tasks = [t for t in tasks if t["id"] == a.only]
if a.limit:
tasks = tasks[:a.limit]
for t in tasks:
out = os.path.join(a.out, t["id"] + ".json")
if os.path.exists(out):
print("skip", t["id"])
continue
p = prompt(t)
if a.dry_run:
print("dry-run", t["id"], "prompt_chars", len(p))
continue
print("run", t["id"], flush=True)
t0 = time.time()
try:
raw = call_llm(p)
obj, err = extract_json(raw)
if obj is None:
open(os.path.join(a.out, t["id"] + ".raw.txt"), "w", encoding="utf-8").write(raw)
print(" bad json:", err)
continue
obj.setdefault("id", t["id"])
obj.setdefault("kind", t["kind"])
obj.setdefault("vendor" if t["kind"] == "vendor" else "practice", t["name"])
open(out, "w", encoding="utf-8").write(json.dumps(obj, ensure_ascii=False, indent=2))
print(" ok %.1fs" % (time.time() - t0))
except Exception as e:
print(" error:", e)
with open(os.path.join(a.out, t["id"] + ".log"), "a", encoding="utf-8") as f:
f.write("%s %s\n" % (time.time(), e))
print("done")
if __name__ == "__main__":
sys.exit(main())