From 4ba95abe8beac6817c31589274c368ef8ec90fab Mon Sep 17 00:00:00 2001 From: Roman Buryan Date: Tue, 17 Nov 2026 20:39:15 +0300 Subject: [PATCH] =?UTF-8?q?[OPS]=20OpenClaw=20guard:=20systemd=20timer=20?= =?UTF-8?q?=D0=BA=D0=B0=D0=B6=D0=B4=D1=83=D1=8E=20=D0=BC=D0=B8=D0=BD=D1=83?= =?UTF-8?q?=D1=82=D1=83=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D1=8F=D0=B5?= =?UTF-8?q?=D1=82=20health;=20=D0=BF=D1=80=D0=B8=20=D0=B7=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=81=D0=B0=D0=BD=D0=B8=D0=B8=20>3=20=D0=BC=D0=B8=D0=BD?= =?UTF-8?q?=20=D1=81=D0=BE=D1=85=D1=80=D0=B0=D0=BD=D1=8F=D0=B5=D1=82=20sta?= =?UTF-8?q?te=20DB,=20=D1=87=D0=B8=D1=81=D1=82=D0=B8=D1=82=20agent=5Fdatab?= =?UTF-8?q?ase=5Fleases/tmp-locks=20=D0=B8=20=D0=BF=D0=B5=D1=80=D0=B5?= =?UTF-8?q?=D0=B7=D0=B0=D0=BF=D1=83=D1=81=D0=BA=D0=B0=D0=B5=D1=82=20=D0=BA?= =?UTF-8?q?=D0=BE=D0=BD=D1=82=D0=B5=D0=B9=D0=BD=D0=B5=D1=80,=20=D1=83?= =?UTF-8?q?=D0=B2=D0=B5=D0=B4=D0=BE=D0=BC=D0=BB=D1=8F=D0=B5=D1=82=20ALERT.?= =?UTF-8?q?=20Eval=20retry=20wait=20=D1=83=D0=B2=D0=B5=D0=BB=D0=B8=D1=87?= =?UTF-8?q?=D0=B5=D0=BD=20=D0=B4=D0=BE=20240=D1=81.=20=D0=AD=D1=82=D0=BE?= =?UTF-8?q?=20=D0=BE=D0=BF=D0=B5=D1=80=D0=B0=D1=86=D0=B8=D0=BE=D0=BD=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D1=84=D0=B8=D0=BA=D1=81=20state-lock=20=D0=BD?= =?UTF-8?q?=D0=B0=20=D0=B4=D0=BB=D0=B8=D0=BD=D0=BD=D1=8B=D1=85=20turn'?= =?UTF-8?q?=D0=B0=D1=85;=20=D0=BA=D0=BE=D1=80=D0=BD=D0=B5=D0=B2=D0=B0?= =?UTF-8?q?=D1=8F=20=D0=BF=D1=80=D0=B8=D1=87=D0=B8=D0=BD=D0=B0=20=D0=B2=20?= =?UTF-8?q?OpenClaw=20=D0=BE=D1=81=D1=82=D0=B0=D1=91=D1=82=D1=81=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ops/audits/OPENCLAW-GUARD-2026-11-17.md | 39 ++++++++++++++++++++ ops/scripts/openclaw_guard.sh | 47 +++++++++++++++++++++++++ ops/scripts/secretary_eval.py | 2 +- 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 ops/audits/OPENCLAW-GUARD-2026-11-17.md create mode 100755 ops/scripts/openclaw_guard.sh diff --git a/ops/audits/OPENCLAW-GUARD-2026-11-17.md b/ops/audits/OPENCLAW-GUARD-2026-11-17.md new file mode 100644 index 0000000..0b1ff58 --- /dev/null +++ b/ops/audits/OPENCLAW-GUARD-2026-11-17.md @@ -0,0 +1,39 @@ +# OpenClaw guard: самовосстановление после state-lock (17.11.2026) + +## Симптом + +На длинных агентских turn'ах (медиа-запросы 80–108 с) gateway становился +недоступен: CLI получал «Gateway not reachable», контейнер уходил в +doctor/maintenance, дальше — ожидание предыдущего owner и цикл. + +## Что сделано + +`ops/scripts/openclaw_guard.sh` + systemd timer (каждую минуту): + +* проверяет `openclaw health`; +* если gateway не отвечает 3 проверки (или 8 на фазе startup) — + останавливает контейнер, сохраняет `openclaw.sqlite.guard-`, + удаляет `agent_database_leases` и tmp-locks, запускает контейнер; +* уведомление в ALERT (high); +* защита от параллельных запусков через flock. + +Установлено: +``` +openclaw-guard.timer active +openclaw-guard.service oneshot +``` + +## Ограничение + +Это **операционный fix**, а не устранение корневой причины в OpenClaw: +долгий turn всё ещё может оборваться, но система восстанавливается +автоматически за ~3–4 минуты вместо ручного recovery. + +Eval увеличен до ожидания 240 с и повтора запроса. + +## Как проверить + +```bash +systemctl status openclaw-guard.timer +tail -5 /var/log/hub/openclaw-guard.log +``` diff --git a/ops/scripts/openclaw_guard.sh b/ops/scripts/openclaw_guard.sh new file mode 100755 index 0000000..da958b2 --- /dev/null +++ b/ops/scripts/openclaw_guard.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# OpenClaw guard: если gateway не отвечает >3 минут — снять lease/locks и перезапустить. +set -u +LOG=/var/log/hub/openclaw-guard.log +STATE=/var/lib/openclaw-guard.state +mkdir -p /var/log/hub +touch "$LOG" +exec 9>/var/lib/openclaw-guard.lock +flock -n 9 || exit 0 + +H=$(pct exec 115 -- docker exec openclaw node openclaw.mjs health 2>&1 | tr '\n' ' ') +if echo "$H" | grep -q "Agents:"; then + echo 0 > "$STATE"; exit 0 +fi +N=$(( $(cat "$STATE" 2>/dev/null || echo 0) + 1 )) +echo "$N" > "$STATE" +echo "$(date '+%F %T') unhealthy=$N: ${H:0:120}" >> "$LOG" + +# стартовая фаза может длиться долго — даём 8 проверок, иначе 3 +LIMIT=3 +echo "$H" | grep -qi "still starting" && LIMIT=8 +[ "$N" -lt "$LIMIT" ] && exit 0 + +echo "$(date '+%F %T') RECOVER after $N checks" >> "$LOG" +pct exec 115 -- docker update --restart=no openclaw >/dev/null 2>&1 || true +pct exec 115 -- docker stop openclaw >/dev/null 2>&1 || true +for i in $(seq 1 10); do + S=$(pct exec 115 -- docker inspect -f '{{.State.Status}}' openclaw 2>/dev/null || echo none) + [ "$S" = "exited" ] && break + sleep 2 +done +TS=$(date +%s) +pct exec 115 -- cp /opt/openclaw/state/openclaw.sqlite /opt/openclaw/state/openclaw.sqlite.guard-$TS 2>/dev/null || true +pct exec 115 -- python3 - <<'PY' >>"$LOG" 2>&1 +import sqlite3 +try: + c=sqlite3.connect("/opt/openclaw/state/openclaw.sqlite") + c.execute("delete from agent_database_leases"); c.commit() + print("leases cleared") +except Exception as e: print("lease clear:", e) +PY +pct exec 115 -- rm -f /opt/openclaw/tmp/openclaw-1000/*.lock +pct exec 115 -- docker start openclaw >/dev/null 2>&1 || true +pct exec 115 -- docker update --restart=unless-stopped openclaw >/dev/null 2>&1 || true +echo 0 > "$STATE" +/usr/bin/python3 /opt/buryan-hub/scripts/notify.py --priority high \ + --title "OpenClaw guard: перезапуск" "Gateway не отвечал; lease/locks сняты, контейнер перезапущен." >/dev/null 2>&1 || true diff --git a/ops/scripts/secretary_eval.py b/ops/scripts/secretary_eval.py index 5e4e273..1819acb 100755 --- a/ops/scripts/secretary_eval.py +++ b/ops/scripts/secretary_eval.py @@ -126,7 +126,7 @@ def agent(prompt, key): except subprocess.TimeoutExpired: return "", round(time.time() - t0, 1), "timeout" if "Gateway not reachable" in raw or "ECONNREFUSED" in raw: - wait_health(150) + wait_health(240) try: r = subprocess.run(cmd, capture_output=True, text=True, timeout=180) raw = r.stdout or ""