48 lines
2.0 KiB
Bash
Executable File
48 lines
2.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# OpenClaw guard: если gateway не отвечает >3 минут — снять lease/locks и перезапустить.
|
|
set -u
|
|
LOG=/var/log/hub/openclaw-guard.log
|
|
STATE=/var/lib/openclaw-guard.state
|
|
mkdir -p /var/log/hub
|
|
touch "$LOG"
|
|
exec 9>/var/lib/openclaw-guard.lock
|
|
flock -n 9 || exit 0
|
|
|
|
H=$(pct exec 115 -- docker exec openclaw node openclaw.mjs health 2>&1 | tr '\n' ' ')
|
|
if echo "$H" | grep -q "Agents:"; then
|
|
echo 0 > "$STATE"; exit 0
|
|
fi
|
|
N=$(( $(cat "$STATE" 2>/dev/null || echo 0) + 1 ))
|
|
echo "$N" > "$STATE"
|
|
echo "$(date '+%F %T') unhealthy=$N: ${H:0:120}" >> "$LOG"
|
|
|
|
# стартовая фаза может длиться долго — даём 8 проверок, иначе 3
|
|
LIMIT=3
|
|
echo "$H" | grep -qi "still starting" && LIMIT=8
|
|
[ "$N" -lt "$LIMIT" ] && exit 0
|
|
|
|
echo "$(date '+%F %T') RECOVER after $N checks" >> "$LOG"
|
|
pct exec 115 -- docker update --restart=no openclaw >/dev/null 2>&1 || true
|
|
pct exec 115 -- docker stop openclaw >/dev/null 2>&1 || true
|
|
for i in $(seq 1 10); do
|
|
S=$(pct exec 115 -- docker inspect -f '{{.State.Status}}' openclaw 2>/dev/null || echo none)
|
|
[ "$S" = "exited" ] && break
|
|
sleep 2
|
|
done
|
|
TS=$(date +%s)
|
|
pct exec 115 -- cp /opt/openclaw/state/openclaw.sqlite /opt/openclaw/state/openclaw.sqlite.guard-$TS 2>/dev/null || true
|
|
pct exec 115 -- python3 - <<'PY' >>"$LOG" 2>&1
|
|
import sqlite3
|
|
try:
|
|
c=sqlite3.connect("/opt/openclaw/state/openclaw.sqlite")
|
|
c.execute("delete from agent_database_leases"); c.commit()
|
|
print("leases cleared")
|
|
except Exception as e: print("lease clear:", e)
|
|
PY
|
|
pct exec 115 -- rm -f /opt/openclaw/tmp/openclaw-1000/*.lock
|
|
pct exec 115 -- docker start openclaw >/dev/null 2>&1 || true
|
|
pct exec 115 -- docker update --restart=unless-stopped openclaw >/dev/null 2>&1 || true
|
|
echo 0 > "$STATE"
|
|
/usr/bin/python3 /opt/buryan-hub/scripts/notify.py --priority high \
|
|
--title "OpenClaw guard: перезапуск" "Gateway не отвечал; lease/locks сняты, контейнер перезапущен." >/dev/null 2>&1 || true
|