[INCIDENT] 18.11: pve-guests masked, stale backup lock, journald volatile, restart policy no; fixes + report
This commit is contained in:
1 parent
4608770e7b
commit
46cfb3e80c
17 files changed
+786
-54
No files matched your search
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
# NIC watchdog: если шлюз недоступен 5 минут — перезапустить линк nic0.
|
||||
set -u
|
||||
STATE=/var/lib/nic-watchdog.fails
|
||||
LOG=/var/log/nic-watchdog.log
|
||||
mkdir -p /var/lib
|
||||
N=$(cat "$STATE" 2>/dev/null || echo 0)
|
||||
if ping -c 1 -W 2 192.168.1.1 >/dev/null 2>&1; then
|
||||
echo 0 > "$STATE"
|
||||
exit 0
|
||||
fi
|
||||
N=$((N+1))
|
||||
echo "$N" > "$STATE"
|
||||
echo "$(date '+%F %T') gateway unreachable ($N)" >> "$LOG"
|
||||
if [ "$N" -ge 5 ]; then
|
||||
echo "$(date '+%F %T') reset nic0" >> "$LOG"
|
||||
timeout 10 ip link set nic0 down >>"$LOG" 2>&1 || true
|
||||
sleep 3
|
||||
timeout 10 ip link set nic0 up >>"$LOG" 2>&1 || true
|
||||
sleep 5
|
||||
echo 0 > "$STATE"
|
||||
fi
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/bash
|
||||
# Снять stale backup-lock с CT, если реального vzdump нет. Запускается до pve-guests.
|
||||
set -u
|
||||
LOG=/var/log/pve-stale-locks.log
|
||||
for f in /etc/pve/lxc/*.conf; do
|
||||
[ -f "$f" ] || continue
|
||||
id=$(basename "$f" .conf)
|
||||
lock=$(grep -E '^lock:' "$f" | awk '{print $2}' | head -1)
|
||||
[ "$lock" = "backup" ] || continue
|
||||
if pgrep -f "vzdump.*$id" >/dev/null 2>&1; then
|
||||
echo "$(date '+%F %T') CT$id backup lock active, skip" >> "$LOG"
|
||||
continue
|
||||
fi
|
||||
if pct unlock "$id" >>"$LOG" 2>&1; then
|
||||
echo "$(date '+%F %T') CT$id stale backup lock removed" >> "$LOG"
|
||||
fi
|
||||
done
|
||||
Reference in new issue
Block a user