[MONITORING] Heartbeat бэкапов для Kuma, ремонт интервалов и push-токенов, swap на общем диске

This commit is contained in:
buryan-agent committed 2026-10-28 09:29:37 +03:00
1 parent 668b73ff17
commit 5eb5ae2eb9
2 files changed
+162

No files matched your search

+48
View File
@@ -0,0 +1,48 @@
#!/bin/bash
# Heartbeat для бэкап-мониторов: пингуем Kuma только если копия действительно свежая.
set -uo pipefail
CFG=/opt/buryan-hub/config/kuma.conf
[ -f "$CFG" ] || { echo "нет $CFG"; exit 1; }
# shellcheck disable=SC1090
source "$CFG"
ping_mon() {
local var="KUMA_$1" status=$2 msg=$3
local url="${!var}"
[ -z "$url" ] && return 0
curl -fsS -m 10 "${url}?status=${status}&msg=$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1]))' "$msg")" >/dev/null 2>&1
echo " $1 -> $status ($msg)"
}
age_h() { # возраст файла в часах
local f=$1
[ -e "$f" ] || { echo 9999; return; }
echo $(( ( $(date +%s) - $(stat -c %Y "$f") ) / 3600 ))
}
# 1. Конфиги хаба (ежедневно 03:45)
F=$(ls -t /mnt/archive/backups/configs/*.tar.gz 2>/dev/null | head -1)
A=$(age_h "${F:-/nonexistent}")
[ "$A" -le 26 ] && ping_mon BURYAN_HUB up "конфиги ${A}ч" || ping_mon BURYAN_HUB down "конфиги ${A}ч — устарели"
# 2. Matrix локальный (ежедневно 03:00)
F=$(ls -t /root/matrix-backups/pg/*.sql.gz 2>/dev/null | head -1)
A=$(age_h "${F:-/nonexistent}")
[ "$A" -le 26 ] && ping_mon MATRIX_BACKUP up "matrix ${A}ч" || ping_mon MATRIX_BACKUP down "matrix ${A}ч — устарел"
# 3. Matrix offsite (ежедневно 04:00)
A=$(age_h /var/log/buryan-hub/backups/offsite-manual.log)
[ "$A" -gt 500 ] && A=$(age_h /var/log/buryan-hub/backups/cron-offsite.log)
[ "$A" -le 26 ] && ping_mon MATRIX_OFFSITE up "offsite ${A}ч" || ping_mon MATRIX_OFFSITE down "offsite ${A}ч — устарел"
# 4. Restore-тест (ежемесячно, 1-е число)
if [ -f /var/log/buryan-hub/backups/restore-test-last.json ]; then
A=$(age_h /var/log/buryan-hub/backups/restore-test-last.json)
if [ "$A" -le 840 ]; then
ping_mon MATRIX_RESTORE up "restore-test ${A}ч назад"
else
ping_mon MATRIX_RESTORE down "restore-test ${A}ч назад — пора запустить"
fi
else
ping_mon MATRIX_RESTORE down "нет отчёта restore-теста"
fi
+114
View File
@@ -0,0 +1,114 @@
#!/bin/bash
# ============================================================
# Restore Test — ежемесячная проверка восстановимости бэкапов
# Cron: 0 6 1 * * (1-е число каждого месяца в 06:00)
# ============================================================
set -o pipefail
LOG_FILE="/var/log/buryan-hub/backups/restore-test.log"
KUMA_NOTIFY="/opt/buryan-hub/scripts/kuma-notify.sh"
RESULT_FILE="/var/log/buryan-hub/backups/restore-test-last.json"
mkdir -p "$(dirname "$LOG_FILE")"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
log "========== RESTORE TEST START =========="
TESTS_PASSED=0
TESTS_FAILED=0
DETAILS=""
run_test() {
local name="$1"
local cmd="$2"
log "--- TEST: $name ---"
if eval "$cmd" >> "$LOG_FILE" 2>&1; then
log " ✅ PASSED: $name"
TESTS_PASSED=$((TESTS_PASSED + 1))
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"passed\"},"
else
log " ❌ FAILED: $name"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"failed\"},"
fi
}
# TEST 1: PostgreSQL — восстановление в temp БД
LATEST_PG=$(ls -t /mnt/backups/postgresql/pg_dump_all_*.sql.gz 2>/dev/null | head -1)
if [ -n "$LATEST_PG" ]; then
run_test "PostgreSQL restore (syntax check)" \
"python3 -c \"import gzip,sys; d=gzip.open('$LATEST_PG','rb').read(200000); sys.exit(0 if b'PostgreSQL database cluster dump' in d else 1)\""
else
log "❌ Нет свежих PostgreSQL бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"PostgreSQL restore\", \"status\": \"no_backup\"},"
fi
# TEST 2: Matrix pg_dump — целостность gzip
LATEST_MATRIX_PG=$(ls -t /root/matrix-backups/pg/matrix_db-*.sql.gz 2>/dev/null | head -1)
if [ -n "$LATEST_MATRIX_PG" ]; then
run_test "Matrix pg_dump integrity (gzip -t)" "gzip -t '$LATEST_MATRIX_PG'"
else
log "❌ Нет свежих Matrix pg бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"Matrix pg_dump\", \"status\": \"no_backup\"},"
fi
# TEST 3: Matrix files — целостность tar.gz
LATEST_MATRIX_FILES=$(ls -t /root/matrix-backups/files/matrix-files-*.tar.gz 2>/dev/null | head -1)
if [ -n "$LATEST_MATRIX_FILES" ]; then
run_test "Matrix files integrity (tar -tzf)" "tar -tzf '$LATEST_MATRIX_FILES' > /dev/null"
else
log "❌ Нет свежих Matrix files бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"Matrix files\", \"status\": \"no_backup\"},"
fi
# TEST 4: Matrix config — наличие signing.key в архиве
LATEST_MATRIX_CONFIG=$(ls -t /root/matrix-backups/config/matrix-config-*.tar.gz 2>/dev/null | head -1)
if [ -n "$LATEST_MATRIX_CONFIG" ]; then
run_test "Matrix config contains signing.key" \
"test \"\$(tar -tzf '$LATEST_MATRIX_CONFIG' | grep -c 'signing.key')\" -gt 0"
else
log "❌ Нет свежих Matrix config бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"Matrix config\", \"status\": \"no_backup\"},"
fi
# TEST 5: Restic offsite — проверка репозитория
if command -v restic &> /dev/null && [ -f /root/.restic-password ]; then
export RESTIC_REPOSITORY="sftp:root@100.64.0.2:/backup/restic-repo"
export RESTIC_PASSWORD_FILE="/root/.restic-password"
run_test "Restic offsite repo check" "restic check --read-data-subset=1% 2>&1 | tail -5"
else
log "⚠️ Restic не настроен — пропускаем"
fi
# ИТОГИ
log "========== RESTORE TEST RESULTS =========="
log "Passed: $TESTS_PASSED, Failed: $TESTS_FAILED"
# JSON отчёт
DETAILS="${DETAILS%,}"
cat > "$RESULT_FILE" << JSONEOF
{
"date": "$(date -Iseconds)",
"passed": $TESTS_PASSED,
"failed": $TESTS_FAILED,
"tests": [$DETAILS]
}
JSONEOF
# Push в Kuma
if [ -x "$KUMA_NOTIFY" ]; then
if [ "$TESTS_FAILED" -eq 0 ]; then
$KUMA_NOTIFY "MATRIX_RESTORE" up "Restore test: $TESTS_PASSED/$TESTS_PASSED passed"
log "✅ Kuma notified: UP"
else
$KUMA_NOTIFY "MATRIX_RESTORE" down "Restore test FAILED: $TESTS_FAILED tests failed"
log "❌ Kuma notified: DOWN"
fi
fi
log "========== RESTORE TEST END =========="
exit $TESTS_FAILED