Files
buryan-infra/ops/ops/scripts/restore-test.sh
T

115 lines
4.3 KiB
Bash
Executable File

#!/bin/bash
# ============================================================
# Restore Test — ежемесячная проверка восстановимости бэкапов
# Cron: 0 6 1 * * (1-е число каждого месяца в 06:00)
# ============================================================
set -o pipefail
LOG_FILE="/var/log/buryan-hub/backups/restore-test.log"
KUMA_NOTIFY="/opt/buryan-hub/scripts/kuma-notify.sh"
RESULT_FILE="/var/log/buryan-hub/backups/restore-test-last.json"
mkdir -p "$(dirname "$LOG_FILE")"
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
log "========== RESTORE TEST START =========="
TESTS_PASSED=0
TESTS_FAILED=0
DETAILS=""
run_test() {
local name="$1"
local cmd="$2"
log "--- TEST: $name ---"
if eval "$cmd" >> "$LOG_FILE" 2>&1; then
log " ✅ PASSED: $name"
TESTS_PASSED=$((TESTS_PASSED + 1))
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"passed\"},"
else
log " ❌ FAILED: $name"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"failed\"},"
fi
}
# TEST 1: PostgreSQL — восстановление в temp БД
LATEST_PG=$(ls -t /mnt/backups/postgresql/pg_dump_all_*.sql.gz 2>/dev/null | head -1)
if [ -n "$LATEST_PG" ]; then
run_test "PostgreSQL restore (syntax check)" \
"python3 -c \"import gzip,sys; d=gzip.open('$LATEST_PG','rb').read(200000); sys.exit(0 if b'PostgreSQL database cluster dump' in d else 1)\""
else
log "❌ Нет свежих PostgreSQL бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"PostgreSQL restore\", \"status\": \"no_backup\"},"
fi
# TEST 2: Matrix pg_dump — целостность gzip
LATEST_MATRIX_PG=$(ls -t /root/matrix-backups/pg/matrix_db-*.sql.gz 2>/dev/null | head -1)
if [ -n "$LATEST_MATRIX_PG" ]; then
run_test "Matrix pg_dump integrity (gzip -t)" "gzip -t '$LATEST_MATRIX_PG'"
else
log "❌ Нет свежих Matrix pg бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"Matrix pg_dump\", \"status\": \"no_backup\"},"
fi
# TEST 3: Matrix files — целостность tar.gz
LATEST_MATRIX_FILES=$(ls -t /root/matrix-backups/files/matrix-files-*.tar.gz 2>/dev/null | head -1)
if [ -n "$LATEST_MATRIX_FILES" ]; then
run_test "Matrix files integrity (tar -tzf)" "tar -tzf '$LATEST_MATRIX_FILES' > /dev/null"
else
log "❌ Нет свежих Matrix files бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"Matrix files\", \"status\": \"no_backup\"},"
fi
# TEST 4: Matrix config — наличие signing.key в архиве
LATEST_MATRIX_CONFIG=$(ls -t /root/matrix-backups/config/matrix-config-*.tar.gz 2>/dev/null | head -1)
if [ -n "$LATEST_MATRIX_CONFIG" ]; then
run_test "Matrix config contains signing.key" \
"test \"\$(tar -tzf '$LATEST_MATRIX_CONFIG' | grep -c 'signing.key')\" -gt 0"
else
log "❌ Нет свежих Matrix config бэкапов!"
TESTS_FAILED=$((TESTS_FAILED + 1))
DETAILS="$DETAILS{\"test\": \"Matrix config\", \"status\": \"no_backup\"},"
fi
# TEST 5: Restic offsite — проверка репозитория
if command -v restic &> /dev/null && [ -f /root/.restic-password ]; then
export RESTIC_REPOSITORY="sftp:root@100.64.0.2:/backup/restic-repo"
export RESTIC_PASSWORD_FILE="/root/.restic-password"
run_test "Restic offsite repo check" "restic check --read-data-subset=1% 2>&1 | tail -5"
else
log "⚠️ Restic не настроен — пропускаем"
fi
# ИТОГИ
log "========== RESTORE TEST RESULTS =========="
log "Passed: $TESTS_PASSED, Failed: $TESTS_FAILED"
# JSON отчёт
DETAILS="${DETAILS%,}"
cat > "$RESULT_FILE" << JSONEOF
{
"date": "$(date -Iseconds)",
"passed": $TESTS_PASSED,
"failed": $TESTS_FAILED,
"tests": [$DETAILS]
}
JSONEOF
# Push в Kuma
if [ -x "$KUMA_NOTIFY" ]; then
if [ "$TESTS_FAILED" -eq 0 ]; then
$KUMA_NOTIFY "MATRIX_RESTORE" up "Restore test: $TESTS_PASSED/$TESTS_PASSED passed"
log "✅ Kuma notified: UP"
else
$KUMA_NOTIFY "MATRIX_RESTORE" down "Restore test FAILED: $TESTS_FAILED tests failed"
log "❌ Kuma notified: DOWN"
fi
fi
log "========== RESTORE TEST END =========="
exit $TESTS_FAILED