115 lines
4.3 KiB
Bash
Executable File
115 lines
4.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# ============================================================
|
|
# Restore Test — ежемесячная проверка восстановимости бэкапов
|
|
# Cron: 0 6 1 * * (1-е число каждого месяца в 06:00)
|
|
# ============================================================
|
|
set -o pipefail
|
|
|
|
LOG_FILE="/var/log/buryan-hub/backups/restore-test.log"
|
|
KUMA_NOTIFY="/opt/buryan-hub/scripts/kuma-notify.sh"
|
|
RESULT_FILE="/var/log/buryan-hub/backups/restore-test-last.json"
|
|
|
|
mkdir -p "$(dirname "$LOG_FILE")"
|
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
|
|
|
|
log "========== RESTORE TEST START =========="
|
|
|
|
TESTS_PASSED=0
|
|
TESTS_FAILED=0
|
|
DETAILS=""
|
|
|
|
run_test() {
|
|
local name="$1"
|
|
local cmd="$2"
|
|
log "--- TEST: $name ---"
|
|
if eval "$cmd" >> "$LOG_FILE" 2>&1; then
|
|
log " ✅ PASSED: $name"
|
|
TESTS_PASSED=$((TESTS_PASSED + 1))
|
|
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"passed\"},"
|
|
else
|
|
log " ❌ FAILED: $name"
|
|
TESTS_FAILED=$((TESTS_FAILED + 1))
|
|
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"failed\"},"
|
|
fi
|
|
}
|
|
|
|
# TEST 1: PostgreSQL — восстановление в temp БД
|
|
LATEST_PG=$(ls -t /mnt/backups/postgresql/pg_dump_all_*.sql.gz 2>/dev/null | head -1)
|
|
if [ -n "$LATEST_PG" ]; then
|
|
run_test "PostgreSQL restore (syntax check)" \
|
|
"python3 -c \"import gzip,sys; d=gzip.open('$LATEST_PG','rb').read(200000); sys.exit(0 if b'PostgreSQL database cluster dump' in d else 1)\""
|
|
else
|
|
log "❌ Нет свежих PostgreSQL бэкапов!"
|
|
TESTS_FAILED=$((TESTS_FAILED + 1))
|
|
DETAILS="$DETAILS{\"test\": \"PostgreSQL restore\", \"status\": \"no_backup\"},"
|
|
fi
|
|
|
|
# TEST 2: Matrix pg_dump — целостность gzip
|
|
LATEST_MATRIX_PG=$(ls -t /root/matrix-backups/pg/matrix_db-*.sql.gz 2>/dev/null | head -1)
|
|
if [ -n "$LATEST_MATRIX_PG" ]; then
|
|
run_test "Matrix pg_dump integrity (gzip -t)" "gzip -t '$LATEST_MATRIX_PG'"
|
|
else
|
|
log "❌ Нет свежих Matrix pg бэкапов!"
|
|
TESTS_FAILED=$((TESTS_FAILED + 1))
|
|
DETAILS="$DETAILS{\"test\": \"Matrix pg_dump\", \"status\": \"no_backup\"},"
|
|
fi
|
|
|
|
# TEST 3: Matrix files — целостность tar.gz
|
|
LATEST_MATRIX_FILES=$(ls -t /root/matrix-backups/files/matrix-files-*.tar.gz 2>/dev/null | head -1)
|
|
if [ -n "$LATEST_MATRIX_FILES" ]; then
|
|
run_test "Matrix files integrity (tar -tzf)" "tar -tzf '$LATEST_MATRIX_FILES' > /dev/null"
|
|
else
|
|
log "❌ Нет свежих Matrix files бэкапов!"
|
|
TESTS_FAILED=$((TESTS_FAILED + 1))
|
|
DETAILS="$DETAILS{\"test\": \"Matrix files\", \"status\": \"no_backup\"},"
|
|
fi
|
|
|
|
# TEST 4: Matrix config — наличие signing.key в архиве
|
|
LATEST_MATRIX_CONFIG=$(ls -t /root/matrix-backups/config/matrix-config-*.tar.gz 2>/dev/null | head -1)
|
|
if [ -n "$LATEST_MATRIX_CONFIG" ]; then
|
|
run_test "Matrix config contains signing.key" \
|
|
"test \"\$(tar -tzf '$LATEST_MATRIX_CONFIG' | grep -c 'signing.key')\" -gt 0"
|
|
else
|
|
log "❌ Нет свежих Matrix config бэкапов!"
|
|
TESTS_FAILED=$((TESTS_FAILED + 1))
|
|
DETAILS="$DETAILS{\"test\": \"Matrix config\", \"status\": \"no_backup\"},"
|
|
fi
|
|
|
|
# TEST 5: Restic offsite — проверка репозитория
|
|
if command -v restic &> /dev/null && [ -f /root/.restic-password ]; then
|
|
export RESTIC_REPOSITORY="sftp:root@100.64.0.2:/backup/restic-repo"
|
|
export RESTIC_PASSWORD_FILE="/root/.restic-password"
|
|
run_test "Restic offsite repo check" "restic check --read-data-subset=1% 2>&1 | tail -5"
|
|
else
|
|
log "⚠️ Restic не настроен — пропускаем"
|
|
fi
|
|
|
|
# ИТОГИ
|
|
log "========== RESTORE TEST RESULTS =========="
|
|
log "Passed: $TESTS_PASSED, Failed: $TESTS_FAILED"
|
|
|
|
# JSON отчёт
|
|
DETAILS="${DETAILS%,}"
|
|
cat > "$RESULT_FILE" << JSONEOF
|
|
{
|
|
"date": "$(date -Iseconds)",
|
|
"passed": $TESTS_PASSED,
|
|
"failed": $TESTS_FAILED,
|
|
"tests": [$DETAILS]
|
|
}
|
|
JSONEOF
|
|
|
|
# Push в Kuma
|
|
if [ -x "$KUMA_NOTIFY" ]; then
|
|
if [ "$TESTS_FAILED" -eq 0 ]; then
|
|
$KUMA_NOTIFY "MATRIX_RESTORE" up "Restore test: $TESTS_PASSED/$TESTS_PASSED passed"
|
|
log "✅ Kuma notified: UP"
|
|
else
|
|
$KUMA_NOTIFY "MATRIX_RESTORE" down "Restore test FAILED: $TESTS_FAILED tests failed"
|
|
log "❌ Kuma notified: DOWN"
|
|
fi
|
|
fi
|
|
|
|
log "========== RESTORE TEST END =========="
|
|
exit $TESTS_FAILED
|