[MONITORING] Heartbeat бэкапов для Kuma, ремонт интервалов и push-токенов, swap на общем диске
This commit is contained in:
1 parent
668b73ff17
commit
5eb5ae2eb9
2 files changed
+162
No files matched your search
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# Heartbeat для бэкап-мониторов: пингуем Kuma только если копия действительно свежая.
|
||||
set -uo pipefail
|
||||
CFG=/opt/buryan-hub/config/kuma.conf
|
||||
[ -f "$CFG" ] || { echo "нет $CFG"; exit 1; }
|
||||
# shellcheck disable=SC1090
|
||||
source "$CFG"
|
||||
|
||||
ping_mon() {
|
||||
local var="KUMA_$1" status=$2 msg=$3
|
||||
local url="${!var}"
|
||||
[ -z "$url" ] && return 0
|
||||
curl -fsS -m 10 "${url}?status=${status}&msg=$(python3 -c 'import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1]))' "$msg")" >/dev/null 2>&1
|
||||
echo " $1 -> $status ($msg)"
|
||||
}
|
||||
|
||||
age_h() { # возраст файла в часах
|
||||
local f=$1
|
||||
[ -e "$f" ] || { echo 9999; return; }
|
||||
echo $(( ( $(date +%s) - $(stat -c %Y "$f") ) / 3600 ))
|
||||
}
|
||||
|
||||
# 1. Конфиги хаба (ежедневно 03:45)
|
||||
F=$(ls -t /mnt/archive/backups/configs/*.tar.gz 2>/dev/null | head -1)
|
||||
A=$(age_h "${F:-/nonexistent}")
|
||||
[ "$A" -le 26 ] && ping_mon BURYAN_HUB up "конфиги ${A}ч" || ping_mon BURYAN_HUB down "конфиги ${A}ч — устарели"
|
||||
|
||||
# 2. Matrix локальный (ежедневно 03:00)
|
||||
F=$(ls -t /root/matrix-backups/pg/*.sql.gz 2>/dev/null | head -1)
|
||||
A=$(age_h "${F:-/nonexistent}")
|
||||
[ "$A" -le 26 ] && ping_mon MATRIX_BACKUP up "matrix ${A}ч" || ping_mon MATRIX_BACKUP down "matrix ${A}ч — устарел"
|
||||
|
||||
# 3. Matrix offsite (ежедневно 04:00)
|
||||
A=$(age_h /var/log/buryan-hub/backups/offsite-manual.log)
|
||||
[ "$A" -gt 500 ] && A=$(age_h /var/log/buryan-hub/backups/cron-offsite.log)
|
||||
[ "$A" -le 26 ] && ping_mon MATRIX_OFFSITE up "offsite ${A}ч" || ping_mon MATRIX_OFFSITE down "offsite ${A}ч — устарел"
|
||||
|
||||
# 4. Restore-тест (ежемесячно, 1-е число)
|
||||
if [ -f /var/log/buryan-hub/backups/restore-test-last.json ]; then
|
||||
A=$(age_h /var/log/buryan-hub/backups/restore-test-last.json)
|
||||
if [ "$A" -le 840 ]; then
|
||||
ping_mon MATRIX_RESTORE up "restore-test ${A}ч назад"
|
||||
else
|
||||
ping_mon MATRIX_RESTORE down "restore-test ${A}ч назад — пора запустить"
|
||||
fi
|
||||
else
|
||||
ping_mon MATRIX_RESTORE down "нет отчёта restore-теста"
|
||||
fi
|
||||
Executable
+114
@@ -0,0 +1,114 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# Restore Test — ежемесячная проверка восстановимости бэкапов
|
||||
# Cron: 0 6 1 * * (1-е число каждого месяца в 06:00)
|
||||
# ============================================================
|
||||
set -o pipefail
|
||||
|
||||
LOG_FILE="/var/log/buryan-hub/backups/restore-test.log"
|
||||
KUMA_NOTIFY="/opt/buryan-hub/scripts/kuma-notify.sh"
|
||||
RESULT_FILE="/var/log/buryan-hub/backups/restore-test-last.json"
|
||||
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
|
||||
|
||||
log "========== RESTORE TEST START =========="
|
||||
|
||||
TESTS_PASSED=0
|
||||
TESTS_FAILED=0
|
||||
DETAILS=""
|
||||
|
||||
run_test() {
|
||||
local name="$1"
|
||||
local cmd="$2"
|
||||
log "--- TEST: $name ---"
|
||||
if eval "$cmd" >> "$LOG_FILE" 2>&1; then
|
||||
log " ✅ PASSED: $name"
|
||||
TESTS_PASSED=$((TESTS_PASSED + 1))
|
||||
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"passed\"},"
|
||||
else
|
||||
log " ❌ FAILED: $name"
|
||||
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||
DETAILS="$DETAILS{\"test\": \"$name\", \"status\": \"failed\"},"
|
||||
fi
|
||||
}
|
||||
|
||||
# TEST 1: PostgreSQL — восстановление в temp БД
|
||||
LATEST_PG=$(ls -t /mnt/backups/postgresql/pg_dump_all_*.sql.gz 2>/dev/null | head -1)
|
||||
if [ -n "$LATEST_PG" ]; then
|
||||
run_test "PostgreSQL restore (syntax check)" \
|
||||
"python3 -c \"import gzip,sys; d=gzip.open('$LATEST_PG','rb').read(200000); sys.exit(0 if b'PostgreSQL database cluster dump' in d else 1)\""
|
||||
else
|
||||
log "❌ Нет свежих PostgreSQL бэкапов!"
|
||||
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||
DETAILS="$DETAILS{\"test\": \"PostgreSQL restore\", \"status\": \"no_backup\"},"
|
||||
fi
|
||||
|
||||
# TEST 2: Matrix pg_dump — целостность gzip
|
||||
LATEST_MATRIX_PG=$(ls -t /root/matrix-backups/pg/matrix_db-*.sql.gz 2>/dev/null | head -1)
|
||||
if [ -n "$LATEST_MATRIX_PG" ]; then
|
||||
run_test "Matrix pg_dump integrity (gzip -t)" "gzip -t '$LATEST_MATRIX_PG'"
|
||||
else
|
||||
log "❌ Нет свежих Matrix pg бэкапов!"
|
||||
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||
DETAILS="$DETAILS{\"test\": \"Matrix pg_dump\", \"status\": \"no_backup\"},"
|
||||
fi
|
||||
|
||||
# TEST 3: Matrix files — целостность tar.gz
|
||||
LATEST_MATRIX_FILES=$(ls -t /root/matrix-backups/files/matrix-files-*.tar.gz 2>/dev/null | head -1)
|
||||
if [ -n "$LATEST_MATRIX_FILES" ]; then
|
||||
run_test "Matrix files integrity (tar -tzf)" "tar -tzf '$LATEST_MATRIX_FILES' > /dev/null"
|
||||
else
|
||||
log "❌ Нет свежих Matrix files бэкапов!"
|
||||
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||
DETAILS="$DETAILS{\"test\": \"Matrix files\", \"status\": \"no_backup\"},"
|
||||
fi
|
||||
|
||||
# TEST 4: Matrix config — наличие signing.key в архиве
|
||||
LATEST_MATRIX_CONFIG=$(ls -t /root/matrix-backups/config/matrix-config-*.tar.gz 2>/dev/null | head -1)
|
||||
if [ -n "$LATEST_MATRIX_CONFIG" ]; then
|
||||
run_test "Matrix config contains signing.key" \
|
||||
"test \"\$(tar -tzf '$LATEST_MATRIX_CONFIG' | grep -c 'signing.key')\" -gt 0"
|
||||
else
|
||||
log "❌ Нет свежих Matrix config бэкапов!"
|
||||
TESTS_FAILED=$((TESTS_FAILED + 1))
|
||||
DETAILS="$DETAILS{\"test\": \"Matrix config\", \"status\": \"no_backup\"},"
|
||||
fi
|
||||
|
||||
# TEST 5: Restic offsite — проверка репозитория
|
||||
if command -v restic &> /dev/null && [ -f /root/.restic-password ]; then
|
||||
export RESTIC_REPOSITORY="sftp:root@100.64.0.2:/backup/restic-repo"
|
||||
export RESTIC_PASSWORD_FILE="/root/.restic-password"
|
||||
run_test "Restic offsite repo check" "restic check --read-data-subset=1% 2>&1 | tail -5"
|
||||
else
|
||||
log "⚠️ Restic не настроен — пропускаем"
|
||||
fi
|
||||
|
||||
# ИТОГИ
|
||||
log "========== RESTORE TEST RESULTS =========="
|
||||
log "Passed: $TESTS_PASSED, Failed: $TESTS_FAILED"
|
||||
|
||||
# JSON отчёт
|
||||
DETAILS="${DETAILS%,}"
|
||||
cat > "$RESULT_FILE" << JSONEOF
|
||||
{
|
||||
"date": "$(date -Iseconds)",
|
||||
"passed": $TESTS_PASSED,
|
||||
"failed": $TESTS_FAILED,
|
||||
"tests": [$DETAILS]
|
||||
}
|
||||
JSONEOF
|
||||
|
||||
# Push в Kuma
|
||||
if [ -x "$KUMA_NOTIFY" ]; then
|
||||
if [ "$TESTS_FAILED" -eq 0 ]; then
|
||||
$KUMA_NOTIFY "MATRIX_RESTORE" up "Restore test: $TESTS_PASSED/$TESTS_PASSED passed"
|
||||
log "✅ Kuma notified: UP"
|
||||
else
|
||||
$KUMA_NOTIFY "MATRIX_RESTORE" down "Restore test FAILED: $TESTS_FAILED tests failed"
|
||||
log "❌ Kuma notified: DOWN"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "========== RESTORE TEST END =========="
|
||||
exit $TESTS_FAILED
|
||||
Reference in new issue
Block a user