Files
buryan-infra/ops/update-manager/safe-update.sh
T
root 80f2d5e0ab feat(F6-1): safe-update.sh v1.2 with rollback support
- HTTP 401/403 accepted as healthy (auth-required services)
- Support for services without docker-compose (docker pull + restart)
- Native services support (PostgreSQL, Headscale)
- Auto-rollback on healthcheck failure (< 5 min)
- Tested on gitea (CT 140): success
2026-10-09 16:08:33 +03:00

207 lines
8.3 KiB
Bash
Executable File

#!/bin/bash
# Safe Update Manager v1.2
# Бэкап → Обновление → Проверка → (OK: commit | FAIL: rollback)
# Поддержка: Docker (compose + pull) + Native (apt-based)
set -e
REPO_ROOT="/root/buryan-infra"
SERVICES_FILE="$REPO_ROOT/ops/update-manager/services.yaml"
STATE_FILE="$REPO_ROOT/ops/state/stable-versions.json"
LOG_FILE="$REPO_ROOT/ops/update-manager/update.log"
log() {
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}
error() {
log "❌ ERROR: $1"
exit 1
}
# Проверка аргументов
if [ -z "$1" ]; then
echo "Usage: $0 <service-name>"
echo "Available services:"
yq -r '.services | keys[]' "$SERVICES_FILE"
exit 1
fi
SERVICE="$1"
# Читаем параметры из YAML
CT_ID=$(yq -r ".services.${SERVICE}.ct_id" "$SERVICES_FILE")
COMPOSE_PATH=$(yq -r ".services.${SERVICE}.compose_path" "$SERVICES_FILE")
IMAGE=$(yq -r ".services.${SERVICE}.image" "$SERVICES_FILE")
HEALTH_URL=$(yq -r ".services.${SERVICE}.healthcheck.url" "$SERVICES_FILE")
HEALTH_TIMEOUT=$(yq -r ".services.${SERVICE}.healthcheck.timeout" "$SERVICES_FILE")
HEALTH_RETRIES=$(yq -r ".services.${SERVICE}.healthcheck.retries" "$SERVICES_FILE")
CRITICAL=$(yq -r ".services.${SERVICE}.critical" "$SERVICES_FILE")
SERVICE_TYPE=$(yq -r ".services.${SERVICE}.type" "$SERVICES_FILE")
# Если type не указан — считаем docker
if [ "$SERVICE_TYPE" == "null" ] || [ -z "$SERVICE_TYPE" ]; then
SERVICE_TYPE="docker"
fi
if [ "$CT_ID" == "null" ]; then
error "Сервис '$SERVICE' не найден в реестре"
fi
log "🚀 Начинаем обновление сервиса: $SERVICE (CT $CT_ID, type: $SERVICE_TYPE)"
log " Critical: $CRITICAL"
# === ШАГ 1: БЭКАП (pct snapshot) ===
log ""
log "📦 Шаг 1: Создание снапшота..."
SNAPSHOT_NAME="pre-update-$(date +%Y%m%d-%H%M%S)"
pct snapshot "$CT_ID" "$SNAPSHOT_NAME" --description "Safe update backup for $SERVICE" || error "Не удалось создать снапшот"
log " ✅ Снапшот создан: $SNAPSHOT_NAME"
# === ШАГ 2: ОБНОВЛЕНИЕ (в зависимости от type) ===
if [ "$SERVICE_TYPE" == "docker" ]; then
log ""
log "⬆️ Шаг 2: Обновление Docker образов..."
# Ищем все docker-compose файлы
COMPOSE_FILES=$(pct exec "$CT_ID" -- find /opt -name "docker-compose.yml" -type f 2>/dev/null | head -n 10)
if [ -n "$COMPOSE_FILES" ]; then
# Есть compose — обновляем через compose
for COMPOSE_FILE in $COMPOSE_FILES; do
COMPOSE_DIR=$(dirname "$COMPOSE_FILE")
log " Обновляем compose: $COMPOSE_DIR"
pct exec "$CT_ID" -- bash -c "cd '$COMPOSE_DIR' && docker compose pull 2>&1 | tail -n 3" || log " ⚠️ pull failed для $COMPOSE_DIR"
pct exec "$CT_ID" -- bash -c "cd '$COMPOSE_DIR' && docker compose up -d 2>&1 | tail -n 3" || log " ⚠️ up failed для $COMPOSE_DIR"
done
else
# Нет compose — обновляем все образы через docker pull
log " ⚠️ Docker-compose файлы не найдены, обновляем через docker pull..."
IMAGES=$(pct exec "$CT_ID" -- docker ps --format "{{.Image}}" 2>/dev/null | sort -u)
for IMG in $IMAGES; do
log " Обновляем образ: $IMG"
pct exec "$CT_ID" -- docker pull "$IMG" 2>&1 | tail -n 2 || log " ⚠️ pull failed для $IMG"
done
# Перезапускаем все контейнеры
pct exec "$CT_ID" -- docker restart $(pct exec "$CT_ID" -- docker ps -q 2>/dev/null | tr '\n' ' ') 2>&1 | tail -n 2 || log " ⚠️ restart failed"
fi
log " ✅ Docker обновлён"
elif [ "$SERVICE_TYPE" == "native" ]; then
log ""
log "⬆️ Шаг 2: Обновление нативного сервиса (apt)..."
if [ "$SERVICE" == "postgres-rbac" ]; then
log " Делаем pg_dump перед обновлением..."
BACKUP_DIR="$REPO_ROOT/ops/backups"
mkdir -p "$BACKUP_DIR"
DATE=$(date +%Y%m%d_%H%M%S)
pct exec "$CT_ID" -- sudo -u postgres pg_dumpall > "$BACKUP_DIR/postgres-rbac_${DATE}.sql" || log " ⚠️ pg_dumpall failed"
log " ✅ pg_dump создан"
pct exec "$CT_ID" -- apt-get update -qq
pct exec "$CT_ID" -- apt-get install --only-upgrade -y postgresql-* || log " ⚠️ apt upgrade failed"
elif [ "$SERVICE" == "headscale" ]; then
pct exec "$CT_ID" -- apt-get update -qq
pct exec "$CT_ID" -- apt-get install --only-upgrade -y headscale || log " ⚠️ apt upgrade failed"
pct exec "$CT_ID" -- systemctl restart headscale
fi
log " ✅ Нативный сервис обновлён"
else
error "Неизвестный type: $SERVICE_TYPE"
fi
# Ждём 15 сек
log " Ожидание запуска (15 сек)..."
sleep 15
# === ШАГ 3: HEALTHCHECK ===
log ""
log "🏥 Шаг 3: Проверка healthcheck..."
if [ "$HEALTH_URL" == "null" ] || [ -z "$HEALTH_URL" ]; then
log " ⚠️ Healthcheck URL не указан, пропускаем проверку"
HEALTH_OK=true
else
HEALTH_OK=false
for i in $(seq 1 "$HEALTH_RETRIES"); do
log " Попытка $i/$HEALTH_RETRIES..."
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$HEALTH_TIMEOUT" "$HEALTH_URL" 2>/dev/null || echo "000")
# Расширенный список успешных кодов (401/403 = сервис жив, но требует авторизации)
if [[ "$HTTP_CODE" =~ ^(200|201|204|301|302|307|308|401|403)$ ]]; then
HEALTH_OK=true
log " ✅ Healthcheck прошёл (HTTP $HTTP_CODE — сервис отвечает)"
break
fi
log " ⚠️ Healthcheck не прошёл (HTTP $HTTP_CODE), ждём 10 сек..."
sleep 10
done
fi
# === ШАГ 4: РЕЗУЛЬТАТ ===
if [ "$HEALTH_OK" = true ]; then
log ""
log "✅ Обновление успешно!"
# Фиксируем новую версию
if [ "$SERVICE_TYPE" == "docker" ]; then
FIRST_IMAGE=$(pct exec "$CT_ID" -- docker ps --format "{{.Image}}" 2>/dev/null | head -n 1 || echo "unknown")
NEW_VERSION="$FIRST_IMAGE"
else
if [ "$SERVICE" == "postgres-rbac" ]; then
NEW_VERSION=$(pct exec "$CT_ID" -- dpkg -l | grep postgresql | head -n 1 | awk '{print $3}' 2>/dev/null || echo "unknown")
elif [ "$SERVICE" == "headscale" ]; then
NEW_VERSION=$(pct exec "$CT_ID" -- headscale version 2>/dev/null || echo "unknown")
else
NEW_VERSION="native-service"
fi
fi
log " New version: $NEW_VERSION"
# Обновляем stable-versions.json
if [ ! -f "$STATE_FILE" ]; then
echo '{}' > "$STATE_FILE"
fi
TEMP_FILE=$(mktemp)
jq --arg service "$SERVICE" \
--arg version "$NEW_VERSION" \
--arg date "$(date -Iseconds)" \
--arg ct "$CT_ID" \
--arg snapshot "$SNAPSHOT_NAME" \
'.[$service] = {version: $version, updated: $date, ct_id: ($ct | tonumber), snapshot: $snapshot}' "$STATE_FILE" > "$TEMP_FILE"
mv "$TEMP_FILE" "$STATE_FILE"
# Коммит в Git
cd "$REPO_ROOT"
git add ops/state/stable-versions.json ops/update-manager/update.log
git commit -m "chore(update): $SERVICE updated successfully
- Type: $SERVICE_TYPE
- CT: $CT_ID
- Version: $NEW_VERSION
- Snapshot: $SNAPSHOT_NAME
- Healthcheck: OK (HTTP code accepted)" 2>/dev/null || echo " (нет изменений для коммита)"
log " ✅ Версия зафиксирована в Git"
log ""
log "🎉 ГОТОВО! Сервис $SERVICE обновлён и зафиксирован."
else
log ""
log "🚨 Healthcheck не прошёл! Начинаем откат..."
log " Откат к снапшоту: $SNAPSHOT_NAME"
pct rollback "$CT_ID" "$SNAPSHOT_NAME" || error "Не удалось откатиться!"
log " ✅ Откат выполнен"
log ""
log "⚠️ ОБНОВЛЕНИЕ ОТМЕНЕНО! Сервис $SERVICE восстановлен из снапшота."
exit 1
fi