- HTTP 401/403 accepted as healthy (auth-required services) - Support for services without docker-compose (docker pull + restart) - Native services support (PostgreSQL, Headscale) - Auto-rollback on healthcheck failure (< 5 min) - Tested on gitea (CT 140): success
207 lines
8.3 KiB
Bash
Executable File
207 lines
8.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Safe Update Manager v1.2
|
|
# Бэкап → Обновление → Проверка → (OK: commit | FAIL: rollback)
|
|
# Поддержка: Docker (compose + pull) + Native (apt-based)
|
|
|
|
set -e
|
|
|
|
REPO_ROOT="/root/buryan-infra"
|
|
SERVICES_FILE="$REPO_ROOT/ops/update-manager/services.yaml"
|
|
STATE_FILE="$REPO_ROOT/ops/state/stable-versions.json"
|
|
LOG_FILE="$REPO_ROOT/ops/update-manager/update.log"
|
|
|
|
log() {
|
|
echo "[$(date +'%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
error() {
|
|
log "❌ ERROR: $1"
|
|
exit 1
|
|
}
|
|
|
|
# Проверка аргументов
|
|
if [ -z "$1" ]; then
|
|
echo "Usage: $0 <service-name>"
|
|
echo "Available services:"
|
|
yq -r '.services | keys[]' "$SERVICES_FILE"
|
|
exit 1
|
|
fi
|
|
|
|
SERVICE="$1"
|
|
|
|
# Читаем параметры из YAML
|
|
CT_ID=$(yq -r ".services.${SERVICE}.ct_id" "$SERVICES_FILE")
|
|
COMPOSE_PATH=$(yq -r ".services.${SERVICE}.compose_path" "$SERVICES_FILE")
|
|
IMAGE=$(yq -r ".services.${SERVICE}.image" "$SERVICES_FILE")
|
|
HEALTH_URL=$(yq -r ".services.${SERVICE}.healthcheck.url" "$SERVICES_FILE")
|
|
HEALTH_TIMEOUT=$(yq -r ".services.${SERVICE}.healthcheck.timeout" "$SERVICES_FILE")
|
|
HEALTH_RETRIES=$(yq -r ".services.${SERVICE}.healthcheck.retries" "$SERVICES_FILE")
|
|
CRITICAL=$(yq -r ".services.${SERVICE}.critical" "$SERVICES_FILE")
|
|
SERVICE_TYPE=$(yq -r ".services.${SERVICE}.type" "$SERVICES_FILE")
|
|
|
|
# Если type не указан — считаем docker
|
|
if [ "$SERVICE_TYPE" == "null" ] || [ -z "$SERVICE_TYPE" ]; then
|
|
SERVICE_TYPE="docker"
|
|
fi
|
|
|
|
if [ "$CT_ID" == "null" ]; then
|
|
error "Сервис '$SERVICE' не найден в реестре"
|
|
fi
|
|
|
|
log "🚀 Начинаем обновление сервиса: $SERVICE (CT $CT_ID, type: $SERVICE_TYPE)"
|
|
log " Critical: $CRITICAL"
|
|
|
|
# === ШАГ 1: БЭКАП (pct snapshot) ===
|
|
log ""
|
|
log "📦 Шаг 1: Создание снапшота..."
|
|
SNAPSHOT_NAME="pre-update-$(date +%Y%m%d-%H%M%S)"
|
|
pct snapshot "$CT_ID" "$SNAPSHOT_NAME" --description "Safe update backup for $SERVICE" || error "Не удалось создать снапшот"
|
|
log " ✅ Снапшот создан: $SNAPSHOT_NAME"
|
|
|
|
# === ШАГ 2: ОБНОВЛЕНИЕ (в зависимости от type) ===
|
|
if [ "$SERVICE_TYPE" == "docker" ]; then
|
|
log ""
|
|
log "⬆️ Шаг 2: Обновление Docker образов..."
|
|
|
|
# Ищем все docker-compose файлы
|
|
COMPOSE_FILES=$(pct exec "$CT_ID" -- find /opt -name "docker-compose.yml" -type f 2>/dev/null | head -n 10)
|
|
|
|
if [ -n "$COMPOSE_FILES" ]; then
|
|
# Есть compose — обновляем через compose
|
|
for COMPOSE_FILE in $COMPOSE_FILES; do
|
|
COMPOSE_DIR=$(dirname "$COMPOSE_FILE")
|
|
log " Обновляем compose: $COMPOSE_DIR"
|
|
pct exec "$CT_ID" -- bash -c "cd '$COMPOSE_DIR' && docker compose pull 2>&1 | tail -n 3" || log " ⚠️ pull failed для $COMPOSE_DIR"
|
|
pct exec "$CT_ID" -- bash -c "cd '$COMPOSE_DIR' && docker compose up -d 2>&1 | tail -n 3" || log " ⚠️ up failed для $COMPOSE_DIR"
|
|
done
|
|
else
|
|
# Нет compose — обновляем все образы через docker pull
|
|
log " ⚠️ Docker-compose файлы не найдены, обновляем через docker pull..."
|
|
IMAGES=$(pct exec "$CT_ID" -- docker ps --format "{{.Image}}" 2>/dev/null | sort -u)
|
|
for IMG in $IMAGES; do
|
|
log " Обновляем образ: $IMG"
|
|
pct exec "$CT_ID" -- docker pull "$IMG" 2>&1 | tail -n 2 || log " ⚠️ pull failed для $IMG"
|
|
done
|
|
# Перезапускаем все контейнеры
|
|
pct exec "$CT_ID" -- docker restart $(pct exec "$CT_ID" -- docker ps -q 2>/dev/null | tr '\n' ' ') 2>&1 | tail -n 2 || log " ⚠️ restart failed"
|
|
fi
|
|
|
|
log " ✅ Docker обновлён"
|
|
|
|
elif [ "$SERVICE_TYPE" == "native" ]; then
|
|
log ""
|
|
log "⬆️ Шаг 2: Обновление нативного сервиса (apt)..."
|
|
|
|
if [ "$SERVICE" == "postgres-rbac" ]; then
|
|
log " Делаем pg_dump перед обновлением..."
|
|
BACKUP_DIR="$REPO_ROOT/ops/backups"
|
|
mkdir -p "$BACKUP_DIR"
|
|
DATE=$(date +%Y%m%d_%H%M%S)
|
|
pct exec "$CT_ID" -- sudo -u postgres pg_dumpall > "$BACKUP_DIR/postgres-rbac_${DATE}.sql" || log " ⚠️ pg_dumpall failed"
|
|
log " ✅ pg_dump создан"
|
|
|
|
pct exec "$CT_ID" -- apt-get update -qq
|
|
pct exec "$CT_ID" -- apt-get install --only-upgrade -y postgresql-* || log " ⚠️ apt upgrade failed"
|
|
|
|
elif [ "$SERVICE" == "headscale" ]; then
|
|
pct exec "$CT_ID" -- apt-get update -qq
|
|
pct exec "$CT_ID" -- apt-get install --only-upgrade -y headscale || log " ⚠️ apt upgrade failed"
|
|
pct exec "$CT_ID" -- systemctl restart headscale
|
|
fi
|
|
|
|
log " ✅ Нативный сервис обновлён"
|
|
else
|
|
error "Неизвестный type: $SERVICE_TYPE"
|
|
fi
|
|
|
|
# Ждём 15 сек
|
|
log " Ожидание запуска (15 сек)..."
|
|
sleep 15
|
|
|
|
# === ШАГ 3: HEALTHCHECK ===
|
|
log ""
|
|
log "🏥 Шаг 3: Проверка healthcheck..."
|
|
|
|
if [ "$HEALTH_URL" == "null" ] || [ -z "$HEALTH_URL" ]; then
|
|
log " ⚠️ Healthcheck URL не указан, пропускаем проверку"
|
|
HEALTH_OK=true
|
|
else
|
|
HEALTH_OK=false
|
|
for i in $(seq 1 "$HEALTH_RETRIES"); do
|
|
log " Попытка $i/$HEALTH_RETRIES..."
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time "$HEALTH_TIMEOUT" "$HEALTH_URL" 2>/dev/null || echo "000")
|
|
|
|
# Расширенный список успешных кодов (401/403 = сервис жив, но требует авторизации)
|
|
if [[ "$HTTP_CODE" =~ ^(200|201|204|301|302|307|308|401|403)$ ]]; then
|
|
HEALTH_OK=true
|
|
log " ✅ Healthcheck прошёл (HTTP $HTTP_CODE — сервис отвечает)"
|
|
break
|
|
fi
|
|
log " ⚠️ Healthcheck не прошёл (HTTP $HTTP_CODE), ждём 10 сек..."
|
|
sleep 10
|
|
done
|
|
fi
|
|
|
|
# === ШАГ 4: РЕЗУЛЬТАТ ===
|
|
if [ "$HEALTH_OK" = true ]; then
|
|
log ""
|
|
log "✅ Обновление успешно!"
|
|
|
|
# Фиксируем новую версию
|
|
if [ "$SERVICE_TYPE" == "docker" ]; then
|
|
FIRST_IMAGE=$(pct exec "$CT_ID" -- docker ps --format "{{.Image}}" 2>/dev/null | head -n 1 || echo "unknown")
|
|
NEW_VERSION="$FIRST_IMAGE"
|
|
else
|
|
if [ "$SERVICE" == "postgres-rbac" ]; then
|
|
NEW_VERSION=$(pct exec "$CT_ID" -- dpkg -l | grep postgresql | head -n 1 | awk '{print $3}' 2>/dev/null || echo "unknown")
|
|
elif [ "$SERVICE" == "headscale" ]; then
|
|
NEW_VERSION=$(pct exec "$CT_ID" -- headscale version 2>/dev/null || echo "unknown")
|
|
else
|
|
NEW_VERSION="native-service"
|
|
fi
|
|
fi
|
|
|
|
log " New version: $NEW_VERSION"
|
|
|
|
# Обновляем stable-versions.json
|
|
if [ ! -f "$STATE_FILE" ]; then
|
|
echo '{}' > "$STATE_FILE"
|
|
fi
|
|
|
|
TEMP_FILE=$(mktemp)
|
|
jq --arg service "$SERVICE" \
|
|
--arg version "$NEW_VERSION" \
|
|
--arg date "$(date -Iseconds)" \
|
|
--arg ct "$CT_ID" \
|
|
--arg snapshot "$SNAPSHOT_NAME" \
|
|
'.[$service] = {version: $version, updated: $date, ct_id: ($ct | tonumber), snapshot: $snapshot}' "$STATE_FILE" > "$TEMP_FILE"
|
|
mv "$TEMP_FILE" "$STATE_FILE"
|
|
|
|
# Коммит в Git
|
|
cd "$REPO_ROOT"
|
|
git add ops/state/stable-versions.json ops/update-manager/update.log
|
|
git commit -m "chore(update): $SERVICE updated successfully
|
|
|
|
- Type: $SERVICE_TYPE
|
|
- CT: $CT_ID
|
|
- Version: $NEW_VERSION
|
|
- Snapshot: $SNAPSHOT_NAME
|
|
- Healthcheck: OK (HTTP code accepted)" 2>/dev/null || echo " (нет изменений для коммита)"
|
|
|
|
log " ✅ Версия зафиксирована в Git"
|
|
log ""
|
|
log "🎉 ГОТОВО! Сервис $SERVICE обновлён и зафиксирован."
|
|
else
|
|
log ""
|
|
log "🚨 Healthcheck не прошёл! Начинаем откат..."
|
|
|
|
log " Откат к снапшоту: $SNAPSHOT_NAME"
|
|
pct rollback "$CT_ID" "$SNAPSHOT_NAME" || error "Не удалось откатиться!"
|
|
|
|
log " ✅ Откат выполнен"
|
|
log ""
|
|
log "⚠️ ОБНОВЛЕНИЕ ОТМЕНЕНО! Сервис $SERVICE восстановлен из снапшота."
|
|
|
|
exit 1
|
|
fi
|