- Implemented ops/audit.sh for sub-second system state snapshotting - Automated CHANGELOG.md updates with daily metrics (LXC, alerts, disk, git) - Verified auto-commit and auto-push to Gitea main branch - Scheduled daily execution via cron at 09:00 - Added supporting scripts to version control (audit.sh, backup.sh)
250 lines
8.1 KiB
Bash
Executable File
250 lines
8.1 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# ============================================
|
||
# BURYAN HUB - ПОЛНЫЙ АУДИТ ИНФРАСТРУКТУРЫ
|
||
# ============================================
|
||
# Сканирует все LXC + Docker контейнеры внутри
|
||
# Находит все сервисы, порты, сети
|
||
# Генерирует inventory.json и runbook шаблоны
|
||
# ============================================
|
||
|
||
set -e
|
||
|
||
OUTPUT_DIR="/root/buryan-infra/ops/audit"
|
||
INVENTORY_FILE="$OUTPUT_DIR/infrastructure-inventory.json"
|
||
RUNBOOKS_DIR="/root/buryan-infra/services"
|
||
|
||
mkdir -p "$OUTPUT_DIR"
|
||
mkdir -p "$RUNBOOKS_DIR"
|
||
|
||
echo "🔍 BURYAN HUB - Полный аудит инфраструктуры"
|
||
echo "============================================"
|
||
echo ""
|
||
|
||
# Инициализируем JSON
|
||
echo '{"scan_date": "'$(date -Iseconds)'", "containers": [' > "$INVENTORY_FILE"
|
||
|
||
FIRST_CONTAINER=true
|
||
|
||
# ============================================
|
||
# 1. Сканируем все LXC контейнеры на PVE
|
||
# ============================================
|
||
echo "📦 Сканирование LXC контейнеров..."
|
||
|
||
for CT_ID in $(pct list | awk 'NR>1 {print $1}'); do
|
||
CT_STATUS=$(pct status $CT_ID | grep -o "status: [a-z]*" | cut -d' ' -f2)
|
||
|
||
if [ "$CT_STATUS" != "running" ]; then
|
||
echo " ⏭️ CT $CT_ID: stopped (пропущен)"
|
||
continue
|
||
fi
|
||
|
||
CT_CONFIG=$(pct config $CT_ID)
|
||
CT_HOSTNAME=$(echo "$CT_CONFIG" | grep "^hostname:" | cut -d':' -f2 | tr -d ' ')
|
||
CT_IP=$(pct exec $CT_ID -- hostname -I 2>/dev/null | awk '{print $1}' || echo "unknown")
|
||
|
||
echo ""
|
||
echo " LXC CT $CT_ID: $CT_HOSTNAME ($CT_IP)"
|
||
echo " ────────────────────────────────────────"
|
||
|
||
# Добавляем запятую для JSON (не для первого элемента)
|
||
if [ "$FIRST_CONTAINER" = true ]; then
|
||
FIRST_CONTAINER=false
|
||
else
|
||
echo "," >> "$INVENTORY_FILE"
|
||
fi
|
||
|
||
# Начинаем запись контейнера в JSON
|
||
cat >> "$INVENTORY_FILE" << CTJSON
|
||
{
|
||
"type": "lxc",
|
||
"ct_id": $CT_ID,
|
||
"hostname": "$CT_HOSTNAME",
|
||
"ip": "$CT_IP",
|
||
"status": "$CT_STATUS",
|
||
"docker_containers": [
|
||
CTJSON
|
||
|
||
# ============================================
|
||
# 2. Проверяем, есть ли Docker внутри LXC
|
||
# ============================================
|
||
DOCKER_CHECK=$(pct exec $CT_ID -- which docker 2>/dev/null || echo "")
|
||
|
||
if [ -z "$DOCKER_CHECK" ]; then
|
||
echo " └─ Docker не найден"
|
||
# Закрываем JSON для этого контейнера
|
||
echo " ]" >> "$INVENTORY_FILE"
|
||
echo " }" >> "$INVENTORY_FILE"
|
||
continue
|
||
fi
|
||
|
||
# ============================================
|
||
# 3. Сканируем Docker контейнеры внутри LXC
|
||
# ============================================
|
||
echo " 🐳 Docker контейнеры:"
|
||
|
||
# Получаем список всех контейнеров (включая stopped)
|
||
CONTAINERS=$(pct exec $CT_ID -- docker ps -a --format "{{.Names}}|{{.Image}}|{{.Status}}|{{.Ports}}" 2>/dev/null || echo "")
|
||
|
||
FIRST_DOCKER=true
|
||
SERVICES_FOUND=()
|
||
|
||
if [ -z "$CONTAINERS" ]; then
|
||
echo " └─ Нет Docker контейнеров"
|
||
else
|
||
while IFS='|' read -r NAME IMAGE STATUS PORTS; do
|
||
[ -z "$NAME" ] && continue
|
||
|
||
# Извлекаем имя сервиса из образа
|
||
SERVICE_NAME=$(echo "$IMAGE" | cut -d':' -f1 | cut -d'/' -f2)
|
||
SERVICE_PORT=$(echo "$PORTS" | grep -o "0.0.0.0:[0-9]*" | cut -d':' -f2 | head -1)
|
||
|
||
echo " • $NAME ($IMAGE) - $STATUS"
|
||
[ -n "$SERVICE_PORT" ] && echo " Порт: $SERVICE_PORT"
|
||
|
||
# Добавляем в JSON
|
||
if [ "$FIRST_DOCKER" = true ]; then
|
||
FIRST_DOCKER=false
|
||
else
|
||
echo "," >> "$INVENTORY_FILE"
|
||
fi
|
||
|
||
cat >> "$INVENTORY_FILE" << DOCKERJSON
|
||
{
|
||
"name": "$NAME",
|
||
"image": "$IMAGE",
|
||
"status": "$STATUS",
|
||
"port": "$SERVICE_PORT",
|
||
"service_type": "$SERVICE_NAME"
|
||
}
|
||
DOCKERJSON
|
||
|
||
# Добавляем в список для генерации runbook
|
||
if [ -n "$SERVICE_NAME" ]; then
|
||
SERVICES_FOUND+=("$SERVICE_NAME:$CT_ID:$SERVICE_PORT")
|
||
fi
|
||
|
||
done <<< "$CONTAINERS"
|
||
fi
|
||
|
||
# Закрываем JSON структуры
|
||
echo " ]" >> "$INVENTORY_FILE"
|
||
echo " }" >> "$INVENTORY_FILE"
|
||
|
||
# ============================================
|
||
# 4. Генерируем runbook для каждого сервиса
|
||
# ============================================
|
||
for SVC_ENTRY in "${SERVICES_FOUND[@]}"; do
|
||
SVC_NAME=$(echo "$SVC_ENTRY" | cut -d':' -f1)
|
||
SVC_CT=$(echo "$SVC_ENTRY" | cut -d':' -f2)
|
||
SVC_PORT=$(echo "$SVC_ENTRY" | cut -d':' -f3)
|
||
|
||
RUNBOOK_DIR="$RUNBOOKS_DIR/$SVC_NAME"
|
||
RUNBOOK_FILE="$RUNBOOK_DIR/runbook.md"
|
||
|
||
if [ ! -f "$RUNBOOK_FILE" ]; then
|
||
mkdir -p "$RUNBOOK_DIR"
|
||
cat > "$RUNBOOK_FILE" << RUNBOOK
|
||
# 📖 Runbook: $SVC_NAME (CT $SVC_CT)
|
||
|
||
**Авто-сгенерировано:** $(date +%Y-%m-%d)
|
||
**Контейнер:** LXC CT $SVC_CT
|
||
**Порт:** ${SVC_PORT:-N/A}
|
||
|
||
---
|
||
|
||
## 1. Проверка статуса сервисов
|
||
|
||
\`\`\`bash
|
||
# Проверка LXC контейнера
|
||
pct status $SVC_CT
|
||
|
||
# Проверка Docker контейнера внутри
|
||
pct exec $SVC_CT -- docker ps | grep $SVC_NAME
|
||
\`\`\`
|
||
|
||
## 2. Проверка здоровья (Healthcheck)
|
||
|
||
\`\`\`bash
|
||
# Проверка порта
|
||
curl -s -o /dev/null -w "%{http_code}" http://192.168.1.$SVC_CT:${SVC_PORT:-80}/health || echo "Port check"
|
||
|
||
# Проверка Docker
|
||
pct exec $SVC_CT -- docker inspect $SVC_NAME | grep -A 5 "State"
|
||
\`\`\`
|
||
|
||
## 3. Просмотр логов (последние 50 строк)
|
||
|
||
\`\`\`bash
|
||
# Логи Docker контейнера
|
||
pct exec $SVC_CT -- docker logs $SVC_NAME --tail 50
|
||
|
||
# Логи системы LXC
|
||
pct exec $SVC_CT -- journalctl -n 50 --no-pager
|
||
\`\`\`
|
||
|
||
## 4. Перезапуск сервисов
|
||
|
||
\`\`\`bash
|
||
# Мягкий перезапуск Docker контейнера
|
||
pct exec $SVC_CT -- docker restart $SVC_NAME
|
||
|
||
# Перезапуск всего LXC (если нужно)
|
||
pct reboot $SVC_CT
|
||
\`\`\`
|
||
|
||
## 5. Частые проблемы и решения
|
||
|
||
### Проблема: Контейнер не запускается
|
||
**Диагностика:**
|
||
\`\`\`bash
|
||
pct exec $SVC_CT -- docker inspect $SVC_NAME | grep -A 10 "State"
|
||
pct exec $SVC_CT -- docker logs $SVC_NAME --tail 100
|
||
\`\`\`
|
||
|
||
**Решение:**
|
||
\`\`\`bash
|
||
# Пересоздать контейнер
|
||
pct exec $SVC_CT -- docker rm -f $SVC_NAME
|
||
pct exec $SVC_CT -- docker run [параметры из docker-compose]
|
||
\`\`\`
|
||
|
||
### Проблема: Порт не отвечает
|
||
**Диагностика:**
|
||
\`\`\`bash
|
||
pct exec $SVC_CT -- netstat -tlnp | grep $SVC_PORT
|
||
pct exec $SVC_CT -- docker port $SVC_NAME
|
||
\`\`\`
|
||
|
||
**Решение:**
|
||
\`\`\`bash
|
||
# Проверить firewall
|
||
pct exec $SVC_CT -- iptables -L -n | grep $SVC_PORT
|
||
\`\`\`
|
||
RUNBOOK
|
||
echo " 📄 Создан runbook: $RUNBOOK_FILE"
|
||
else
|
||
echo " ️ Runbook существует: $RUNBOOK_FILE"
|
||
fi
|
||
done
|
||
|
||
done
|
||
|
||
# Закрываем главный JSON
|
||
echo "]}" >> "$INVENTORY_FILE"
|
||
|
||
echo ""
|
||
echo "============================================"
|
||
echo "✅ АУДИТ ЗАВЕРШЕН!"
|
||
echo "============================================"
|
||
echo ""
|
||
echo "📁 Инвентарь: $INVENTORY_FILE"
|
||
echo "📚 Runbook'и: $RUNBOOKS_DIR/"
|
||
echo ""
|
||
echo "📊 Статистика:"
|
||
echo " LXC контейнеров: $(pct list | awk 'NR>1' | wc -l)"
|
||
echo " Runbook'ов создано: $(ls -1 $RUNBOOKS_DIR/*/runbook.md 2>/dev/null | wc -l)"
|
||
echo ""
|
||
echo "🔍 Для просмотра инвентаря:"
|
||
echo " cat $INVENTORY_FILE | python3 -m json.tool"
|