feat(monitoring): контроль пула томов с оповещением в Telegram; пароль вынесен из расписания
This commit is contained in:
1 parent
d73db3a826
commit
304980266c
1 file changed
+261
Executable
+261
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
check_thin_pool.py — метрики пула тонких томов PVE и оповещение о переполнении.
|
||||
|
||||
Что изменено 2026-10-14:
|
||||
* добавлено ОПОВЕЩЕНИЕ. Раньше скрипт каждые 5 минут исправно записывал
|
||||
в базу статус CRITICAL, но никого не уведомлял — поэтому переполнение
|
||||
пула 14.10.2026 прошло незамеченным, хотя данные о нём были всё время
|
||||
(последняя запись: data_percent 80.66, vg_free 1.00, status CRITICAL).
|
||||
* оповещение идёт через бота алертов @Buryans_AlertBot — в соответствии
|
||||
с разделением ролей в docs/BOTS.md (аварии и мониторинг).
|
||||
* добавлена защита от повторов: один статус — не чаще раза в ALERT_COOLDOWN.
|
||||
* добавлено сообщение о восстановлении, когда статус возвращается в норму.
|
||||
* пороги можно переопределить переменными THRESH_WARNING / THRESH_CRITICAL —
|
||||
это нужно для проверки оповещения без реальной аварии.
|
||||
|
||||
Запуск: PGPASSWORD=... python3 check_thin_pool.py
|
||||
Проверка оповещения: THRESH_CRITICAL=0 PGPASSWORD=... python3 check_thin_pool.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
|
||||
import psycopg2
|
||||
|
||||
PG_CONFIG = {
|
||||
"host": os.environ.get("PG_HOST", "192.168.1.151"),
|
||||
"port": int(os.environ.get("PG_PORT", "5432")),
|
||||
"database": os.environ.get("PG_DB", "buryan_monitoring"),
|
||||
"user": os.environ.get("PG_USER", "buryan_monitor"),
|
||||
"password": os.environ.get("PGPASSWORD"),
|
||||
}
|
||||
|
||||
THRESHOLDS = {
|
||||
"warning": float(os.environ.get("THRESH_WARNING", "70")),
|
||||
"critical": float(os.environ.get("THRESH_CRITICAL", "80")),
|
||||
}
|
||||
|
||||
LVS = "/sbin/lvs"
|
||||
VGS = "/sbin/vgs"
|
||||
|
||||
TOKENS_FILE = os.environ.get("TOKENS_FILE", "/root/bots-tokens.env")
|
||||
TG_PROXY = os.environ.get("TG_PROXY", "http://192.168.1.10:10888")
|
||||
ALERT_CHAT_ID = os.environ.get("ALERT_CHAT_ID", "273442500")
|
||||
ALERT_STATE = os.environ.get("ALERT_STATE", "/var/lib/buryan-monitor/last-alert.json")
|
||||
ALERT_COOLDOWN = int(os.environ.get("ALERT_COOLDOWN", "3600"))
|
||||
|
||||
|
||||
def run(cmd):
|
||||
result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=30)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"Command failed: {cmd}\n{result.stderr}")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def collect_metrics():
|
||||
pool_out = run(
|
||||
f"{LVS} --noheadings --nosuffix --units g "
|
||||
f"-o lv_name,vg_name,data_percent,metadata_percent,lv_size "
|
||||
f"| grep -E '^[[:space:]]*data[[:space:]]+pve'"
|
||||
)
|
||||
parts = pool_out.split()
|
||||
if len(parts) < 5:
|
||||
raise RuntimeError(f"Cannot parse lvs: {pool_out}")
|
||||
pool_name, vg_name, data_pct, meta_pct, data_size = parts[:5]
|
||||
|
||||
meta_size = run(
|
||||
f"{LVS} --noheadings --nosuffix --units g -o lv_size pve/data_tmeta 2>/dev/null || echo 0"
|
||||
).strip()
|
||||
vg_free = run(f"{VGS} --noheadings --nosuffix --units g -o vg_free {vg_name}").strip()
|
||||
|
||||
vms_out = run(
|
||||
f"{LVS} --noheadings --nosuffix --units g -o lv_name,data_percent "
|
||||
f"| grep 'vm-' | sort -k2 -nr | head -5"
|
||||
)
|
||||
top_vms = []
|
||||
for line in vms_out.splitlines():
|
||||
p = line.split()
|
||||
if len(p) >= 2:
|
||||
try:
|
||||
top_vms.append({"vm": p[0], "data_pct": float(p[1])})
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
vm_count = int(run(f"{LVS} --noheadings | grep -c 'vm-' || echo 0"))
|
||||
|
||||
data_pct_f = float(data_pct)
|
||||
if data_pct_f > THRESHOLDS["critical"]:
|
||||
status = "CRITICAL"
|
||||
elif data_pct_f > THRESHOLDS["warning"]:
|
||||
status = "WARNING"
|
||||
else:
|
||||
status = "OK"
|
||||
|
||||
return {
|
||||
"pool_name": pool_name,
|
||||
"vg_name": vg_name,
|
||||
"data_size_gib": float(data_size),
|
||||
"data_percent": data_pct_f,
|
||||
"metadata_size_gib": float(meta_size),
|
||||
"metadata_percent": float(meta_pct),
|
||||
"vg_free_gib": float(vg_free),
|
||||
"vm_count": vm_count,
|
||||
"top_vms": top_vms,
|
||||
"status": status,
|
||||
}
|
||||
|
||||
|
||||
def save_to_db(metrics):
|
||||
with psycopg2.connect(**PG_CONFIG) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO thin_pool_metrics
|
||||
(pool_name, vg_name, data_size_gib, data_percent, metadata_size_gib,
|
||||
metadata_percent, vg_free_gib, vm_count, top_vms, status)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
""",
|
||||
(
|
||||
metrics["pool_name"],
|
||||
metrics["vg_name"],
|
||||
metrics["data_size_gib"],
|
||||
metrics["data_percent"],
|
||||
metrics["metadata_size_gib"],
|
||||
metrics["metadata_percent"],
|
||||
metrics["vg_free_gib"],
|
||||
metrics["vm_count"],
|
||||
json.dumps(metrics["top_vms"]),
|
||||
metrics["status"],
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
# --- Оповещение -------------------------------------------------------------
|
||||
def alert_token():
|
||||
try:
|
||||
with open(TOKENS_FILE, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
key, _, value = line.strip().partition("=")
|
||||
if key == "ALERT_TOKEN":
|
||||
return value
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def send_alert(text):
|
||||
token = alert_token()
|
||||
if not token:
|
||||
print(f"нет токена бота алертов в {TOKENS_FILE} — уведомление не отправлено", file=sys.stderr)
|
||||
return False
|
||||
|
||||
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
||||
payload = urllib.parse.urlencode(
|
||||
{"chat_id": ALERT_CHAT_ID, "text": text, "parse_mode": "HTML"}
|
||||
).encode()
|
||||
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({"http": TG_PROXY, "https": TG_PROXY})
|
||||
)
|
||||
try:
|
||||
with opener.open(url, data=payload, timeout=25) as resp:
|
||||
return resp.status == 200
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"не удалось отправить уведомление: {exc}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
|
||||
def read_alert_state():
|
||||
try:
|
||||
with open(ALERT_STATE, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
|
||||
|
||||
def write_alert_state(status):
|
||||
os.makedirs(os.path.dirname(ALERT_STATE), exist_ok=True)
|
||||
with open(ALERT_STATE, "w", encoding="utf-8") as f:
|
||||
json.dump({"status": status, "ts": time.time()}, f)
|
||||
|
||||
|
||||
def should_alert(status):
|
||||
state = read_alert_state()
|
||||
if state.get("status") == status and time.time() - state.get("ts", 0) < ALERT_COOLDOWN:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def format_alert(metrics, recovered=False):
|
||||
pct = metrics["data_percent"]
|
||||
free = metrics["vg_free_gib"]
|
||||
if recovered:
|
||||
return (
|
||||
"✅ <b>Пул томов PVE в норме</b>\n\n"
|
||||
f"Занято: {pct}%\nСвободно в группе томов: {free} ГБ"
|
||||
)
|
||||
|
||||
lines = [
|
||||
"🔴 <b>Пул томов PVE переполнен</b>",
|
||||
"",
|
||||
f"Занято: <b>{pct}%</b>",
|
||||
f"Свободно в группе томов: <b>{free} ГБ</b>",
|
||||
]
|
||||
if metrics["top_vms"]:
|
||||
lines.append("")
|
||||
lines.append("Самые заполненные:")
|
||||
for item in metrics["top_vms"][:5]:
|
||||
lines.append(f"• {item['vm']} — {item['data_pct']}%")
|
||||
lines += [
|
||||
"",
|
||||
"При 100% данные контейнеров и виртуальных машин повреждаются.",
|
||||
"Что делать: проверить /mnt/backups/dump и запустить "
|
||||
"<code>/opt/buryan-hub/scripts/vzdump-rotate.sh</code>, затем "
|
||||
"<code>fstrim -v /mnt/backups</code>.",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def maybe_alert(metrics):
|
||||
status = metrics["status"]
|
||||
previous = read_alert_state().get("status")
|
||||
|
||||
if status == "OK":
|
||||
if previous and previous != "OK":
|
||||
send_alert(format_alert(metrics, recovered=True))
|
||||
write_alert_state("OK")
|
||||
return
|
||||
|
||||
if should_alert(status):
|
||||
send_alert(format_alert(metrics))
|
||||
write_alert_state(status)
|
||||
|
||||
|
||||
def main():
|
||||
if not PG_CONFIG["password"]:
|
||||
print("PGPASSWORD не установлен", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
metrics = collect_metrics()
|
||||
save_to_db(metrics)
|
||||
maybe_alert(metrics)
|
||||
print(
|
||||
f"[{datetime.now().isoformat()}] Data={metrics['data_percent']}% "
|
||||
f"Meta={metrics['metadata_percent']}% VG_Free={metrics['vg_free_gib']}G "
|
||||
f"Status={metrics['status']}"
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"Ошибка: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user