106 lines
4.0 KiB
Python
106 lines
4.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Учения по восстановлению: проверяем, что бэкапы реально разворачиваются.
|
|
|
|
1. Целостность последнего архива конфигов (tar -tzf).
|
|
2. Восстановление дампа PostgreSQL в черновую базу + сверка таблиц.
|
|
3. Отчёт в Matrix и журнал.
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
BACKUPS = "/mnt/backups"
|
|
JOURNAL = "/var/lib/hub-agents/journal"
|
|
NOTIFY = "http://127.0.0.1:9120/notify"
|
|
PG = "PGPASSWORD='%s' psql -h 192.168.1.151 -U postgres" % open("/root/.pg-superuser-pass").read().strip()
|
|
|
|
|
|
def sh(cmd, timeout=180):
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
|
return r.returncode, (r.stdout or ""), (r.stderr or "")
|
|
|
|
|
|
def check_configs():
|
|
files = sorted(glob.glob(os.path.join(BACKUPS, "configs", "configs-*.tar.gz")))
|
|
if not files:
|
|
return False, "нет архивов конфигов"
|
|
latest = files[-1]
|
|
rc, out, err = sh(f"tar tzf '{latest}' | wc -l")
|
|
if rc != 0:
|
|
return False, f"архив повреждён: {err[:120]}"
|
|
n = (out or "0").strip()
|
|
age_h = (time.time() - os.path.getmtime(latest)) / 3600
|
|
return True, f"{os.path.basename(latest)}: {n} файлов, возраст {age_h:.1f} ч"
|
|
|
|
|
|
def check_database():
|
|
dumps = sorted(glob.glob(os.path.join(BACKUPS, "*.sql.gz")))
|
|
if not dumps:
|
|
dumps = sorted(glob.glob(os.path.join(BACKUPS, "**", "*.sql.gz"), recursive=True))
|
|
if not dumps:
|
|
return False, "нет дампов .sql.gz"
|
|
prefs = ("n8n", "authentik", "gitea", "hub")
|
|
src = None
|
|
for pref in prefs:
|
|
for d in reversed(dumps):
|
|
if pref in os.path.basename(d).lower():
|
|
src = d
|
|
break
|
|
if src:
|
|
break
|
|
if src is None:
|
|
src = dumps[-1]
|
|
scratch = "drill_restore_check"
|
|
sh(f"{PG} -c \"drop database if exists {scratch}\"")
|
|
rc, _, err = sh(f"{PG} -c \"create database {scratch}\"")
|
|
if rc != 0:
|
|
return False, f"не создалась черновая база: {err[:120]}"
|
|
rc, _, err = sh(f"gunzip -c '{src}' | {PG} -d {scratch} >/dev/null 2>&1")
|
|
rc2, out, _ = sh(f"{PG} -d {scratch} -t -A -c "
|
|
"\"select count(*) from information_schema.tables "
|
|
"where table_schema='public'\"")
|
|
sh(f"{PG} -c \"drop database if exists {scratch}\"")
|
|
tables = (out or "0").strip()
|
|
if rc != 0:
|
|
return False, f"дамп не развернулся: {src}"
|
|
if tables == "0":
|
|
return False, f"{os.path.basename(src)}: 0 таблиц — дамп неполный"
|
|
return True, f"{os.path.basename(src)}: {tables} таблиц"
|
|
|
|
|
|
def notify(title, text):
|
|
body = json.dumps({"title": title, "message": text}).encode()
|
|
req = urllib.request.Request(NOTIFY, data=body, method="POST",
|
|
headers={"Content-Type": "application/json"})
|
|
try:
|
|
urllib.request.urlopen(req, timeout=15).read()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def main():
|
|
ok_c, msg_c = check_configs()
|
|
ok_d, msg_d = check_database()
|
|
ok = ok_c and ok_d
|
|
text = (f"{'✅' if ok else '⚠️'} Учения восстановления\n"
|
|
f"Конфиги: {'ok' if ok_c else 'ПРОБЛЕМА'} — {msg_c}\n"
|
|
f"База: {'ok' if ok_d else 'ПРОБЛЕМА'} — {msg_d}")
|
|
os.makedirs(JOURNAL, exist_ok=True)
|
|
rec = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "ok": ok,
|
|
"configs": {"ok": ok_c, "detail": msg_c},
|
|
"database": {"ok": ok_d, "detail": msg_d}}
|
|
with open(os.path.join(JOURNAL, f"{time.strftime('%Y-%m-%d')}-drill.jsonl"), "a",
|
|
encoding="utf-8") as f:
|
|
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
print(text)
|
|
if not ok:
|
|
notify("Учения восстановления: проблемы", text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|