Files
buryan-infra/ops/scripts/deep_audit.py
T

165 lines
7.3 KiB
Python
Executable File
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Полный read-only аудит Buryans Hub. Ничего не меняет — только читает.
Запускается после того, как LLM-сбор закончит очередь (см. after-llm-audit.sh).
Отчёт: /root/buryan-infra/ops/audits/<дата>-deep-audit.md
"""
import os
import re
import subprocess
import sys
import time
from collections import Counter
OUTDIR = "/root/buryan-infra/ops/audits"
DATE = time.strftime("%Y-%m-%d")
OUT = os.path.join(OUTDIR, DATE + "-deep-audit.md")
SECTIONS = []
def sh(cmd, timeout=90):
try:
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
return (r.stdout or "") + (r.stderr or "")
except Exception as e:
return "ERR: %s" % str(e)[:80]
def add(title, body):
SECTIONS.append((title, body.rstrip()))
def host_health():
up = sh("uptime")
mem = sh("free -h | head -2")
disk = sh("df -h / /var/lib/vz 2>/dev/null | grep -v Filesystem")
load = sh("cat /proc/loadavg")
top = sh("ps -eo pcpu,pmem,comm --sort=-pcpu | head -6")
add("Хост: ресурсы", "```\n%s\n%s\n%s\nload: %s\n\ntop CPU:\n%s\n```"
% (up.strip(), mem.strip(), disk.strip(), load.strip(), top.strip()))
def services():
failed = sh("systemctl list-units --state=failed --no-pager --no-legend")
timers = sh("systemctl list-timers --no-pager --no-legend | head -12")
add("Сбои systemd", "```\n%s\n```" % (failed.strip() or "нет сбоев"))
add("Таймеры (первые 12)", "```\n%s\n```" % timers.strip())
def containers():
rows = []
for ct in sh("pct list | awk 'NR>1{print $1}'").split():
cnt = sh("pct exec %s -- sh -c 'docker ps -q 2>/dev/null | wc -l'" % ct, 30).strip()
if cnt and cnt.isdigit() and int(cnt) > 0:
names = sh("pct exec %s -- sh -c \"docker ps --format '{{.Names}}' 2>/dev/null | tr '\\n' ','\"" % ct, 30)
rows.append("CT%s: %s контейнеров — %s" % (ct, cnt, names.strip()[:120]))
add("Контейнеры по CT", "```\n%s\n```" % ("\n".join(rows) or "нет данных"))
def journal_errors():
counts = Counter()
for unit in sh("systemctl list-units --type=service --state=running --no-legend | awk '{print $1}'").split():
n = sh("journalctl -u %s --since '7 days ago' --no-pager -p err 2>/dev/null | wc -l" % unit, 30).strip()
if n.isdigit() and int(n) > 0:
counts[unit] = int(n)
top = "\n".join("%-40s %d" % (u, n) for u, n in counts.most_common(15))
add("Ошибки (prio err) за 7 дней по службам", "```\n%s\n```" % (top or "ошибок нет"))
def error_patterns():
raw = sh("journalctl --since '7 days ago' --no-pager -p err 2>/dev/null | head -4000")
pats = Counter()
for line in raw.splitlines():
m = re.sub(r"\d+", "N", line.split(" ", 5)[-1] if len(line.split(" ", 5)) > 5 else line)
pats[m.strip()[:110]] += 1
top = "\n".join("%4d %s" % (n, p) for p, n in pats.most_common(20))
add("Топ повторяющихся ошибок (7 дней)", "```\n%s\n```" % (top or "нет"))
def disk_hogs():
logs = sh("du -sh /var/log /var/lib/vz /root/.cache 2>/dev/null")
journal = sh("journalctl --disk-usage 2>/dev/null")
big = sh("du -ahx / 2>/dev/null | sort -rh | head -12")
add("Место: где занято", "```\n%s\n%s\n\nкрупнейшее:\n%s\n```"
% (logs.strip(), journal.strip(), big.strip()))
def backup_status():
b = sh("ls -la /var/backups 2>/dev/null | tail -8")
dz = sh("find / -maxdepth 3 -name '*.dump' -o -maxdepth 3 -name '*.sql.gz' 2>/dev/null | head -5")
add("Бэкапы: свежесть", "```\n%s\n\nдампы:\n%s\n```" % (b.strip(), dz.strip() or "не найдены"))
def network():
wg = sh("wg show 2>/dev/null | head -12")
ports = sh("ss -tlnp 2>/dev/null | awk '{print $4}' | sort -u | head -25")
dns = sh("cat /etc/resolv.conf | grep -v '^#' | head -5")
add("Сеть: VPN, порты, DNS", "```\n%s\n\nпорты:\n%s\n\nDNS:\n%s\n```"
% (wg.strip() or "wg нет", ports.strip(), dns.strip()))
def hygiene():
dup = sh("systemctl list-unit-files --type=service --no-pager --no-legend | awk '{print $1}'"
" | sed 's/@.*//' | sort | uniq -d | head -10")
cron = sh("crontab -l 2>/dev/null | grep -c '^[^#]'")
jobs = sh("grep -c '^[a-z0-9-]*:' /opt/hermes/jobs.yaml 2>/dev/null")
old = sh("find /root /opt -maxdepth 2 -name '*.bak*' -o -maxdepth 2 -name '*.old' 2>/dev/null | head -10")
pycache = sh("find /opt -name '__pycache__' -maxdepth 3 2>/dev/null | wc -l")
add("Гигиена: дубли, кроны, мусор", "```\nдубли unit-файлов:\n%s\n\n"
"crontab строк: %s | заданий hermes: %s\n\nстарые копии:\n%s\n\n__pycache__: %s\n```"
% (dup.strip() or "нет", cron.strip(), jobs.strip(),
old.strip() or "нет", pycache.strip()))
def security():
certs = sh("find /etc/letsencrypt/live -name fullchain.pem -maxdepth 2 2>/dev/null | while read f; do"
" e=$(openssl x509 -enddate -noout -in \"$f\" 2>/dev/null | cut -d= -f2); echo \"$f | $e\"; done | head -10")
fail = sh("journalctl --since '7 days ago' --no-pager 2>/dev/null | grep -ciE 'failed password|invalid user'")
add("Безопасность: сертификаты, попытки входа", "```\n%s\n\nнеудачных входов за 7 дней: %s\n```"
% (certs.strip() or "не найдено", (fail or "0").strip()))
def llm_state():
disp = sh("systemctl is-active llm-dispatcher")
log = sh("tail -5 /var/log/llm-dispatch.log 2>/dev/null")
prog = sh("ls /opt/work-hub/research/progress-*.json 2>/dev/null | wc -l")
add("LLM-сбор: состояние", "```\nдиспетчер: %s\nфайлов прогресса: %s\n\n%s\n```"
% (disp.strip(), prog.strip(), log.strip()))
def main():
os.makedirs(OUTDIR, exist_ok=True)
t0 = time.time()
for fn in (host_health, services, containers, journal_errors, error_patterns,
disk_hogs, backup_status, network, hygiene, security, llm_state):
try:
fn()
except Exception as e:
add(fn.__name__, "ОШИБКА аудита: %s" % str(e)[:120])
body = ["# Глубокий аудит Buryans Hub — %s" % DATE,
"",
"Только чтение. Ничего не менялось.",
"",
"Итог: разделов %d, собрано за %.0f с." % (len(SECTIONS), time.time() - t0),
""]
for title, text in SECTIONS:
body += ["## " + title, "", text, ""]
open(OUT, "w", encoding="utf-8").write("\n".join(body))
print("отчёт:", OUT, "| разделов:", len(SECTIONS), "| %.0f с" % (time.time() - t0))
# короткая сводка в уведомление
summary = []
for title, text in SECTIONS:
if title.startswith("Сбои systemd") and "нет сбоев" not in text:
summary.append("есть сбои служб")
if title.startswith("Ошибки") and "ошибок нет" not in text:
summary.append("ошибки в журнале")
if summary:
print("внимание:", ", ".join(summary))
if __name__ == "__main__":
main()