165 lines
7.3 KiB
Python
Executable File
165 lines
7.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Полный read-only аудит Buryans Hub. Ничего не меняет — только читает.
|
||
|
||
Запускается после того, как LLM-сбор закончит очередь (см. after-llm-audit.sh).
|
||
Отчёт: /root/buryan-infra/ops/audits/<дата>-deep-audit.md
|
||
"""
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from collections import Counter
|
||
|
||
OUTDIR = "/root/buryan-infra/ops/audits"
|
||
DATE = time.strftime("%Y-%m-%d")
|
||
OUT = os.path.join(OUTDIR, DATE + "-deep-audit.md")
|
||
|
||
SECTIONS = []
|
||
|
||
|
||
def sh(cmd, timeout=90):
|
||
try:
|
||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
||
return (r.stdout or "") + (r.stderr or "")
|
||
except Exception as e:
|
||
return "ERR: %s" % str(e)[:80]
|
||
|
||
|
||
def add(title, body):
|
||
SECTIONS.append((title, body.rstrip()))
|
||
|
||
|
||
def host_health():
|
||
up = sh("uptime")
|
||
mem = sh("free -h | head -2")
|
||
disk = sh("df -h / /var/lib/vz 2>/dev/null | grep -v Filesystem")
|
||
load = sh("cat /proc/loadavg")
|
||
top = sh("ps -eo pcpu,pmem,comm --sort=-pcpu | head -6")
|
||
add("Хост: ресурсы", "```\n%s\n%s\n%s\nload: %s\n\ntop CPU:\n%s\n```"
|
||
% (up.strip(), mem.strip(), disk.strip(), load.strip(), top.strip()))
|
||
|
||
|
||
def services():
|
||
failed = sh("systemctl list-units --state=failed --no-pager --no-legend")
|
||
timers = sh("systemctl list-timers --no-pager --no-legend | head -12")
|
||
add("Сбои systemd", "```\n%s\n```" % (failed.strip() or "нет сбоев"))
|
||
add("Таймеры (первые 12)", "```\n%s\n```" % timers.strip())
|
||
|
||
|
||
def containers():
|
||
rows = []
|
||
for ct in sh("pct list | awk 'NR>1{print $1}'").split():
|
||
cnt = sh("pct exec %s -- sh -c 'docker ps -q 2>/dev/null | wc -l'" % ct, 30).strip()
|
||
if cnt and cnt.isdigit() and int(cnt) > 0:
|
||
names = sh("pct exec %s -- sh -c \"docker ps --format '{{.Names}}' 2>/dev/null | tr '\\n' ','\"" % ct, 30)
|
||
rows.append("CT%s: %s контейнеров — %s" % (ct, cnt, names.strip()[:120]))
|
||
add("Контейнеры по CT", "```\n%s\n```" % ("\n".join(rows) or "нет данных"))
|
||
|
||
|
||
def journal_errors():
|
||
counts = Counter()
|
||
for unit in sh("systemctl list-units --type=service --state=running --no-legend | awk '{print $1}'").split():
|
||
n = sh("journalctl -u %s --since '7 days ago' --no-pager -p err 2>/dev/null | wc -l" % unit, 30).strip()
|
||
if n.isdigit() and int(n) > 0:
|
||
counts[unit] = int(n)
|
||
top = "\n".join("%-40s %d" % (u, n) for u, n in counts.most_common(15))
|
||
add("Ошибки (prio err) за 7 дней по службам", "```\n%s\n```" % (top or "ошибок нет"))
|
||
|
||
|
||
def error_patterns():
|
||
raw = sh("journalctl --since '7 days ago' --no-pager -p err 2>/dev/null | head -4000")
|
||
pats = Counter()
|
||
for line in raw.splitlines():
|
||
m = re.sub(r"\d+", "N", line.split(" ", 5)[-1] if len(line.split(" ", 5)) > 5 else line)
|
||
pats[m.strip()[:110]] += 1
|
||
top = "\n".join("%4d %s" % (n, p) for p, n in pats.most_common(20))
|
||
add("Топ повторяющихся ошибок (7 дней)", "```\n%s\n```" % (top or "нет"))
|
||
|
||
|
||
def disk_hogs():
|
||
logs = sh("du -sh /var/log /var/lib/vz /root/.cache 2>/dev/null")
|
||
journal = sh("journalctl --disk-usage 2>/dev/null")
|
||
big = sh("du -ahx / 2>/dev/null | sort -rh | head -12")
|
||
add("Место: где занято", "```\n%s\n%s\n\nкрупнейшее:\n%s\n```"
|
||
% (logs.strip(), journal.strip(), big.strip()))
|
||
|
||
|
||
def backup_status():
|
||
b = sh("ls -la /var/backups 2>/dev/null | tail -8")
|
||
dz = sh("find / -maxdepth 3 -name '*.dump' -o -maxdepth 3 -name '*.sql.gz' 2>/dev/null | head -5")
|
||
add("Бэкапы: свежесть", "```\n%s\n\nдампы:\n%s\n```" % (b.strip(), dz.strip() or "не найдены"))
|
||
|
||
|
||
def network():
|
||
wg = sh("wg show 2>/dev/null | head -12")
|
||
ports = sh("ss -tlnp 2>/dev/null | awk '{print $4}' | sort -u | head -25")
|
||
dns = sh("cat /etc/resolv.conf | grep -v '^#' | head -5")
|
||
add("Сеть: VPN, порты, DNS", "```\n%s\n\nпорты:\n%s\n\nDNS:\n%s\n```"
|
||
% (wg.strip() or "wg нет", ports.strip(), dns.strip()))
|
||
|
||
|
||
def hygiene():
|
||
dup = sh("systemctl list-unit-files --type=service --no-pager --no-legend | awk '{print $1}'"
|
||
" | sed 's/@.*//' | sort | uniq -d | head -10")
|
||
cron = sh("crontab -l 2>/dev/null | grep -c '^[^#]'")
|
||
jobs = sh("grep -c '^[a-z0-9-]*:' /opt/hermes/jobs.yaml 2>/dev/null")
|
||
old = sh("find /root /opt -maxdepth 2 -name '*.bak*' -o -maxdepth 2 -name '*.old' 2>/dev/null | head -10")
|
||
pycache = sh("find /opt -name '__pycache__' -maxdepth 3 2>/dev/null | wc -l")
|
||
add("Гигиена: дубли, кроны, мусор", "```\nдубли unit-файлов:\n%s\n\n"
|
||
"crontab строк: %s | заданий hermes: %s\n\nстарые копии:\n%s\n\n__pycache__: %s\n```"
|
||
% (dup.strip() or "нет", cron.strip(), jobs.strip(),
|
||
old.strip() or "нет", pycache.strip()))
|
||
|
||
|
||
def security():
|
||
certs = sh("find /etc/letsencrypt/live -name fullchain.pem -maxdepth 2 2>/dev/null | while read f; do"
|
||
" e=$(openssl x509 -enddate -noout -in \"$f\" 2>/dev/null | cut -d= -f2); echo \"$f | $e\"; done | head -10")
|
||
fail = sh("journalctl --since '7 days ago' --no-pager 2>/dev/null | grep -ciE 'failed password|invalid user'")
|
||
add("Безопасность: сертификаты, попытки входа", "```\n%s\n\nнеудачных входов за 7 дней: %s\n```"
|
||
% (certs.strip() or "не найдено", (fail or "0").strip()))
|
||
|
||
|
||
def llm_state():
|
||
disp = sh("systemctl is-active llm-dispatcher")
|
||
log = sh("tail -5 /var/log/llm-dispatch.log 2>/dev/null")
|
||
prog = sh("ls /opt/work-hub/research/progress-*.json 2>/dev/null | wc -l")
|
||
add("LLM-сбор: состояние", "```\nдиспетчер: %s\nфайлов прогресса: %s\n\n%s\n```"
|
||
% (disp.strip(), prog.strip(), log.strip()))
|
||
|
||
|
||
def main():
|
||
os.makedirs(OUTDIR, exist_ok=True)
|
||
t0 = time.time()
|
||
for fn in (host_health, services, containers, journal_errors, error_patterns,
|
||
disk_hogs, backup_status, network, hygiene, security, llm_state):
|
||
try:
|
||
fn()
|
||
except Exception as e:
|
||
add(fn.__name__, "ОШИБКА аудита: %s" % str(e)[:120])
|
||
|
||
body = ["# Глубокий аудит Buryans Hub — %s" % DATE,
|
||
"",
|
||
"Только чтение. Ничего не менялось.",
|
||
"",
|
||
"Итог: разделов %d, собрано за %.0f с." % (len(SECTIONS), time.time() - t0),
|
||
""]
|
||
for title, text in SECTIONS:
|
||
body += ["## " + title, "", text, ""]
|
||
open(OUT, "w", encoding="utf-8").write("\n".join(body))
|
||
print("отчёт:", OUT, "| разделов:", len(SECTIONS), "| %.0f с" % (time.time() - t0))
|
||
|
||
# короткая сводка в уведомление
|
||
summary = []
|
||
for title, text in SECTIONS:
|
||
if title.startswith("Сбои systemd") and "нет сбоев" not in text:
|
||
summary.append("есть сбои служб")
|
||
if title.startswith("Ошибки") and "ошибок нет" not in text:
|
||
summary.append("ошибки в журнале")
|
||
if summary:
|
||
print("внимание:", ", ".join(summary))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|