Files
buryan-infra/ops/hermes/guards.py

291 lines
8.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Предохранители Hermes: single-flight, cooldown, circuit breaker, budget, dry-run, kill-switch.
Использование из кода:
import guards
ok, why = guards.allow("restart-failed-ct")
if ok:
try:
... действие ...
guards.record("restart-failed-ct", True)
except Exception:
guards.record("restart-failed-ct", False)
CLI для проверки:
guards.py check <rule>
guards.py record <rule> ok|fail
guards.py status
guards.py selftest
"""
import fcntl
import json
import os
import sys
import time
import uuid
from datetime import datetime
POLICY = "/root/buryan-infra/ops/hermes/policy.yaml"
STATE = "/var/lib/hermes/guards.json"
LOCKDIR = "/var/lib/hermes/locks"
JOURNAL = "/var/log/hermes/actions.jsonl"
KILL = "/etc/hermes/KILL"
def _policy():
import yaml
try:
return yaml.safe_load(open(POLICY, encoding="utf-8")) or {}
except Exception:
return {}
def _defaults():
p = _policy()
return p.get("defaults") or {}
def _parse_dur(s, default=600):
if isinstance(s, (int, float)):
return float(s)
if not s:
return default
m = {"s": 1, "m": 60, "h": 3600, "d": 86400}
try:
return float(str(s)[:-1]) * m.get(str(s)[-1], 60)
except Exception:
return default
def _load():
try:
with open(STATE, encoding="utf-8") as f:
return json.load(f)
except Exception:
return {"rules": {}, "budget": {}}
def _save(st):
os.makedirs(os.path.dirname(STATE), exist_ok=True)
tmp = STATE + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(st, f, ensure_ascii=False, indent=1)
os.replace(tmp, STATE)
def _lock(rule):
"""single-flight: эксклюзивный flock на правило"""
os.makedirs(LOCKDIR, exist_ok=True)
path = os.path.join(LOCKDIR, rule + ".lock")
fh = open(path, "w")
try:
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
return fh
except OSError:
fh.close()
return None
def _journal(rec):
os.makedirs(os.path.dirname(JOURNAL), exist_ok=True)
rec["ts"] = datetime.now().isoformat(timespec="seconds")
with open(JOURNAL, "a", encoding="utf-8") as f:
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
def allow(rule, dry_run=None):
"""Можно ли запускать правило. Возвращает (bool, причина)."""
d = _defaults()
st = _load()
r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0,
"blocked": False, "total": 0})
if os.path.exists(KILL):
return False, "kill-switch активен (%s)" % KILL
if r.get("blocked"):
return False, "circuit breaker: правило заблокировано до ручного сброса"
now = time.time()
cooldown = _parse_dur(d.get("cooldown"), 600)
if r.get("last_run") and now - r["last_run"] < cooldown:
left = int(cooldown - (now - r["last_run"]))
return False, "cooldown: ещё %d с" % left
budget = int(d.get("budget_per_hour", 20))
hour = int(now // 3600)
b = st["budget"].setdefault(str(hour), 0)
if b >= budget:
return False, "budget: исчерпано %d действий в час" % budget
# очистка старых корзин бюджета
for k in list(st["budget"].keys()):
if int(k) < hour - 2:
del st["budget"][k]
return True, "ok (dry_run=%s)" % (d.get("dry_run") if dry_run is None else dry_run)
def begin(rule):
"""Атомарно: проверить политику + взять lock + отметить запуск."""
ok, why = allow(rule)
if not ok:
_journal({"rule": rule, "event": "denied", "reason": why})
return None, why
lock = _lock(rule)
if lock is None:
_journal({"rule": rule, "event": "denied", "reason": "single-flight: уже выполняется"})
return None, "single-flight: задача уже выполняется"
st = _load()
r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0,
"blocked": False, "total": 0})
r["last_run"] = time.time()
r["total"] = r.get("total", 0) + 1
hour = str(int(time.time() // 3600))
st["budget"][hour] = st["budget"].get(hour, 0) + 1
_save(st)
_journal({"rule": rule, "event": "start", "cid": uuid.uuid4().hex[:12]})
return lock, "started"
def finish(rule, lock, ok, detail=""):
"""Зафиксировать результат: статистика, circuit breaker, журнал."""
d = _defaults()
st = _load()
r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0,
"blocked": False, "total": 0})
if ok:
r["successes"] = r.get("successes", 0) + 1
r["fails"] = 0
else:
r["fails"] = r.get("fails", 0) + 1
if r["fails"] >= int(d.get("circuit_breaker", 3)):
r["blocked"] = True
_save(st)
_journal({"rule": rule, "event": "done" if ok else "failed",
"successes": r["successes"], "fails": r["fails"],
"blocked": r["blocked"], "detail": str(detail)[:300]})
if lock:
try:
fcntl.flock(lock, fcntl.LOCK_UN)
lock.close()
except Exception:
pass
return r
def reset(rule=None):
st = _load()
if rule:
st["rules"].pop(rule, None)
else:
st["rules"] = {}
st["budget"] = {}
_save(st)
def status():
st = _load()
print("dry_run:", _defaults().get("dry_run"), "| budget/h:", _defaults().get("budget_per_hour"))
print("kill-switch:", "АКТИВЕН" if os.path.exists(KILL) else "нет")
print("правил в состоянии:", len(st["rules"]))
for k, v in sorted(st["rules"].items()):
print(" %-28s total=%-3s ok=%-3s fail=%-2s blocked=%s" % (
k, v.get("total", 0), v.get("successes", 0), v.get("fails", 0), v.get("blocked")))
def selftest():
"""Проверка всех предохранителей на временном правиле."""
rule = "selftest-guards"
reset(rule)
fails = []
# 1. обычный запуск
lock, why = begin(rule)
if lock is None:
fails.append("1: не разрешил первый запуск: " + why)
else:
finish(rule, lock, True)
# 2. cooldown: сразу второй раз нельзя
lock2, why2 = begin(rule)
if lock2 is not None:
fails.append("2: cooldown не сработал")
finish(rule, lock2, True)
else:
print(" cooldown ок:", why2)
# 3. single-flight: обходим cooldown, берём два lock подряд
reset(rule)
l1, _ = begin(rule)
if l1 is None:
fails.append("3: не взял lock")
else:
# второй begin должен упасть на cooldown; проверим сам lock напрямую
l2 = _lock(rule)
if l2 is not None:
fails.append("3: single-flight не сработал (второй lock получен)")
l2.close()
else:
print(" single-flight ок")
finish(rule, l1, True)
# 4. circuit breaker: 3 провала
reset(rule)
for i in range(3):
l, _ = begin(rule)
if l is None:
# обходим cooldown вручную для теста
st = _load(); st["rules"][rule]["last_run"] = 0; _save(st)
l, _ = begin(rule)
finish(rule, l, False)
st = _load()
if not st["rules"][rule].get("blocked"):
fails.append("4: circuit breaker не сработал")
else:
print(" circuit breaker ок")
# 5. kill-switch
reset(rule)
os.makedirs(os.path.dirname(KILL), exist_ok=True)
open(KILL, "w").close()
ok, why = allow(rule)
os.remove(KILL)
if ok:
fails.append("5: kill-switch не сработал")
else:
print(" kill-switch ок:", why)
reset(rule)
if fails:
print("\nПРОВАЛЫ:")
for f in fails:
print(" ✗", f)
return 1
print("\nвсе предохранители работают ✓")
return 0
if __name__ == "__main__":
if len(sys.argv) < 2:
print(__doc__)
sys.exit(1)
cmd = sys.argv[1]
if cmd == "check":
ok, why = allow(sys.argv[2])
print("МОЖНО" if ok else "НЕЛЬЗЯ", "—", why)
sys.exit(0 if ok else 2)
if cmd == "record":
ok = sys.argv[3] == "ok"
r = _load()["rules"].get(sys.argv[2], {})
print("записано:", sys.argv[2], "ok" if ok else "fail")
if cmd == "status":
status()
if cmd == "reset":
reset(sys.argv[2] if len(sys.argv) > 2 else None)
print("сброшено")
if cmd == "selftest":
sys.exit(selftest())