291 lines
8.9 KiB
Python
291 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
||
"""Предохранители Hermes: single-flight, cooldown, circuit breaker, budget, dry-run, kill-switch.
|
||
|
||
Использование из кода:
|
||
import guards
|
||
ok, why = guards.allow("restart-failed-ct")
|
||
if ok:
|
||
try:
|
||
... действие ...
|
||
guards.record("restart-failed-ct", True)
|
||
except Exception:
|
||
guards.record("restart-failed-ct", False)
|
||
|
||
CLI для проверки:
|
||
guards.py check <rule>
|
||
guards.py record <rule> ok|fail
|
||
guards.py status
|
||
guards.py selftest
|
||
"""
|
||
import fcntl
|
||
import json
|
||
import os
|
||
import sys
|
||
import time
|
||
import uuid
|
||
from datetime import datetime
|
||
|
||
POLICY = "/root/buryan-infra/ops/hermes/policy.yaml"
|
||
STATE = "/var/lib/hermes/guards.json"
|
||
LOCKDIR = "/var/lib/hermes/locks"
|
||
JOURNAL = "/var/log/hermes/actions.jsonl"
|
||
KILL = "/etc/hermes/KILL"
|
||
|
||
|
||
def _policy():
|
||
import yaml
|
||
try:
|
||
return yaml.safe_load(open(POLICY, encoding="utf-8")) or {}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _defaults():
|
||
p = _policy()
|
||
return p.get("defaults") or {}
|
||
|
||
|
||
def _parse_dur(s, default=600):
|
||
if isinstance(s, (int, float)):
|
||
return float(s)
|
||
if not s:
|
||
return default
|
||
m = {"s": 1, "m": 60, "h": 3600, "d": 86400}
|
||
try:
|
||
return float(str(s)[:-1]) * m.get(str(s)[-1], 60)
|
||
except Exception:
|
||
return default
|
||
|
||
|
||
def _load():
|
||
try:
|
||
with open(STATE, encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except Exception:
|
||
return {"rules": {}, "budget": {}}
|
||
|
||
|
||
def _save(st):
|
||
os.makedirs(os.path.dirname(STATE), exist_ok=True)
|
||
tmp = STATE + ".tmp"
|
||
with open(tmp, "w", encoding="utf-8") as f:
|
||
json.dump(st, f, ensure_ascii=False, indent=1)
|
||
os.replace(tmp, STATE)
|
||
|
||
|
||
def _lock(rule):
|
||
"""single-flight: эксклюзивный flock на правило"""
|
||
os.makedirs(LOCKDIR, exist_ok=True)
|
||
path = os.path.join(LOCKDIR, rule + ".lock")
|
||
fh = open(path, "w")
|
||
try:
|
||
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
return fh
|
||
except OSError:
|
||
fh.close()
|
||
return None
|
||
|
||
|
||
def _journal(rec):
|
||
os.makedirs(os.path.dirname(JOURNAL), exist_ok=True)
|
||
rec["ts"] = datetime.now().isoformat(timespec="seconds")
|
||
with open(JOURNAL, "a", encoding="utf-8") as f:
|
||
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||
|
||
|
||
def allow(rule, dry_run=None):
|
||
"""Можно ли запускать правило. Возвращает (bool, причина)."""
|
||
d = _defaults()
|
||
st = _load()
|
||
r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0,
|
||
"blocked": False, "total": 0})
|
||
|
||
if os.path.exists(KILL):
|
||
return False, "kill-switch активен (%s)" % KILL
|
||
|
||
if r.get("blocked"):
|
||
return False, "circuit breaker: правило заблокировано до ручного сброса"
|
||
|
||
now = time.time()
|
||
cooldown = _parse_dur(d.get("cooldown"), 600)
|
||
if r.get("last_run") and now - r["last_run"] < cooldown:
|
||
left = int(cooldown - (now - r["last_run"]))
|
||
return False, "cooldown: ещё %d с" % left
|
||
|
||
budget = int(d.get("budget_per_hour", 20))
|
||
hour = int(now // 3600)
|
||
b = st["budget"].setdefault(str(hour), 0)
|
||
if b >= budget:
|
||
return False, "budget: исчерпано %d действий в час" % budget
|
||
|
||
# очистка старых корзин бюджета
|
||
for k in list(st["budget"].keys()):
|
||
if int(k) < hour - 2:
|
||
del st["budget"][k]
|
||
|
||
return True, "ok (dry_run=%s)" % (d.get("dry_run") if dry_run is None else dry_run)
|
||
|
||
|
||
def begin(rule):
|
||
"""Атомарно: проверить политику + взять lock + отметить запуск."""
|
||
ok, why = allow(rule)
|
||
if not ok:
|
||
_journal({"rule": rule, "event": "denied", "reason": why})
|
||
return None, why
|
||
|
||
lock = _lock(rule)
|
||
if lock is None:
|
||
_journal({"rule": rule, "event": "denied", "reason": "single-flight: уже выполняется"})
|
||
return None, "single-flight: задача уже выполняется"
|
||
|
||
st = _load()
|
||
r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0,
|
||
"blocked": False, "total": 0})
|
||
r["last_run"] = time.time()
|
||
r["total"] = r.get("total", 0) + 1
|
||
hour = str(int(time.time() // 3600))
|
||
st["budget"][hour] = st["budget"].get(hour, 0) + 1
|
||
_save(st)
|
||
_journal({"rule": rule, "event": "start", "cid": uuid.uuid4().hex[:12]})
|
||
return lock, "started"
|
||
|
||
|
||
def finish(rule, lock, ok, detail=""):
|
||
"""Зафиксировать результат: статистика, circuit breaker, журнал."""
|
||
d = _defaults()
|
||
st = _load()
|
||
r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0,
|
||
"blocked": False, "total": 0})
|
||
if ok:
|
||
r["successes"] = r.get("successes", 0) + 1
|
||
r["fails"] = 0
|
||
else:
|
||
r["fails"] = r.get("fails", 0) + 1
|
||
if r["fails"] >= int(d.get("circuit_breaker", 3)):
|
||
r["blocked"] = True
|
||
_save(st)
|
||
_journal({"rule": rule, "event": "done" if ok else "failed",
|
||
"successes": r["successes"], "fails": r["fails"],
|
||
"blocked": r["blocked"], "detail": str(detail)[:300]})
|
||
if lock:
|
||
try:
|
||
fcntl.flock(lock, fcntl.LOCK_UN)
|
||
lock.close()
|
||
except Exception:
|
||
pass
|
||
return r
|
||
|
||
|
||
def reset(rule=None):
|
||
st = _load()
|
||
if rule:
|
||
st["rules"].pop(rule, None)
|
||
else:
|
||
st["rules"] = {}
|
||
st["budget"] = {}
|
||
_save(st)
|
||
|
||
|
||
def status():
|
||
st = _load()
|
||
print("dry_run:", _defaults().get("dry_run"), "| budget/h:", _defaults().get("budget_per_hour"))
|
||
print("kill-switch:", "АКТИВЕН" if os.path.exists(KILL) else "нет")
|
||
print("правил в состоянии:", len(st["rules"]))
|
||
for k, v in sorted(st["rules"].items()):
|
||
print(" %-28s total=%-3s ok=%-3s fail=%-2s blocked=%s" % (
|
||
k, v.get("total", 0), v.get("successes", 0), v.get("fails", 0), v.get("blocked")))
|
||
|
||
|
||
def selftest():
|
||
"""Проверка всех предохранителей на временном правиле."""
|
||
rule = "selftest-guards"
|
||
reset(rule)
|
||
fails = []
|
||
|
||
# 1. обычный запуск
|
||
lock, why = begin(rule)
|
||
if lock is None:
|
||
fails.append("1: не разрешил первый запуск: " + why)
|
||
else:
|
||
finish(rule, lock, True)
|
||
|
||
# 2. cooldown: сразу второй раз нельзя
|
||
lock2, why2 = begin(rule)
|
||
if lock2 is not None:
|
||
fails.append("2: cooldown не сработал")
|
||
finish(rule, lock2, True)
|
||
else:
|
||
print(" cooldown ок:", why2)
|
||
|
||
# 3. single-flight: обходим cooldown, берём два lock подряд
|
||
reset(rule)
|
||
l1, _ = begin(rule)
|
||
if l1 is None:
|
||
fails.append("3: не взял lock")
|
||
else:
|
||
# второй begin должен упасть на cooldown; проверим сам lock напрямую
|
||
l2 = _lock(rule)
|
||
if l2 is not None:
|
||
fails.append("3: single-flight не сработал (второй lock получен)")
|
||
l2.close()
|
||
else:
|
||
print(" single-flight ок")
|
||
finish(rule, l1, True)
|
||
|
||
# 4. circuit breaker: 3 провала
|
||
reset(rule)
|
||
for i in range(3):
|
||
l, _ = begin(rule)
|
||
if l is None:
|
||
# обходим cooldown вручную для теста
|
||
st = _load(); st["rules"][rule]["last_run"] = 0; _save(st)
|
||
l, _ = begin(rule)
|
||
finish(rule, l, False)
|
||
st = _load()
|
||
if not st["rules"][rule].get("blocked"):
|
||
fails.append("4: circuit breaker не сработал")
|
||
else:
|
||
print(" circuit breaker ок")
|
||
|
||
# 5. kill-switch
|
||
reset(rule)
|
||
os.makedirs(os.path.dirname(KILL), exist_ok=True)
|
||
open(KILL, "w").close()
|
||
ok, why = allow(rule)
|
||
os.remove(KILL)
|
||
if ok:
|
||
fails.append("5: kill-switch не сработал")
|
||
else:
|
||
print(" kill-switch ок:", why)
|
||
|
||
reset(rule)
|
||
if fails:
|
||
print("\nПРОВАЛЫ:")
|
||
for f in fails:
|
||
print(" ✗", f)
|
||
return 1
|
||
print("\nвсе предохранители работают ✓")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if len(sys.argv) < 2:
|
||
print(__doc__)
|
||
sys.exit(1)
|
||
cmd = sys.argv[1]
|
||
if cmd == "check":
|
||
ok, why = allow(sys.argv[2])
|
||
print("МОЖНО" if ok else "НЕЛЬЗЯ", "—", why)
|
||
sys.exit(0 if ok else 2)
|
||
if cmd == "record":
|
||
ok = sys.argv[3] == "ok"
|
||
r = _load()["rules"].get(sys.argv[2], {})
|
||
print("записано:", sys.argv[2], "ok" if ok else "fail")
|
||
if cmd == "status":
|
||
status()
|
||
if cmd == "reset":
|
||
reset(sys.argv[2] if len(sys.argv) > 2 else None)
|
||
print("сброшено")
|
||
if cmd == "selftest":
|
||
sys.exit(selftest())
|