#!/usr/bin/env python3 """Предохранители Hermes: single-flight, cooldown, circuit breaker, budget, dry-run, kill-switch. Использование из кода: import guards ok, why = guards.allow("restart-failed-ct") if ok: try: ... действие ... guards.record("restart-failed-ct", True) except Exception: guards.record("restart-failed-ct", False) CLI для проверки: guards.py check guards.py record ok|fail guards.py status guards.py selftest """ import fcntl import json import os import sys import time import uuid from datetime import datetime POLICY = "/root/buryan-infra/ops/hermes/policy.yaml" STATE = "/var/lib/hermes/guards.json" LOCKDIR = "/var/lib/hermes/locks" JOURNAL = "/var/log/hermes/actions.jsonl" KILL = "/etc/hermes/KILL" def _policy(): import yaml try: return yaml.safe_load(open(POLICY, encoding="utf-8")) or {} except Exception: return {} def _defaults(): p = _policy() return p.get("defaults") or {} def _parse_dur(s, default=600): if isinstance(s, (int, float)): return float(s) if not s: return default m = {"s": 1, "m": 60, "h": 3600, "d": 86400} try: return float(str(s)[:-1]) * m.get(str(s)[-1], 60) except Exception: return default def _load(): try: with open(STATE, encoding="utf-8") as f: return json.load(f) except Exception: return {"rules": {}, "budget": {}} def _save(st): os.makedirs(os.path.dirname(STATE), exist_ok=True) tmp = STATE + ".tmp" with open(tmp, "w", encoding="utf-8") as f: json.dump(st, f, ensure_ascii=False, indent=1) os.replace(tmp, STATE) def _lock(rule): """single-flight: эксклюзивный flock на правило""" os.makedirs(LOCKDIR, exist_ok=True) path = os.path.join(LOCKDIR, rule + ".lock") fh = open(path, "w") try: fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) return fh except OSError: fh.close() return None def _journal(rec): os.makedirs(os.path.dirname(JOURNAL), exist_ok=True) rec["ts"] = datetime.now().isoformat(timespec="seconds") with open(JOURNAL, "a", encoding="utf-8") as f: f.write(json.dumps(rec, ensure_ascii=False) + "\n") def allow(rule, dry_run=None): """Можно ли запускать правило. Возвращает (bool, причина).""" d = _defaults() st = _load() r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0, "blocked": False, "total": 0}) if os.path.exists(KILL): return False, "kill-switch активен (%s)" % KILL if r.get("blocked"): return False, "circuit breaker: правило заблокировано до ручного сброса" now = time.time() cooldown = _parse_dur(d.get("cooldown"), 600) if r.get("last_run") and now - r["last_run"] < cooldown: left = int(cooldown - (now - r["last_run"])) return False, "cooldown: ещё %d с" % left budget = int(d.get("budget_per_hour", 20)) hour = int(now // 3600) b = st["budget"].setdefault(str(hour), 0) if b >= budget: return False, "budget: исчерпано %d действий в час" % budget # очистка старых корзин бюджета for k in list(st["budget"].keys()): if int(k) < hour - 2: del st["budget"][k] return True, "ok (dry_run=%s)" % (d.get("dry_run") if dry_run is None else dry_run) def begin(rule): """Атомарно: проверить политику + взять lock + отметить запуск.""" ok, why = allow(rule) if not ok: _journal({"rule": rule, "event": "denied", "reason": why}) return None, why lock = _lock(rule) if lock is None: _journal({"rule": rule, "event": "denied", "reason": "single-flight: уже выполняется"}) return None, "single-flight: задача уже выполняется" st = _load() r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0, "blocked": False, "total": 0}) r["last_run"] = time.time() r["total"] = r.get("total", 0) + 1 hour = str(int(time.time() // 3600)) st["budget"][hour] = st["budget"].get(hour, 0) + 1 _save(st) _journal({"rule": rule, "event": "start", "cid": uuid.uuid4().hex[:12]}) return lock, "started" def finish(rule, lock, ok, detail=""): """Зафиксировать результат: статистика, circuit breaker, журнал.""" d = _defaults() st = _load() r = st["rules"].setdefault(rule, {"last_run": 0, "fails": 0, "successes": 0, "blocked": False, "total": 0}) if ok: r["successes"] = r.get("successes", 0) + 1 r["fails"] = 0 else: r["fails"] = r.get("fails", 0) + 1 if r["fails"] >= int(d.get("circuit_breaker", 3)): r["blocked"] = True _save(st) _journal({"rule": rule, "event": "done" if ok else "failed", "successes": r["successes"], "fails": r["fails"], "blocked": r["blocked"], "detail": str(detail)[:300]}) if lock: try: fcntl.flock(lock, fcntl.LOCK_UN) lock.close() except Exception: pass return r def reset(rule=None): st = _load() if rule: st["rules"].pop(rule, None) else: st["rules"] = {} st["budget"] = {} _save(st) def status(): st = _load() print("dry_run:", _defaults().get("dry_run"), "| budget/h:", _defaults().get("budget_per_hour")) print("kill-switch:", "АКТИВЕН" if os.path.exists(KILL) else "нет") print("правил в состоянии:", len(st["rules"])) for k, v in sorted(st["rules"].items()): print(" %-28s total=%-3s ok=%-3s fail=%-2s blocked=%s" % ( k, v.get("total", 0), v.get("successes", 0), v.get("fails", 0), v.get("blocked"))) def selftest(): """Проверка всех предохранителей на временном правиле.""" rule = "selftest-guards" reset(rule) fails = [] # 1. обычный запуск lock, why = begin(rule) if lock is None: fails.append("1: не разрешил первый запуск: " + why) else: finish(rule, lock, True) # 2. cooldown: сразу второй раз нельзя lock2, why2 = begin(rule) if lock2 is not None: fails.append("2: cooldown не сработал") finish(rule, lock2, True) else: print(" cooldown ок:", why2) # 3. single-flight: обходим cooldown, берём два lock подряд reset(rule) l1, _ = begin(rule) if l1 is None: fails.append("3: не взял lock") else: # второй begin должен упасть на cooldown; проверим сам lock напрямую l2 = _lock(rule) if l2 is not None: fails.append("3: single-flight не сработал (второй lock получен)") l2.close() else: print(" single-flight ок") finish(rule, l1, True) # 4. circuit breaker: 3 провала reset(rule) for i in range(3): l, _ = begin(rule) if l is None: # обходим cooldown вручную для теста st = _load(); st["rules"][rule]["last_run"] = 0; _save(st) l, _ = begin(rule) finish(rule, l, False) st = _load() if not st["rules"][rule].get("blocked"): fails.append("4: circuit breaker не сработал") else: print(" circuit breaker ок") # 5. kill-switch reset(rule) os.makedirs(os.path.dirname(KILL), exist_ok=True) open(KILL, "w").close() ok, why = allow(rule) os.remove(KILL) if ok: fails.append("5: kill-switch не сработал") else: print(" kill-switch ок:", why) reset(rule) if fails: print("\nПРОВАЛЫ:") for f in fails: print(" ✗", f) return 1 print("\nвсе предохранители работают ✓") return 0 if __name__ == "__main__": if len(sys.argv) < 2: print(__doc__) sys.exit(1) cmd = sys.argv[1] if cmd == "check": ok, why = allow(sys.argv[2]) print("МОЖНО" if ok else "НЕЛЬЗЯ", "—", why) sys.exit(0 if ok else 2) if cmd == "record": ok = sys.argv[3] == "ok" r = _load()["rules"].get(sys.argv[2], {}) print("записано:", sys.argv[2], "ok" if ok else "fail") if cmd == "status": status() if cmd == "reset": reset(sys.argv[2] if len(sys.argv) > 2 else None) print("сброшено") if cmd == "selftest": sys.exit(selftest())