[HUB] Смоук платформы: 9 проверок того, что ломается молча (ядро, шина, три базы Postgres, поиск по документации, шлюз, боты, реестр). Встроен в pre-commit и в задание Hermes каждые 6 часов
This commit is contained in:
1 parent
73e81c22c1
commit
7309580bf7
2 files changed
+183
No files matched your search
@@ -11,6 +11,10 @@ $PY $W/test_engine.py > /tmp/pc_unit.log 2>&1 || { echo "ОШИБКА: юнит-
|
||||
echo "[pre-commit] целостность UI…"
|
||||
$PY $W/check_webui.py > /tmp/pc_ui.log 2>&1 || { echo "ОШИБКА: проверка UI"; tail -8 /tmp/pc_ui.log; exit 1; }
|
||||
|
||||
echo "[pre-commit] смоук платформы…"
|
||||
/usr/bin/python3 ops/scripts/smoke_hub.py > /tmp/pc_hub.log 2>&1 || {
|
||||
echo "ОШИБКА: смоук платформы"; tail -12 /tmp/pc_hub.log; exit 1; }
|
||||
|
||||
echo "[pre-commit] смоук сервисов…"
|
||||
if $PY $W/smoke_services.py > /tmp/pc_smoke.log 2>&1; then
|
||||
echo "[pre-commit] ок"
|
||||
|
||||
Executable
+179
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Смоук платформы Buryans Hub: то, что ломается молча.
|
||||
|
||||
Проверяет ровно те механизмы, которые переделывались 10.11.2026:
|
||||
ядро, шина, Postgres (hermes/rag/hub_state), поиск по документации,
|
||||
шлюз уведомлений, боты, реестр автоматизаций.
|
||||
|
||||
Запуск: python3 ops/scripts/smoke_hub.py
|
||||
Код возврата 0 — всё живо; 1 — есть провал (с названием проверки).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
HUB = "http://127.0.0.1:9120"
|
||||
HERMES = "http://127.0.0.1:9130"
|
||||
BOTS = "http://127.0.0.1:9110"
|
||||
REGISTRY = "/root/buryan-infra/ops/automations/registry.yaml"
|
||||
QUEUE = "/var/lib/hub-notify/queue.json"
|
||||
|
||||
FAILS = []
|
||||
OKS = []
|
||||
|
||||
|
||||
def ok(name, detail=""):
|
||||
OKS.append(name)
|
||||
print(" ✓ %-38s %s" % (name, detail))
|
||||
|
||||
|
||||
def fail(name, detail=""):
|
||||
FAILS.append(name)
|
||||
print(" ✗ %-38s %s" % (name, detail))
|
||||
|
||||
|
||||
def get(url, timeout=15):
|
||||
with urllib.request.urlopen(url, timeout=timeout) as r:
|
||||
return json.loads(r.read() or b"{}")
|
||||
|
||||
|
||||
def check_core():
|
||||
try:
|
||||
d = get(HUB + "/health")
|
||||
if d.get("ok"):
|
||||
ok("hub-core /health")
|
||||
else:
|
||||
fail("hub-core /health", str(d)[:80])
|
||||
except Exception as e:
|
||||
fail("hub-core /health", str(e)[:80])
|
||||
try:
|
||||
d = get(HUB + "/status.json", 30)
|
||||
total = len(d.get("services") or [])
|
||||
down = len([s for s in (d.get("services") or []) if not s.get("up")])
|
||||
if total >= 70 and down <= 3:
|
||||
ok("службы", "%d всего, лежит %d" % (total, down))
|
||||
else:
|
||||
fail("службы", "всего %d, лежит %d" % (total, down))
|
||||
except Exception as e:
|
||||
fail("hub-core /status.json", str(e)[:80])
|
||||
|
||||
|
||||
def check_hermes():
|
||||
try:
|
||||
d = get(HERMES + "/health")
|
||||
if d.get("status") == "ok":
|
||||
ok("hermes /health", "событий за час: %s" % d.get("events_1h"))
|
||||
else:
|
||||
fail("hermes /health", str(d)[:80])
|
||||
except Exception as e:
|
||||
fail("hermes /health", str(e)[:80])
|
||||
try:
|
||||
d = get(HERMES + "/events?limit=1")
|
||||
if (d.get("count") or 0) >= 1:
|
||||
ok("журнал событий")
|
||||
else:
|
||||
fail("журнал событий", "пусто")
|
||||
except Exception as e:
|
||||
fail("журнал событий", str(e)[:80])
|
||||
|
||||
|
||||
def check_postgres():
|
||||
try:
|
||||
sys.path.insert(0, "/opt/hub-core")
|
||||
import psycopg2
|
||||
pw = open("/root/.pg-superuser-pass").read().strip()
|
||||
conns = {
|
||||
"hermes": "host=192.168.1.151 dbname=hermes user=postgres password=%s",
|
||||
"rag": "host=192.168.1.151 dbname=rag user=postgres password=%s",
|
||||
"hub_state": "host=192.168.1.151 dbname=hub_state user=postgres password=%s",
|
||||
}
|
||||
for name, dsn in conns.items():
|
||||
c = psycopg2.connect(dsn % pw)
|
||||
cur = c.cursor()
|
||||
cur.execute("select 1")
|
||||
cur.fetchone()
|
||||
c.close()
|
||||
ok("Postgres: три базы")
|
||||
except Exception as e:
|
||||
fail("Postgres", str(e)[:100])
|
||||
|
||||
|
||||
def check_rag():
|
||||
try:
|
||||
sys.path.insert(0, "/opt/hub-core")
|
||||
import rag
|
||||
hits = rag.find("единый шлюз уведомлений", 3)
|
||||
if hits:
|
||||
ok("поиск по документации", "%.2f %s" % (hits[0]["score"],
|
||||
hits[0]["ref"].split("/")[-1][:30]))
|
||||
else:
|
||||
fail("поиск по документации", "0 результатов")
|
||||
except Exception as e:
|
||||
fail("поиск по документации", str(e)[:80])
|
||||
|
||||
|
||||
def check_gateway():
|
||||
try:
|
||||
body = json.dumps({"title": "smoke-test", "message": "проверка шлюза",
|
||||
"priority": "low"}).encode()
|
||||
req = urllib.request.Request(HUB + "/notify", data=body, method="POST",
|
||||
headers={"Content-Type": "application/json"})
|
||||
d = json.loads(urllib.request.urlopen(req, timeout=15).read())
|
||||
if d.get("ok"):
|
||||
ok("шлюз /notify", d.get("action"))
|
||||
else:
|
||||
fail("шлюз /notify", str(d)[:80])
|
||||
except Exception as e:
|
||||
fail("шлюз /notify", str(e)[:80])
|
||||
finally:
|
||||
# убираем след проверки из очереди
|
||||
try:
|
||||
q = json.load(open(QUEUE, encoding="utf-8"))
|
||||
q = [x for x in q if x.get("title") != "smoke-test"]
|
||||
json.dump(q, open(QUEUE, "w", encoding="utf-8"), ensure_ascii=False)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def check_bots():
|
||||
try:
|
||||
d = get(BOTS + "/health")
|
||||
if d.get("ok") and not d.get("token_failures"):
|
||||
ok("боты", "%s модулей" % d.get("bots_count"))
|
||||
else:
|
||||
fail("боты", str(d)[:100])
|
||||
except Exception as e:
|
||||
fail("боты", str(e)[:80])
|
||||
|
||||
|
||||
def check_registry():
|
||||
try:
|
||||
n = sum(1 for l in open(REGISTRY, encoding="utf-8") if l.startswith("- "))
|
||||
if n >= 100:
|
||||
ok("реестр автоматизаций", "%d записей" % n)
|
||||
else:
|
||||
fail("реестр автоматизаций", "%d записей" % n)
|
||||
except Exception as e:
|
||||
fail("реестр автоматизаций", str(e)[:80])
|
||||
|
||||
|
||||
def main():
|
||||
print("смоук платформы:")
|
||||
check_core()
|
||||
check_hermes()
|
||||
check_postgres()
|
||||
check_rag()
|
||||
check_gateway()
|
||||
check_bots()
|
||||
check_registry()
|
||||
print("итог: ок %d, провалов %d" % (len(OKS), len(FAILS)))
|
||||
if FAILS:
|
||||
print("провалы: " + ", ".join(FAILS))
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user