180 lines
5.8 KiB
Python
Executable File
180 lines
5.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Смоук платформы Buryans Hub: то, что ломается молча.
|
|
|
|
Проверяет ровно те механизмы, которые переделывались 10.11.2026:
|
|
ядро, шина, Postgres (hermes/rag/hub_state), поиск по документации,
|
|
шлюз уведомлений, боты, реестр автоматизаций.
|
|
|
|
Запуск: python3 ops/scripts/smoke_hub.py
|
|
Код возврата 0 — всё живо; 1 — есть провал (с названием проверки).
|
|
"""
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
HUB = "http://127.0.0.1:9120"
|
|
HERMES = "http://127.0.0.1:9130"
|
|
BOTS = "http://127.0.0.1:9110"
|
|
REGISTRY = "/root/buryan-infra/ops/automations/registry.yaml"
|
|
QUEUE = "/var/lib/hub-notify/queue.json"
|
|
|
|
FAILS = []
|
|
OKS = []
|
|
|
|
|
|
def ok(name, detail=""):
|
|
OKS.append(name)
|
|
print(" ✓ %-38s %s" % (name, detail))
|
|
|
|
|
|
def fail(name, detail=""):
|
|
FAILS.append(name)
|
|
print(" ✗ %-38s %s" % (name, detail))
|
|
|
|
|
|
def get(url, timeout=15):
|
|
with urllib.request.urlopen(url, timeout=timeout) as r:
|
|
return json.loads(r.read() or b"{}")
|
|
|
|
|
|
def check_core():
|
|
try:
|
|
d = get(HUB + "/health")
|
|
if d.get("ok"):
|
|
ok("hub-core /health")
|
|
else:
|
|
fail("hub-core /health", str(d)[:80])
|
|
except Exception as e:
|
|
fail("hub-core /health", str(e)[:80])
|
|
try:
|
|
d = get(HUB + "/status.json", 30)
|
|
total = len(d.get("services") or [])
|
|
down = len([s for s in (d.get("services") or []) if not s.get("up")])
|
|
if total >= 70 and down <= 3:
|
|
ok("службы", "%d всего, лежит %d" % (total, down))
|
|
else:
|
|
fail("службы", "всего %d, лежит %d" % (total, down))
|
|
except Exception as e:
|
|
fail("hub-core /status.json", str(e)[:80])
|
|
|
|
|
|
def check_hermes():
|
|
try:
|
|
d = get(HERMES + "/health")
|
|
if d.get("status") == "ok":
|
|
ok("hermes /health", "событий за час: %s" % d.get("events_1h"))
|
|
else:
|
|
fail("hermes /health", str(d)[:80])
|
|
except Exception as e:
|
|
fail("hermes /health", str(e)[:80])
|
|
try:
|
|
d = get(HERMES + "/events?limit=1")
|
|
if (d.get("count") or 0) >= 1:
|
|
ok("журнал событий")
|
|
else:
|
|
fail("журнал событий", "пусто")
|
|
except Exception as e:
|
|
fail("журнал событий", str(e)[:80])
|
|
|
|
|
|
def check_postgres():
|
|
try:
|
|
sys.path.insert(0, "/opt/hub-core")
|
|
import psycopg2
|
|
pw = open("/root/.pg-superuser-pass").read().strip()
|
|
conns = {
|
|
"hermes": "host=192.168.1.151 dbname=hermes user=postgres password=%s",
|
|
"rag": "host=192.168.1.151 dbname=rag user=postgres password=%s",
|
|
"hub_state": "host=192.168.1.151 dbname=hub_state user=postgres password=%s",
|
|
}
|
|
for name, dsn in conns.items():
|
|
c = psycopg2.connect(dsn % pw)
|
|
cur = c.cursor()
|
|
cur.execute("select 1")
|
|
cur.fetchone()
|
|
c.close()
|
|
ok("Postgres: три базы")
|
|
except Exception as e:
|
|
fail("Postgres", str(e)[:100])
|
|
|
|
|
|
def check_rag():
|
|
try:
|
|
sys.path.insert(0, "/opt/hub-core")
|
|
import rag
|
|
hits = rag.find("единый шлюз уведомлений", 3)
|
|
if hits:
|
|
ok("поиск по документации", "%.2f %s" % (hits[0]["score"],
|
|
hits[0]["ref"].split("/")[-1][:30]))
|
|
else:
|
|
fail("поиск по документации", "0 результатов")
|
|
except Exception as e:
|
|
fail("поиск по документации", str(e)[:80])
|
|
|
|
|
|
def check_gateway():
|
|
try:
|
|
body = json.dumps({"title": "smoke-test", "message": "проверка шлюза",
|
|
"priority": "low"}).encode()
|
|
req = urllib.request.Request(HUB + "/notify", data=body, method="POST",
|
|
headers={"Content-Type": "application/json"})
|
|
d = json.loads(urllib.request.urlopen(req, timeout=15).read())
|
|
if d.get("ok"):
|
|
ok("шлюз /notify", d.get("action"))
|
|
else:
|
|
fail("шлюз /notify", str(d)[:80])
|
|
except Exception as e:
|
|
fail("шлюз /notify", str(e)[:80])
|
|
finally:
|
|
# убираем след проверки из очереди
|
|
try:
|
|
q = json.load(open(QUEUE, encoding="utf-8"))
|
|
q = [x for x in q if x.get("title") != "smoke-test"]
|
|
json.dump(q, open(QUEUE, "w", encoding="utf-8"), ensure_ascii=False)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def check_bots():
|
|
try:
|
|
d = get(BOTS + "/health")
|
|
if d.get("ok") and not d.get("token_failures"):
|
|
ok("боты", "%s модулей" % d.get("bots_count"))
|
|
else:
|
|
fail("боты", str(d)[:100])
|
|
except Exception as e:
|
|
fail("боты", str(e)[:80])
|
|
|
|
|
|
def check_registry():
|
|
try:
|
|
n = sum(1 for l in open(REGISTRY, encoding="utf-8") if l.startswith("- "))
|
|
if n >= 100:
|
|
ok("реестр автоматизаций", "%d записей" % n)
|
|
else:
|
|
fail("реестр автоматизаций", "%d записей" % n)
|
|
except Exception as e:
|
|
fail("реестр автоматизаций", str(e)[:80])
|
|
|
|
|
|
def main():
|
|
print("смоук платформы:")
|
|
check_core()
|
|
check_hermes()
|
|
check_postgres()
|
|
check_rag()
|
|
check_gateway()
|
|
check_bots()
|
|
check_registry()
|
|
print("итог: ок %d, провалов %d" % (len(OKS), len(FAILS)))
|
|
if FAILS:
|
|
print("провалы: " + ", ".join(FAILS))
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|