Files
buryan-infra/ops/scripts/build_registry_v2.py

219 lines
8.5 KiB
Python
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Единый реестр инфраструктуры: хосты + сервисы + домены + зависимости + стоимость.
Заменяет три параллельных источника (services.yaml, topology.json, ledger.md)
одним файлом, из которого генерируются отчёты.
Выход: ops/infra/registry.yaml, ops/infra/README.md
"""
import json
import os
import re
import subprocess
from datetime import datetime
ROOT = "/root/buryan-infra"
OUT = ROOT + "/ops/infra"
def sh(cmd, timeout=90):
try:
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
return (r.stdout or "").strip()
except Exception:
return ""
def hosts():
out = []
for line in sh("pct list").splitlines()[1:]:
p = line.split()
if len(p) < 2:
continue
ct, status = p[0], p[1]
name = p[2] if len(p) > 2 else "?"
cfg = sh("cat /etc/pve/lxc/%s.conf 2>/dev/null" % ct, 15)
mem = re.search(r"^memory:\s*(\d+)", cfg, re.M)
out.append({"id": "ct%s" % ct, "vmid": int(ct), "type": "lxc", "name": name,
"status": status, "onboot": "onboot: 1" in cfg,
"ram_limit_mb": int(mem.group(1)) if mem else None})
for line in sh("qm list 2>/dev/null").splitlines()[1:]:
p = line.split()
if len(p) < 3:
continue
cfg = sh("cat /etc/pve/qemu-server/%s.conf 2>/dev/null" % p[0], 15)
mem = re.search(r"^memory:\s*(\d+)", cfg, re.M)
out.append({"id": "vm%s" % p[0], "vmid": int(p[0]), "type": "qemu", "name": p[1],
"status": p[2], "onboot": "onboot: 1" in cfg,
"ram_limit_mb": int(mem.group(1)) if mem else None})
return out
def services(hostlist):
out = []
for h in hostlist:
if h["status"] != "running":
continue
if h["type"] == "lxc":
cmd = "pct exec %s -- " % h["vmid"]
else:
cmd = ""
raw = sh(cmd + "docker ps --format '{{.Names}}|{{.Image}}|{{.Ports}}' 2>/dev/null", 60)
for ln in raw.splitlines():
f = ln.split("|")
if len(f) < 3:
continue
ports = re.findall(r"0\.0\.0\.0:(\d+)->", f[2])
out.append({"name": f[0], "host": h["id"], "image": f[1],
"ports": [int(x) for x in ports] or None,
"ip": "192.168.1.%s" % h["vmid"]})
return out
def _blocks(text):
"""Разбирает nginx-конфиг по балансу скобок: возвращает тексты server-блоков."""
blocks, depth, buf, in_server = [], 0, [], False
for line in text.splitlines():
if not in_server and re.match(r"\s*server\s*\{", line):
in_server, depth, buf = True, 0, []
if in_server:
buf.append(line)
depth += line.count("{") - line.count("}")
if depth <= 0:
blocks.append("\n".join(buf))
in_server = False
return blocks
def costs(hostlist):
"""RAM по контейнерам (docker stats, один вызов на хост)."""
out = {}
for h in hostlist:
if h["status"] != "running":
continue
if h["type"] == "lxc":
cmd = "pct exec %s -- " % h["vmid"]
else:
cmd = ""
raw = sh(cmd + "docker stats --no-stream --format '{{.Name}}|{{.MemUsage}}' 2>/dev/null", 90)
for ln in raw.splitlines():
f = ln.split("|")
if len(f) < 2:
continue
m = re.match(r"([\d.]+)([KMG]iB)", f[1].strip())
if m:
v = float(m.group(1))
mb = v / 1024 if m.group(2) == "KiB" else (v * 1024 if m.group(2) == "GiB" else v)
out[f[0]] = round(mb, 1)
return out
OAUTH_PORTS = set()
def collect_oauth_ports(services_list):
"""Порты хостов, на которых слушают oauth2-proxy (для детекта SSO)."""
global OAUTH_PORTS
for sv in services_list:
if sv["name"].startswith("oauth2-proxy") and sv.get("ports"):
OAUTH_PORTS.update(sv["ports"])
return OAUTH_PORTS
def domains():
"""Домены из nginx: server_name + proxy_pass (хост и VPS)."""
seen, out = set(), []
sources = [("host", sh("cat /etc/nginx/sites-enabled/* /etc/nginx/conf.d/* 2>/dev/null", 40))]
vps = sh("ssh -o ConnectTimeout=8 -o BatchMode=yes root@193.233.210.184 "
"'cat /etc/nginx/sites-enabled/* 2>/dev/null'", 40)
if vps:
sources.append(("vps", vps))
for where, text in sources:
for block in _blocks(text or ""):
names = re.findall(r"server_name\s+([^;]+);", block)
if not names:
continue
ups = [u for u in re.findall(r"proxy_pass\s+https?://([^;\s]+)", block)
if "oauth2" not in u]
ssl = "ssl_certificate" in block
auth = ("auth_request" in block) or ("oauth2" in block.lower())
if not auth and ups:
try:
up_port = int(ups[0].rsplit(":", 1)[-1].split("/")[0])
auth = up_port in OAUTH_PORTS
except Exception:
pass
for n in names[0].split():
n = n.strip()
if not n or n in ("_", "localhost") or "$" in n or n in seen:
continue
seen.add(n)
out.append({"domain": n, "upstream": ups[0] if ups else None,
"ssl": ssl, "sso": auth, "edge": where})
return sorted(out, key=lambda x: x["domain"])
def dependencies(hosts_list, services_list, domains_list):
"""Рёбра: хост содержит сервис; домен проксируется на сервис."""
edges = []
for sv in services_list:
edges.append({"from": sv["host"], "to": sv["name"], "kind": "runs"})
for d in domains_list:
up = (d.get("upstream") or "").split("/")[0]
if not up:
continue
target = None
for sv in services_list:
if any(":%d" % p in up for p in (sv.get("ports") or [])):
target = sv["name"]
break
if not target and up:
target = up
edges.append({"from": "domain:%s" % d["domain"], "to": target or up, "kind": "proxy"})
return edges
def main():
os.makedirs(OUT, exist_ok=True)
h = hosts()
s = services(h)
collect_oauth_ports(s)
d = domains()
c = costs(h)
e = dependencies(h, s, d)
for sv in s:
sv["ram_mb"] = c.get(sv["name"])
import yaml
reg = {"version": 2,
"generated": datetime.now().isoformat(timespec="seconds"),
"counts": {"hosts": len(h), "services": len(s), "domains": len(d), "dependencies": len(e)},
"hosts": h, "services": s, "domains": d, "dependencies": e}
path = os.path.join(OUT, "registry.yaml")
with open(path, "w", encoding="utf-8") as f:
yaml.safe_dump(reg, f, allow_unicode=True, sort_keys=False, width=200)
# README с картой
L = ["# Реестр инфраструктуры (единый источник правды)", "",
"_Обновлено: %s_" % reg["generated"], "",
"| Сущность | Количество |", "|---|---|",
"| Хосты (CT + VM) | %d |" % len(h),
"| Сервисы (docker) | %d |" % len(s),
"| Домены | %d |" % len(d), ""]
if d:
L += ["## Домены", "", "| Домен | Куда ведёт | SSL | SSO | Где |", "|---|---|---|---|---|"]
for x in sorted(d, key=lambda v: v["domain"])[:40]:
L.append("| %s | %s | %s | %s | %s |" %
(x["domain"], (x.get("upstream") or "—")[:30],
"да" if x["ssl"] else "нет", "да" if x["sso"] else "нет",
x.get("edge", "хост")))
open(os.path.join(OUT, "README.md"), "w", encoding="utf-8").write("\n".join(L) + "\n")
print("реестр:", path)
print("хостов: %d | сервисов: %d | доменов: %d" % (len(h), len(s), len(d)))
print("домены с SSO: %d | без SSO: %d"
% (sum(1 for x in d if x["sso"]), sum(1 for x in d if not x["sso"])))
print("зависимостей: %d | сервисов со стоимостью: %d" % (len(e), len(c)))
if __name__ == "__main__":
main()