Files
buryan-infra/ops/scripts/build_topology.py

245 lines
9.4 KiB
Python

#!/usr/bin/env python3
"""V1 / Фаза A: инвентаризация топологии и лёджер стоимости.
Только чтение. Пишет:
ops/topology/topology.yaml — узлы (CT, сервисы) и рёбра зависимостей
ops/topology/ledger.md — человекочитаемый лёджер (RAM/CPU/порт/доступность)
"""
import json
import os
import re
import subprocess
import sys
from datetime import datetime
OUT_DIR = "/root/buryan-infra/ops/topology"
VPS = "193.233.210.184"
def sh(cmd, timeout=60):
try:
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
return (r.stdout or "").strip()
except Exception:
return ""
def shv(cmd, timeout=60):
"""команда на VPS"""
return sh("ssh -o ConnectTimeout=10 -o BatchMode=yes root@%s '%s'" % (VPS, cmd.replace("'", "'\\''")), timeout)
print("1) контейнеры", file=sys.stderr)
cts = []
for line in sh("pct list").splitlines()[1:]:
parts = line.split()
if not parts:
continue
cid = parts[0]
status = parts[1] if len(parts) > 1 else "?"
name = parts[2] if len(parts) > 2 else ""
ip = sh("pct exec %s -- sh -c \"ip -4 -br a 2>/dev/null | awk '/eth0/ {print \\$3}' | cut -d/ -f1\"" % cid, 30)
cts.append({"id": "ct" + cid, "ct": cid, "name": name, "status": status, "ip": ip or ""})
print("2) docker-контейнеры", file=sys.stderr)
docker = []
for c in cts:
if c["status"] != "running":
continue
out = sh("pct exec %s -- docker ps --format '{{.Names}}|{{.Image}}|{{.Ports}}' 2>/dev/null" % c["ct"], 40)
for ln in out.splitlines():
p = ln.split("|")
if len(p) < 2:
continue
ports = p[2] if len(p) > 2 else ""
m = re.findall(r"0\.0\.0\.0:(\d+)->", ports)
docker.append({"ct": c["id"], "name": p[0], "image": p[1][:60],
"ports": [int(x) for x in m]})
print("3) слушающие порты хоста", file=sys.stderr)
host_ports = []
for ln in sh("ss -tlnp").splitlines()[1:]:
m = re.search(r"0\.0\.0\.0:(\d+)", ln)
if m:
proc = ""
pm = re.search(r'users:\(\("([^"]+)"', ln)
if pm:
proc = pm.group(1)
host_ports.append({"port": int(m.group(1)), "proc": proc})
print("4) nginx: кто кому проксирует", file=sys.stderr)
edges = []
def parse_nginx(text):
cur = None
for ln in text.splitlines():
s = ln.strip()
m = re.match(r"server_name\s+([^;]+);", s)
if m:
cur = m.group(1).split()[0]
continue
m = re.match(r"proxy_pass\s+(\S+);", s)
if m and cur:
tgt = m.group(1)
mm = re.search(r"//([\d.]+):(\d+)", tgt)
if mm:
edges.append({"from": "nginx:" + cur, "to": "ip:%s:%s" % (mm.group(1), mm.group(2)),
"type": "proxy_pass"})
elif re.search(r"//([\w.-]+)", tgt):
host = re.search(r"//([\w.-]+)", tgt).group(1)
if not host.startswith("127.") and not host.startswith("unix"):
edges.append({"from": "nginx:" + cur, "to": "host:" + host, "type": "proxy_pass"})
parse_nginx(sh("cat /etc/nginx/conf.d/*.conf 2>/dev/null"))
parse_nginx(shv("cat /etc/nginx/conf.d/*.conf 2>/dev/null"))
print("5) зависимости из *arr API", file=sys.stderr)
def key(p):
try:
return open(p).read().strip()
except Exception:
return ""
def api(url, hdr, timeout=15):
out = sh("curl -s -m %d -H '%s' '%s'" % (timeout, hdr, url), timeout + 10)
try:
return json.loads(out) if out else None
except Exception:
return None
rk = key("/root/.radarr-key")
sk = key("/root/.sonarr-key")
pk = key("/root/.prowlarr-key")
for nm, base, kk, cat in (("radarr", "http://192.168.1.118:7878", rk, "X-Api-Key"),
("sonarr", "http://192.168.1.117:8989", sk, "X-Api-Key"),
("prowlarr", "http://192.168.1.119:9696", pk, "X-Api-Key")):
d = api("%s/api/v3/downloadclient" % base, "%s: %s" % (cat, kk)) or \
api("%s/api/v1/downloadclient" % base, "%s: %s" % (cat, kk))
for c in (d or []):
f = {x.get("name"): x.get("value") for x in (c.get("fields") or [])}
h, p = f.get("host"), f.get("port")
if h and p:
edges.append({"from": "svc:" + nm, "to": "ip:%s:%s" % (h, p), "type": "download_client"})
idx = api("%s/api/v3/indexer" % base, "%s: %s" % (cat, kk)) or \
api("%s/api/v1/indexer" % base, "%s: %s" % (cat, kk))
for i in (idx or []):
nm2 = (i.get("name") or "").lower()
if "prowlarr" in nm2:
edges.append({"from": "svc:" + nm, "to": "svc:prowlarr", "type": "indexer_sync"})
print("6) hub-core: каталог сервисов", file=sys.stderr)
status = api("http://127.0.0.1:9120/status.json", "X-Ignore: 1", 70)
services = (status or {}).get("services") or []
if not services:
print(" ВНИМАНИЕ: /status.json не ответил — узлы каталога пропущены", file=sys.stderr)
# дедупликация рёбер
edges = list({(e["from"], e["to"], e["type"]): e for e in edges}.values())
print("7) docker stats (лёджер)", file=sys.stderr)
stats = {}
for c in cts:
if c["status"] != "running":
continue
out = sh("pct exec %s -- docker stats --no-stream --format '{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}' 2>/dev/null" % c["ct"], 45)
for ln in out.splitlines():
p = ln.split("|")
if len(p) >= 3:
stats[p[0]] = {"cpu": p[1], "mem": p[2]}
print("8) использование по доменам (nginx, 24ч)", file=sys.stderr)
usage = {}
for src in ("host", "vps"):
cmd = ("awk '{print $7}' /var/log/nginx/access.log 2>/dev/null | head -200000"
if src == "host" else
"grep -ho '\"[A-Z]* /[^\"]*' /var/log/nginx/access.log 2>/dev/null | head -200000")
# упрощённо: считаем по vhost из access.log (поле $host)
cmd = "awk '{print $1}' /var/log/nginx/access.log 2>/dev/null | wc -l"
n = shv(cmd) if src == "vps" else sh(cmd)
usage[src] = n.strip()
os.makedirs(OUT_DIR, exist_ok=True)
nodes = []
for c in cts:
nodes.append({"id": c["id"], "kind": "lxc", "name": c["name"], "ip": c["ip"], "status": c["status"]})
for d in docker:
nodes.append({"id": "docker:" + d["name"], "kind": "container", "host": d["ct"],
"image": d["image"], "ports": d["ports"]})
for p in host_ports:
nodes.append({"id": "port:%d" % p["port"], "kind": "host_port", "port": p["port"], "proc": p["proc"]})
for s in services:
nodes.append({"id": "catalog:" + str(s.get("name")), "kind": "service",
"ip": s.get("ip"), "port": s.get("port"), "up": s.get("up")})
# ---------- нормализация: ip:port -> канонический узел ----------
ip2ct = {c["ip"]: c["id"] for c in cts if c.get("ip")}
port2docker = {}
for d in docker:
ct_ip = next((c["ip"] for c in cts if c["id"] == d["ct"]), "")
for pt in d["ports"]:
port2docker[(ct_ip, pt)] = "docker:" + d["name"]
port2svc = {}
for s in services:
if s.get("ip") and s.get("port"):
port2svc[(str(s["ip"]), int(s["port"]))] = "catalog:" + str(s.get("name"))
def canon(endpoint):
m = re.match(r"ip:([\d.]+):(\d+)$", endpoint)
if not m:
return endpoint
ip, port = m.group(1), int(m.group(2))
return port2docker.get((ip, port)) or port2svc.get((ip, port)) or ip2ct.get(ip) or endpoint
for e in edges:
e["from"] = canon(e["from"])
e["to"] = canon(e["to"])
# рёбра «контейнер живёт в CT»
for d in docker:
edges.append({"from": "docker:" + d["name"], "to": d["ct"], "type": "runs_on"})
# рёбра «каталог ↔ контейнер» по совпадению имён
docker_names = {d["name"]: "docker:" + d["name"] for d in docker}
for s in services:
nm = str(s.get("name") or "").lower()
for dn, did in docker_names.items():
if nm and (nm in dn.lower() or dn.lower() in nm):
edges.append({"from": "catalog:" + str(s.get("name")), "to": did, "type": "same_service"})
break
edges = list({(e["from"], e["to"], e["type"]): e for e in edges}.values())
topo = {
"version": 1,
"generated": datetime.now().isoformat(timespec="seconds"),
"node_count": len(nodes),
"edge_count": len(edges),
}
with open(OUT_DIR + "/topology.json", "w", encoding="utf-8") as f:
json.dump({"topo": topo, "nodes": nodes, "edges": edges}, f, ensure_ascii=False, indent=1)
# лёджер
lines = ["# Лёджер сервисов (V1, Фаза A)", "", "Сгенерировано: " + topo["generated"], "",
"| Сервис | CT | Образ | Порты | CPU | RAM |", "|---|---|---|---|---|---|"]
for d in sorted(docker, key=lambda x: x["name"]):
st = stats.get(d["name"], {})
lines.append("| %s | %s | %s | %s | %s | %s |" % (
d["name"], d["ct"], d["image"][:38], ",".join(map(str, d["ports"])) or "—",
st.get("cpu", "—"), st.get("mem", "—")))
lines += ["", "## Порты хоста", "", "| Порт | Процесс |", "|---|---|"]
for p in sorted(host_ports, key=lambda x: x["port"]):
lines.append("| %d | %s |" % (p["port"], p["proc"]))
with open(OUT_DIR + "/ledger.md", "w", encoding="utf-8") as f:
f.write("\n".join(lines) + "\n")
print("узлов:", len(nodes), "| рёбер:", len(edges), "| файлы в", OUT_DIR)