#!/usr/bin/env python3 """Collect SwarmUI/Comfy diagnostics on the VM. Stdlib only. Prints a report to stdout and writes /mnt/swarm_data/.gpu-rent-last-diag.txt so local `gpu-rent up` / wait failures show why backend=errored. """ from __future__ import annotations import json import os import subprocess import time import urllib.error import urllib.request from pathlib import Path SWARM = "http://127.0.0.1:7801" DATA = Path("/mnt/swarm_data") OUT = DATA / ".gpu-rent-last-diag.txt" DLBACKEND = DATA / "dlbackend" COMFY = DLBACKEND / "ComfyUI" VENV_PY = COMFY / "venv" / "bin" / "python" SETTINGS = DATA / "Data" / "Settings.fds" BACKENDS_FDS = DATA / "Data" / "Backends.fds" def _run(cmd: list[str], timeout: float = 20.0) -> str: try: p = subprocess.run( cmd, capture_output=True, text=True, timeout=timeout, errors="replace", ) out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "") return out.strip() except (OSError, subprocess.TimeoutExpired) as exc: return f"(fail: {exc})" def _post(path: str, payload: dict, timeout: float = 12.0) -> dict: body = json.dumps(payload).encode("utf-8") req = urllib.request.Request( f"{SWARM}{path}", data=body, headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(req, timeout=timeout) as resp: return json.loads(resp.read().decode("utf-8")) def section(title: str, body: str) -> str: body = (body or "").strip() or "(empty)" # Cap huge journals lines = body.splitlines() if len(lines) > 120: body = "\n".join(lines[-120:]) body = f"… ({len(lines)} lines, last 120) …\n{body}" return f"=== {title} ===\n{body}\n" def swarm_api_bits() -> str: chunks: list[str] = [] try: sess = _post("/API/GetNewSession", {}) sid = str(sess.get("session_id") or "") if not sid: return "GetNewSession: no session_id" st = _post("/API/GetCurrentStatus", {"session_id": sid}) be = st.get("backend_status") or {} status = st.get("status") or {} chunks.append( "backend_status=" + json.dumps(be, ensure_ascii=False) + "\nqueue=" + json.dumps(status, ensure_ascii=False) ) try: backends = _post( "/API/ListBackends", {"session_id": sid, "nonreal": False, "full_data": True}, ) # Compact: id → type/status/title/enabled summary = {} if isinstance(backends, dict): for key, val in backends.items(): if not isinstance(val, dict): continue summary[key] = { "id": val.get("id"), "type": val.get("type"), "status": val.get("status"), "enabled": val.get("enabled"), "title": val.get("title"), "current_model": val.get("current_model"), } chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2)) try: listed = _post( "/API/ListModels", { "session_id": sid, "path": "", "depth": 3, "subtype": "Stable-Diffusion", }, ) files = listed.get("files") if isinstance(listed, dict) else None folders = listed.get("folders") if isinstance(listed, dict) else None nfiles = len(files) if isinstance(files, list) else "?" nfolders = len(folders) if isinstance(folders, list) else "?" chunks.append(f"ListModels Stable-Diffusion files={nfiles} folders={nfolders}") except Exception as exc: chunks.append(f"ListModels fail: {exc}") # Full settings for first backend (StartScript path matters) for key, val in (backends or {}).items(): if isinstance(val, dict) and val.get("settings"): chunks.append( f"backend[{key}].settings=" + json.dumps(val.get("settings"), ensure_ascii=False) ) break except Exception as exc: chunks.append(f"ListBackends fail: {exc}") except Exception as exc: chunks.append(f"Swarm API fail: {exc}") return "\n".join(chunks) def paths_bits() -> str: rows = [ f"DATA={DATA} exists={DATA.is_dir()}", f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}", f"ComfyUI={COMFY} exists={COMFY.is_dir()}", f"main.py={(COMFY / 'main.py')} exists={(COMFY / 'main.py').is_file()}", f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}", f"opt_dlbackend_main={Path('/opt/swarmui/dlbackend/ComfyUI/main.py')} " f"exists={Path('/opt/swarmui/dlbackend/ComfyUI/main.py').is_file()}", f"Settings.fds exists={SETTINGS.is_file()}", f"Backends.fds exists={BACKENDS_FDS.is_file()}", ] rows.append("findmnt /opt/swarmui/dlbackend:\n" + _run(["findmnt", "/opt/swarmui/dlbackend"])) rows.append("findmnt /opt/swarmui/Models:\n" + _run(["findmnt", "/opt/swarmui/Models"])) data_models = DATA / "Models" opt_models = Path("/opt/swarmui/Models") def _n_weights(root: Path) -> int: if not root.is_dir(): return 0 n = 0 try: for p in root.rglob("*"): if p.suffix.lower() in {".safetensors", ".ckpt", ".sft"}: n += 1 except OSError: return -1 return n rows.append( f"weights data={_n_weights(data_models)} swarm={_n_weights(opt_models)}" ) if SETTINGS.is_file(): try: for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines(): if "IsInstalled" in line: rows.append(f"Settings: {line.strip()}") break except OSError as exc: rows.append(f"Settings read: {exc}") if BACKENDS_FDS.is_file(): try: text = BACKENDS_FDS.read_text(encoding="utf-8", errors="replace") rows.append(f"Backends.fds size={len(text)}b head:\n" + "\n".join(text.splitlines()[:40])) except OSError as exc: rows.append(f"Backends read: {exc}") return "\n".join(rows) def collect() -> str: ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) parts = [ f"gpu-rent swarm diagnostics @ {ts}", section("systemctl swarmui", _run(["systemctl", "is-active", "swarmui"])), section("nvidia-smi", _run(["nvidia-smi", "-L"])), section("paths", paths_bits()), section("SwarmUI API", swarm_api_bits()), section( "journalctl -u swarmui -n 80", _run( ["journalctl", "-u", "swarmui", "-n", "80", "--no-pager", "-o", "short-iso"], timeout=30.0, ), ), section( "journalctl -u swarmui priority=err..alert -n 40", _run( [ "journalctl", "-u", "swarmui", "-p", "err", "-n", "40", "--no-pager", "-o", "short-iso", ], timeout=30.0, ), ), ] # Comfy often logs under dlbackend for cand in ( COMFY / "user" / "comfyui.log", COMFY / "comfyui.log", DATA / "Logs" / "recent.log", ): if cand.is_file(): try: text = cand.read_text(encoding="utf-8", errors="replace") parts.append(section(f"file {cand}", "\n".join(text.splitlines()[-60:]))) except OSError: pass return "\n".join(parts) def main() -> int: report = collect() try: OUT.parent.mkdir(parents=True, exist_ok=True) OUT.write_text(report + "\n", encoding="utf-8") print(f"DIAG saved → {OUT}", flush=True) except OSError as exc: print(f"DIAG save fail: {exc}", flush=True) print(report, flush=True) return 0 if __name__ == "__main__": raise SystemExit(main())