Add diagnostics command and enhance error handling for backend states
- Introduced a new CLI command `diag` to collect diagnostics from SwarmUI/Comfy, including API, journal, and paths. - Enhanced the `ensure_swarm_comfy_installed` function to include diagnostic script handling for errored backends. - Updated `wait_backend_idle` to trigger diagnostics when backends are in an errored state, improving error recovery. - Implemented fallback mechanisms for diagnostics in the `run_diagnostics` function, ensuring better visibility into backend issues. - Added tests to validate the new diagnostic functionality and error handling, ensuring robustness in backend management.
This commit is contained in:
Leonid Pershin
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect SwarmUI/Comfy diagnostics on the VM. Stdlib only.
|
||||
|
||||
Prints a report to stdout and writes /mnt/swarm_data/.gpu-rent-last-diag.txt
|
||||
so local `gpu-rent up` / wait failures show why backend=errored.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
SWARM = "http://127.0.0.1:7801"
|
||||
DATA = Path("/mnt/swarm_data")
|
||||
OUT = DATA / ".gpu-rent-last-diag.txt"
|
||||
DLBACKEND = DATA / "dlbackend"
|
||||
COMFY = DLBACKEND / "ComfyUI"
|
||||
VENV_PY = COMFY / "venv" / "bin" / "python"
|
||||
SETTINGS = DATA / "Data" / "Settings.fds"
|
||||
BACKENDS_FDS = DATA / "Data" / "Backends.fds"
|
||||
|
||||
|
||||
def _run(cmd: list[str], timeout: float = 20.0) -> str:
|
||||
try:
|
||||
p = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
errors="replace",
|
||||
)
|
||||
out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "")
|
||||
return out.strip()
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
return f"(fail: {exc})"
|
||||
|
||||
|
||||
def _post(path: str, payload: dict, timeout: float = 12.0) -> dict:
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{SWARM}{path}",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def section(title: str, body: str) -> str:
|
||||
body = (body or "").strip() or "(empty)"
|
||||
# Cap huge journals
|
||||
lines = body.splitlines()
|
||||
if len(lines) > 120:
|
||||
body = "\n".join(lines[-120:])
|
||||
body = f"… ({len(lines)} lines, last 120) …\n{body}"
|
||||
return f"=== {title} ===\n{body}\n"
|
||||
|
||||
|
||||
def swarm_api_bits() -> str:
|
||||
chunks: list[str] = []
|
||||
try:
|
||||
sess = _post("/API/GetNewSession", {})
|
||||
sid = str(sess.get("session_id") or "")
|
||||
if not sid:
|
||||
return "GetNewSession: no session_id"
|
||||
st = _post("/API/GetCurrentStatus", {"session_id": sid})
|
||||
be = st.get("backend_status") or {}
|
||||
status = st.get("status") or {}
|
||||
chunks.append(
|
||||
"backend_status="
|
||||
+ json.dumps(be, ensure_ascii=False)
|
||||
+ "\nqueue="
|
||||
+ json.dumps(status, ensure_ascii=False)
|
||||
)
|
||||
try:
|
||||
backends = _post(
|
||||
"/API/ListBackends",
|
||||
{"session_id": sid, "nonreal": False, "full_data": True},
|
||||
)
|
||||
# Compact: id → type/status/title/enabled
|
||||
summary = {}
|
||||
if isinstance(backends, dict):
|
||||
for key, val in backends.items():
|
||||
if not isinstance(val, dict):
|
||||
continue
|
||||
summary[key] = {
|
||||
"id": val.get("id"),
|
||||
"type": val.get("type"),
|
||||
"status": val.get("status"),
|
||||
"enabled": val.get("enabled"),
|
||||
"title": val.get("title"),
|
||||
"current_model": val.get("current_model"),
|
||||
}
|
||||
chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
except Exception as exc:
|
||||
chunks.append(f"ListBackends fail: {exc}")
|
||||
except Exception as exc:
|
||||
chunks.append(f"Swarm API fail: {exc}")
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
def paths_bits() -> str:
|
||||
rows = [
|
||||
f"DATA={DATA} exists={DATA.is_dir()}",
|
||||
f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}",
|
||||
f"ComfyUI={COMFY} exists={COMFY.is_dir()}",
|
||||
f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}",
|
||||
f"Settings.fds exists={SETTINGS.is_file()}",
|
||||
f"Backends.fds exists={BACKENDS_FDS.is_file()}",
|
||||
]
|
||||
if SETTINGS.is_file():
|
||||
try:
|
||||
for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if "IsInstalled" in line:
|
||||
rows.append(f"Settings: {line.strip()}")
|
||||
break
|
||||
except OSError as exc:
|
||||
rows.append(f"Settings read: {exc}")
|
||||
if BACKENDS_FDS.is_file():
|
||||
try:
|
||||
text = BACKENDS_FDS.read_text(encoding="utf-8", errors="replace")
|
||||
rows.append(f"Backends.fds size={len(text)}b head:\n" + "\n".join(text.splitlines()[:40]))
|
||||
except OSError as exc:
|
||||
rows.append(f"Backends read: {exc}")
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def collect() -> str:
|
||||
ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
parts = [
|
||||
f"gpu-rent swarm diagnostics @ {ts}",
|
||||
section("systemctl swarmui", _run(["systemctl", "is-active", "swarmui"])),
|
||||
section("nvidia-smi", _run(["nvidia-smi", "-L"])),
|
||||
section("paths", paths_bits()),
|
||||
section("SwarmUI API", swarm_api_bits()),
|
||||
section(
|
||||
"journalctl -u swarmui -n 80",
|
||||
_run(
|
||||
["journalctl", "-u", "swarmui", "-n", "80", "--no-pager", "-o", "short-iso"],
|
||||
timeout=30.0,
|
||||
),
|
||||
),
|
||||
section(
|
||||
"journalctl -u swarmui priority=err..alert -n 40",
|
||||
_run(
|
||||
[
|
||||
"journalctl",
|
||||
"-u",
|
||||
"swarmui",
|
||||
"-p",
|
||||
"err",
|
||||
"-n",
|
||||
"40",
|
||||
"--no-pager",
|
||||
"-o",
|
||||
"short-iso",
|
||||
],
|
||||
timeout=30.0,
|
||||
),
|
||||
),
|
||||
]
|
||||
# Comfy often logs under dlbackend
|
||||
for cand in (
|
||||
COMFY / "user" / "comfyui.log",
|
||||
COMFY / "comfyui.log",
|
||||
DATA / "Logs" / "recent.log",
|
||||
):
|
||||
if cand.is_file():
|
||||
try:
|
||||
text = cand.read_text(encoding="utf-8", errors="replace")
|
||||
parts.append(section(f"file {cand}", "\n".join(text.splitlines()[-60:])))
|
||||
except OSError:
|
||||
pass
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report = collect()
|
||||
try:
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(report + "\n", encoding="utf-8")
|
||||
print(f"DIAG saved → {OUT}", flush=True)
|
||||
except OSError as exc:
|
||||
print(f"DIAG save fail: {exc}", flush=True)
|
||||
print(report, flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user