Leonid Pershin
f437cd0373
- Clarified the behavior of `Ctrl+C` and `Ctrl+D` in the README and other documentation, specifying that `Ctrl+C` only stops the tunnel while keeping the GPU active, and `Ctrl+D` stops the GPU while preserving disk data. - Enhanced the CLI documentation to reflect these changes, ensuring users understand the implications of these commands during GPU operations. - Improved the handling of data bindings and remounting logic in the codebase to prevent issues with empty model tabs in the UI. - Added tests to validate the new command behaviors and ensure proper documentation alignment.
243 lines
8.7 KiB
Python
243 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Collect SwarmUI/Comfy diagnostics on the VM. Stdlib only.
|
|
|
|
Prints a report to stdout and writes /mnt/swarm_data/.gpu-rent-last-diag.txt
|
|
so local `gpu-rent up` / wait failures show why backend=errored.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
SWARM = "http://127.0.0.1:7801"
|
|
DATA = Path("/mnt/swarm_data")
|
|
OUT = DATA / ".gpu-rent-last-diag.txt"
|
|
DLBACKEND = DATA / "dlbackend"
|
|
COMFY = DLBACKEND / "ComfyUI"
|
|
VENV_PY = COMFY / "venv" / "bin" / "python"
|
|
SETTINGS = DATA / "Data" / "Settings.fds"
|
|
BACKENDS_FDS = DATA / "Data" / "Backends.fds"
|
|
|
|
|
|
def _run(cmd: list[str], timeout: float = 20.0) -> str:
|
|
try:
|
|
p = subprocess.run(
|
|
cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
errors="replace",
|
|
)
|
|
out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "")
|
|
return out.strip()
|
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
return f"(fail: {exc})"
|
|
|
|
|
|
def _post(path: str, payload: dict, timeout: float = 12.0) -> dict:
|
|
body = json.dumps(payload).encode("utf-8")
|
|
req = urllib.request.Request(
|
|
f"{SWARM}{path}",
|
|
data=body,
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
return json.loads(resp.read().decode("utf-8"))
|
|
|
|
|
|
def section(title: str, body: str) -> str:
|
|
body = (body or "").strip() or "(empty)"
|
|
# Cap huge journals
|
|
lines = body.splitlines()
|
|
if len(lines) > 120:
|
|
body = "\n".join(lines[-120:])
|
|
body = f"… ({len(lines)} lines, last 120) …\n{body}"
|
|
return f"=== {title} ===\n{body}\n"
|
|
|
|
|
|
def swarm_api_bits() -> str:
|
|
chunks: list[str] = []
|
|
try:
|
|
sess = _post("/API/GetNewSession", {})
|
|
sid = str(sess.get("session_id") or "")
|
|
if not sid:
|
|
return "GetNewSession: no session_id"
|
|
st = _post("/API/GetCurrentStatus", {"session_id": sid})
|
|
be = st.get("backend_status") or {}
|
|
status = st.get("status") or {}
|
|
chunks.append(
|
|
"backend_status="
|
|
+ json.dumps(be, ensure_ascii=False)
|
|
+ "\nqueue="
|
|
+ json.dumps(status, ensure_ascii=False)
|
|
)
|
|
try:
|
|
backends = _post(
|
|
"/API/ListBackends",
|
|
{"session_id": sid, "nonreal": False, "full_data": True},
|
|
)
|
|
# Compact: id → type/status/title/enabled
|
|
summary = {}
|
|
if isinstance(backends, dict):
|
|
for key, val in backends.items():
|
|
if not isinstance(val, dict):
|
|
continue
|
|
summary[key] = {
|
|
"id": val.get("id"),
|
|
"type": val.get("type"),
|
|
"status": val.get("status"),
|
|
"enabled": val.get("enabled"),
|
|
"title": val.get("title"),
|
|
"current_model": val.get("current_model"),
|
|
}
|
|
chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2))
|
|
try:
|
|
listed = _post(
|
|
"/API/ListModels",
|
|
{
|
|
"session_id": sid,
|
|
"path": "",
|
|
"depth": 3,
|
|
"subtype": "Stable-Diffusion",
|
|
},
|
|
)
|
|
files = listed.get("files") if isinstance(listed, dict) else None
|
|
folders = listed.get("folders") if isinstance(listed, dict) else None
|
|
nfiles = len(files) if isinstance(files, list) else "?"
|
|
nfolders = len(folders) if isinstance(folders, list) else "?"
|
|
chunks.append(f"ListModels Stable-Diffusion files={nfiles} folders={nfolders}")
|
|
except Exception as exc:
|
|
chunks.append(f"ListModels fail: {exc}")
|
|
# Full settings for first backend (StartScript path matters)
|
|
for key, val in (backends or {}).items():
|
|
if isinstance(val, dict) and val.get("settings"):
|
|
chunks.append(
|
|
f"backend[{key}].settings="
|
|
+ json.dumps(val.get("settings"), ensure_ascii=False)
|
|
)
|
|
break
|
|
except Exception as exc:
|
|
chunks.append(f"ListBackends fail: {exc}")
|
|
except Exception as exc:
|
|
chunks.append(f"Swarm API fail: {exc}")
|
|
return "\n".join(chunks)
|
|
|
|
|
|
def paths_bits() -> str:
|
|
rows = [
|
|
f"DATA={DATA} exists={DATA.is_dir()}",
|
|
f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}",
|
|
f"ComfyUI={COMFY} exists={COMFY.is_dir()}",
|
|
f"main.py={(COMFY / 'main.py')} exists={(COMFY / 'main.py').is_file()}",
|
|
f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}",
|
|
f"opt_dlbackend_main={Path('/opt/swarmui/dlbackend/ComfyUI/main.py')} "
|
|
f"exists={Path('/opt/swarmui/dlbackend/ComfyUI/main.py').is_file()}",
|
|
f"Settings.fds exists={SETTINGS.is_file()}",
|
|
f"Backends.fds exists={BACKENDS_FDS.is_file()}",
|
|
]
|
|
rows.append("findmnt /opt/swarmui/dlbackend:\n" + _run(["findmnt", "/opt/swarmui/dlbackend"]))
|
|
rows.append("findmnt /opt/swarmui/Models:\n" + _run(["findmnt", "/opt/swarmui/Models"]))
|
|
data_models = DATA / "Models"
|
|
opt_models = Path("/opt/swarmui/Models")
|
|
def _n_weights(root: Path) -> int:
|
|
if not root.is_dir():
|
|
return 0
|
|
n = 0
|
|
try:
|
|
for p in root.rglob("*"):
|
|
if p.suffix.lower() in {".safetensors", ".ckpt", ".sft"}:
|
|
n += 1
|
|
except OSError:
|
|
return -1
|
|
return n
|
|
|
|
rows.append(
|
|
f"weights data={_n_weights(data_models)} swarm={_n_weights(opt_models)}"
|
|
)
|
|
if SETTINGS.is_file():
|
|
try:
|
|
for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
if "IsInstalled" in line:
|
|
rows.append(f"Settings: {line.strip()}")
|
|
break
|
|
except OSError as exc:
|
|
rows.append(f"Settings read: {exc}")
|
|
if BACKENDS_FDS.is_file():
|
|
try:
|
|
text = BACKENDS_FDS.read_text(encoding="utf-8", errors="replace")
|
|
rows.append(f"Backends.fds size={len(text)}b head:\n" + "\n".join(text.splitlines()[:40]))
|
|
except OSError as exc:
|
|
rows.append(f"Backends read: {exc}")
|
|
return "\n".join(rows)
|
|
|
|
|
|
def collect() -> str:
|
|
ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
parts = [
|
|
f"gpu-rent swarm diagnostics @ {ts}",
|
|
section("systemctl swarmui", _run(["systemctl", "is-active", "swarmui"])),
|
|
section("nvidia-smi", _run(["nvidia-smi", "-L"])),
|
|
section("paths", paths_bits()),
|
|
section("SwarmUI API", swarm_api_bits()),
|
|
section(
|
|
"journalctl -u swarmui -n 80",
|
|
_run(
|
|
["journalctl", "-u", "swarmui", "-n", "80", "--no-pager", "-o", "short-iso"],
|
|
timeout=30.0,
|
|
),
|
|
),
|
|
section(
|
|
"journalctl -u swarmui priority=err..alert -n 40",
|
|
_run(
|
|
[
|
|
"journalctl",
|
|
"-u",
|
|
"swarmui",
|
|
"-p",
|
|
"err",
|
|
"-n",
|
|
"40",
|
|
"--no-pager",
|
|
"-o",
|
|
"short-iso",
|
|
],
|
|
timeout=30.0,
|
|
),
|
|
),
|
|
]
|
|
# Comfy often logs under dlbackend
|
|
for cand in (
|
|
COMFY / "user" / "comfyui.log",
|
|
COMFY / "comfyui.log",
|
|
DATA / "Logs" / "recent.log",
|
|
):
|
|
if cand.is_file():
|
|
try:
|
|
text = cand.read_text(encoding="utf-8", errors="replace")
|
|
parts.append(section(f"file {cand}", "\n".join(text.splitlines()[-60:])))
|
|
except OSError:
|
|
pass
|
|
return "\n".join(parts)
|
|
|
|
|
|
def main() -> int:
|
|
report = collect()
|
|
try:
|
|
OUT.parent.mkdir(parents=True, exist_ok=True)
|
|
OUT.write_text(report + "\n", encoding="utf-8")
|
|
print(f"DIAG saved → {OUT}", flush=True)
|
|
except OSError as exc:
|
|
print(f"DIAG save fail: {exc}", flush=True)
|
|
print(report, flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|