Add diagnostics command and enhance error handling for backend states
- Introduced a new CLI command `diag` to collect diagnostics from SwarmUI/Comfy, including API, journal, and paths. - Enhanced the `ensure_swarm_comfy_installed` function to include diagnostic script handling for errored backends. - Updated `wait_backend_idle` to trigger diagnostics when backends are in an errored state, improving error recovery. - Implemented fallback mechanisms for diagnostics in the `run_diagnostics` function, ensuring better visibility into backend issues. - Added tests to validate the new diagnostic functionality and error handling, ensuring robustness in backend management.
This commit is contained in:
Leonid Pershin
@@ -712,6 +712,28 @@ def ssh() -> None:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@app.command("diag")
|
||||
def diag_cmd(
|
||||
lines: int = typer.Option(80, "--lines", "-n", help="Строк journal в отчёте"),
|
||||
) -> None:
|
||||
"""Снять диагностику SwarmUI/Comfy с VM (API + journal + paths)."""
|
||||
try:
|
||||
cfg = load_config(require_auth=True)
|
||||
state = load_state()
|
||||
if not state.floating_ip:
|
||||
raise GpuRentError("нет IP — VM не поднята")
|
||||
from gpu_rent.ready import collect_swarm_diagnostics
|
||||
|
||||
# lines reserved for future; swarm_diag uses fixed 80 for now
|
||||
_ = lines
|
||||
collect_swarm_diagnostics(cfg, state.floating_ip, console.print)
|
||||
console.print(
|
||||
"[dim]/mnt/swarm_data/.gpu-rent-last-diag.txt на VM[/dim]"
|
||||
)
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@app.command()
|
||||
def logs(
|
||||
unit: Optional[str] = typer.Option(
|
||||
|
||||
@@ -93,8 +93,10 @@ def tune_swarm_perf(cfg: Config, host: str, log: Log) -> bool:
|
||||
|
||||
|
||||
def ensure_swarm_comfy_installed(cfg: Config, host: str, log: Log) -> None:
|
||||
"""Headless SwarmUI InstallConfirmWS (ComfyUI) when backends are still empty."""
|
||||
log("SwarmUI first-install: ComfyUI backend (если ещё empty)…")
|
||||
"""Headless Comfy install / recover errored backends before ready wait."""
|
||||
# Diag script next to install so recover-fail can subprocess it on the VM.
|
||||
put_text(cfg, host, "/tmp/gpu-rent-swarm_diag.py", _pkg_text("swarm_diag.py"))
|
||||
log("SwarmUI Comfy: install если empty, RestartBackends если errored…")
|
||||
run_python(
|
||||
cfg,
|
||||
host,
|
||||
|
||||
+61
-3
@@ -16,7 +16,7 @@ from dataclasses import dataclass
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.llm_runtime import normalize_runtime
|
||||
from gpu_rent.ssh_ops import run_ssh
|
||||
from gpu_rent.ssh_ops import run_python, run_ssh
|
||||
from gpu_rent.timing import WaitLog
|
||||
|
||||
Log = Callable[[str], None]
|
||||
@@ -157,6 +157,48 @@ class ServiceCheck:
|
||||
where: str = "vm" # vm | local
|
||||
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
|
||||
def collect_swarm_diagnostics(cfg: Config, host: str, log: Log) -> str:
|
||||
"""Run remote swarm_diag.py; stream to log; return full report text."""
|
||||
from importlib.resources import files
|
||||
|
||||
log("диагностика SwarmUI/Comfy…")
|
||||
try:
|
||||
script = files("gpu_rent.remote").joinpath("swarm_diag.py").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
out = run_python(
|
||||
cfg,
|
||||
host,
|
||||
script,
|
||||
remote_path="/tmp/gpu-rent-swarm_diag.py",
|
||||
timeout=90,
|
||||
log=log,
|
||||
)
|
||||
return out or ""
|
||||
except Exception as exc:
|
||||
# Best-effort fallback when full script fails
|
||||
log(f"diag script fail: {exc}")
|
||||
try:
|
||||
fb = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"echo '=== systemctl swarmui ==='; systemctl is-active swarmui; "
|
||||
"echo; echo '=== journalctl -u swarmui -n 60 ==='; "
|
||||
"sudo -n journalctl -u swarmui -n 60 --no-pager 2>/dev/null || true",
|
||||
check=False,
|
||||
timeout=45,
|
||||
)
|
||||
for line in (fb or "").splitlines():
|
||||
log(line)
|
||||
return fb or ""
|
||||
except Exception as exc2:
|
||||
log(f"diag fallback fail: {exc2}")
|
||||
return ""
|
||||
|
||||
|
||||
def wait_backend_idle(
|
||||
cfg: Config,
|
||||
host: str,
|
||||
@@ -164,15 +206,17 @@ def wait_backend_idle(
|
||||
*,
|
||||
timeout: float = 2400.0,
|
||||
poll_every: float = 15.0,
|
||||
errored_fail_sec: float = 120.0,
|
||||
) -> None:
|
||||
"""Block until SwarmUI backends are ready (status=running, no queue).
|
||||
|
||||
Note: SwarmUI ``idle`` means suspended backends (cannot generate).
|
||||
Ready-to-use is ``running``.
|
||||
Ready-to-use is ``running``. Sustained ``errored`` fail-fasts (won't self-heal).
|
||||
"""
|
||||
deadline = time.time() + timeout
|
||||
log("жду ready backend (running)…")
|
||||
last = ""
|
||||
errored_since: float | None = None
|
||||
pretty = {
|
||||
"BUSY backend=loading (Comfy стартует)": "… Comfy стартует",
|
||||
"BUSY backend=some_loading (Comfy стартует)": "… Comfy стартует (часть бэкендов)",
|
||||
@@ -199,10 +243,24 @@ def wait_backend_idle(
|
||||
last = shown
|
||||
if line.startswith("READY"):
|
||||
return
|
||||
if "BUSY backend=errored" in line:
|
||||
now = time.time()
|
||||
if errored_since is None:
|
||||
errored_since = now
|
||||
elif now - errored_since >= errored_fail_sec:
|
||||
collect_swarm_diagnostics(cfg, host, log)
|
||||
raise CloudError(
|
||||
f"backend=errored уже {int(now - errored_since)} с — не заживёт само. "
|
||||
"Диагностика выше (+ /mnt/swarm_data/.gpu-rent-last-diag.txt на VM). "
|
||||
"Server → Backends → Restart или gpu-rent stop / up после фикса."
|
||||
)
|
||||
else:
|
||||
errored_since = None
|
||||
time.sleep(poll_every)
|
||||
collect_swarm_diagnostics(cfg, host, log)
|
||||
raise CloudError(
|
||||
f"backend не стал ready (running) за {int(timeout)} с. "
|
||||
"Проверь journalctl -u swarmui на VM; GPU всё ещё жив."
|
||||
"Диагностика выше (+ .gpu-rent-last-diag.txt на data volume). GPU ещё жив."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -202,6 +202,86 @@ def backend_status() -> str:
|
||||
return f"error:{exc}"
|
||||
|
||||
|
||||
def backend_status_detail() -> tuple[str, str]:
|
||||
"""Return (status, message) from GetCurrentStatus."""
|
||||
try:
|
||||
sid = str(post("/API/GetNewSession", {}).get("session_id") or "")
|
||||
if not sid:
|
||||
return "unknown", ""
|
||||
data = post("/API/GetCurrentStatus", {"session_id": sid})
|
||||
be = data.get("backend_status") or {}
|
||||
return (
|
||||
str(be.get("status") or "unknown").lower(),
|
||||
str(be.get("message") or "").strip(),
|
||||
)
|
||||
except Exception as exc:
|
||||
return f"error:{exc}", ""
|
||||
|
||||
|
||||
def restart_backends(sid: str, *, which: str = "all") -> dict:
|
||||
"""POST /API/RestartBackends — recovers ERRORED Comfy self-start."""
|
||||
return post(
|
||||
"/API/RestartBackends",
|
||||
{"session_id": sid, "backend": which},
|
||||
timeout=180.0,
|
||||
)
|
||||
|
||||
|
||||
def recover_errored_backends(*, wait_sec: float = 180.0) -> str:
|
||||
"""RestartBackends then poll until not errored (or timeout). Returns last status."""
|
||||
bstat, msg = backend_status_detail()
|
||||
print(
|
||||
f"backend errored — RestartBackends(all)"
|
||||
+ (f" ({msg[:120]})" if msg else ""),
|
||||
flush=True,
|
||||
)
|
||||
sid = get_session(time.time() + 60)
|
||||
try:
|
||||
result = restart_backends(sid)
|
||||
print(f"RestartBackends: {result}", flush=True)
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, json.JSONDecodeError) as exc:
|
||||
print(f"RestartBackends FAIL: {exc}", flush=True)
|
||||
return bstat
|
||||
|
||||
deadline = time.time() + wait_sec
|
||||
last = "errored"
|
||||
while time.time() < deadline:
|
||||
time.sleep(8)
|
||||
last, msg2 = backend_status_detail()
|
||||
extra = f" — {msg2[:100]}" if msg2 else ""
|
||||
print(f"after restart: backend_status={last}{extra}", flush=True)
|
||||
if last != "errored" and not last.startswith("error:"):
|
||||
return last
|
||||
return last
|
||||
|
||||
|
||||
def run_diagnostics() -> None:
|
||||
"""Best-effort: prefer uploaded swarm_diag.py, else journalctl snippet."""
|
||||
diag = Path("/tmp/gpu-rent-swarm_diag.py")
|
||||
if diag.is_file():
|
||||
try:
|
||||
subprocess.run(
|
||||
[sys.executable, str(diag)],
|
||||
check=False,
|
||||
timeout=90,
|
||||
)
|
||||
return
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f"diag script fail: {exc}", flush=True)
|
||||
print("=== fallback journalctl -u swarmui -n 60 ===", flush=True)
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
["journalctl", "-u", "swarmui", "-n", "60", "--no-pager"],
|
||||
text=True,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=30,
|
||||
errors="replace",
|
||||
)
|
||||
print(out, flush=True)
|
||||
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
|
||||
print(f"journalctl fail: {exc}", flush=True)
|
||||
|
||||
|
||||
def settings_is_installed() -> bool | None:
|
||||
if not SETTINGS.is_file():
|
||||
return None
|
||||
@@ -448,15 +528,33 @@ def main() -> int:
|
||||
# WorkingDirectory for comfy-install-linux.sh is Swarm root when launched by
|
||||
# Installation.cs; InstallConfirmWS handles that for us.
|
||||
wait_http(time.time() + 180)
|
||||
bstat = backend_status()
|
||||
print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
|
||||
f"IsInstalled={settings_is_installed()}")
|
||||
bstat, bmsg = backend_status_detail()
|
||||
print(
|
||||
f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
|
||||
f"IsInstalled={settings_is_installed()}"
|
||||
+ (f" msg={bmsg[:160]}" if bmsg else ""),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
|
||||
# Errored backends are registered but dead — restart, don't skip as "present".
|
||||
if bstat == "errored":
|
||||
bstat = recover_errored_backends(wait_sec=180.0)
|
||||
if bstat == "errored" or bstat.startswith("error:"):
|
||||
print(
|
||||
"FAIL: backend still errored after RestartBackends — collecting diag",
|
||||
file=sys.stderr,
|
||||
)
|
||||
run_diagnostics()
|
||||
return 1
|
||||
if bstat in ("running", "idle", "loading", "some_loading") and comfy_venv_ok():
|
||||
print(f"recovered to {bstat} — skip InstallConfirmWS")
|
||||
return 0
|
||||
|
||||
# Backends already registered (healthy / loading / suspended) + venv → skip.
|
||||
if bstat == "idle":
|
||||
print("backends present (idle/suspended) + skip install")
|
||||
return 0
|
||||
if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
|
||||
if bstat not in ("empty", "unknown", "errored") and not bstat.startswith("error:"):
|
||||
if comfy_venv_ok():
|
||||
print(f"backends present ({bstat}) + venv — skip install")
|
||||
return 0
|
||||
@@ -468,9 +566,10 @@ def main() -> int:
|
||||
"Open SwarmUI → Server → Backends and add ComfyUI Self-Starting, "
|
||||
"or delete Data/Settings.fds IsInstalled and re-run up."
|
||||
)
|
||||
run_diagnostics()
|
||||
return 1
|
||||
|
||||
if installed is True and comfy_venv_ok():
|
||||
if installed is True and comfy_venv_ok() and bstat not in ("empty", "errored"):
|
||||
print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
|
||||
return 0
|
||||
|
||||
@@ -478,6 +577,7 @@ def main() -> int:
|
||||
if installed is not True:
|
||||
print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
|
||||
else:
|
||||
run_diagnostics()
|
||||
return 1
|
||||
|
||||
marker = DATA / ".gpu-rent-comfy-installing"
|
||||
@@ -501,8 +601,15 @@ def main() -> int:
|
||||
f"after install: backend_status={bstat2} "
|
||||
f"venv={'yes' if comfy_venv_ok() else 'no'}"
|
||||
)
|
||||
if bstat2 == "errored":
|
||||
bstat2 = recover_errored_backends(wait_sec=120.0)
|
||||
if bstat2 == "errored":
|
||||
print("FAIL: install done but backend errored", file=sys.stderr)
|
||||
run_diagnostics()
|
||||
return 1
|
||||
if not comfy_venv_ok() and bstat2 == "empty":
|
||||
print("FAIL: install finished but still empty / no venv", file=sys.stderr)
|
||||
run_diagnostics()
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Collect SwarmUI/Comfy diagnostics on the VM. Stdlib only.
|
||||
|
||||
Prints a report to stdout and writes /mnt/swarm_data/.gpu-rent-last-diag.txt
|
||||
so local `gpu-rent up` / wait failures show why backend=errored.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
SWARM = "http://127.0.0.1:7801"
|
||||
DATA = Path("/mnt/swarm_data")
|
||||
OUT = DATA / ".gpu-rent-last-diag.txt"
|
||||
DLBACKEND = DATA / "dlbackend"
|
||||
COMFY = DLBACKEND / "ComfyUI"
|
||||
VENV_PY = COMFY / "venv" / "bin" / "python"
|
||||
SETTINGS = DATA / "Data" / "Settings.fds"
|
||||
BACKENDS_FDS = DATA / "Data" / "Backends.fds"
|
||||
|
||||
|
||||
def _run(cmd: list[str], timeout: float = 20.0) -> str:
|
||||
try:
|
||||
p = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
errors="replace",
|
||||
)
|
||||
out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "")
|
||||
return out.strip()
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
return f"(fail: {exc})"
|
||||
|
||||
|
||||
def _post(path: str, payload: dict, timeout: float = 12.0) -> dict:
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
f"{SWARM}{path}",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
|
||||
def section(title: str, body: str) -> str:
|
||||
body = (body or "").strip() or "(empty)"
|
||||
# Cap huge journals
|
||||
lines = body.splitlines()
|
||||
if len(lines) > 120:
|
||||
body = "\n".join(lines[-120:])
|
||||
body = f"… ({len(lines)} lines, last 120) …\n{body}"
|
||||
return f"=== {title} ===\n{body}\n"
|
||||
|
||||
|
||||
def swarm_api_bits() -> str:
|
||||
chunks: list[str] = []
|
||||
try:
|
||||
sess = _post("/API/GetNewSession", {})
|
||||
sid = str(sess.get("session_id") or "")
|
||||
if not sid:
|
||||
return "GetNewSession: no session_id"
|
||||
st = _post("/API/GetCurrentStatus", {"session_id": sid})
|
||||
be = st.get("backend_status") or {}
|
||||
status = st.get("status") or {}
|
||||
chunks.append(
|
||||
"backend_status="
|
||||
+ json.dumps(be, ensure_ascii=False)
|
||||
+ "\nqueue="
|
||||
+ json.dumps(status, ensure_ascii=False)
|
||||
)
|
||||
try:
|
||||
backends = _post(
|
||||
"/API/ListBackends",
|
||||
{"session_id": sid, "nonreal": False, "full_data": True},
|
||||
)
|
||||
# Compact: id → type/status/title/enabled
|
||||
summary = {}
|
||||
if isinstance(backends, dict):
|
||||
for key, val in backends.items():
|
||||
if not isinstance(val, dict):
|
||||
continue
|
||||
summary[key] = {
|
||||
"id": val.get("id"),
|
||||
"type": val.get("type"),
|
||||
"status": val.get("status"),
|
||||
"enabled": val.get("enabled"),
|
||||
"title": val.get("title"),
|
||||
"current_model": val.get("current_model"),
|
||||
}
|
||||
chunks.append("ListBackends=" + json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
except Exception as exc:
|
||||
chunks.append(f"ListBackends fail: {exc}")
|
||||
except Exception as exc:
|
||||
chunks.append(f"Swarm API fail: {exc}")
|
||||
return "\n".join(chunks)
|
||||
|
||||
|
||||
def paths_bits() -> str:
|
||||
rows = [
|
||||
f"DATA={DATA} exists={DATA.is_dir()}",
|
||||
f"dlbackend={DLBACKEND} exists={DLBACKEND.is_dir()}",
|
||||
f"ComfyUI={COMFY} exists={COMFY.is_dir()}",
|
||||
f"venv_python={VENV_PY} exists={VENV_PY.is_file()} exec={os.access(VENV_PY, os.X_OK) if VENV_PY.is_file() else False}",
|
||||
f"Settings.fds exists={SETTINGS.is_file()}",
|
||||
f"Backends.fds exists={BACKENDS_FDS.is_file()}",
|
||||
]
|
||||
if SETTINGS.is_file():
|
||||
try:
|
||||
for line in SETTINGS.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
if "IsInstalled" in line:
|
||||
rows.append(f"Settings: {line.strip()}")
|
||||
break
|
||||
except OSError as exc:
|
||||
rows.append(f"Settings read: {exc}")
|
||||
if BACKENDS_FDS.is_file():
|
||||
try:
|
||||
text = BACKENDS_FDS.read_text(encoding="utf-8", errors="replace")
|
||||
rows.append(f"Backends.fds size={len(text)}b head:\n" + "\n".join(text.splitlines()[:40]))
|
||||
except OSError as exc:
|
||||
rows.append(f"Backends read: {exc}")
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def collect() -> str:
|
||||
ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
parts = [
|
||||
f"gpu-rent swarm diagnostics @ {ts}",
|
||||
section("systemctl swarmui", _run(["systemctl", "is-active", "swarmui"])),
|
||||
section("nvidia-smi", _run(["nvidia-smi", "-L"])),
|
||||
section("paths", paths_bits()),
|
||||
section("SwarmUI API", swarm_api_bits()),
|
||||
section(
|
||||
"journalctl -u swarmui -n 80",
|
||||
_run(
|
||||
["journalctl", "-u", "swarmui", "-n", "80", "--no-pager", "-o", "short-iso"],
|
||||
timeout=30.0,
|
||||
),
|
||||
),
|
||||
section(
|
||||
"journalctl -u swarmui priority=err..alert -n 40",
|
||||
_run(
|
||||
[
|
||||
"journalctl",
|
||||
"-u",
|
||||
"swarmui",
|
||||
"-p",
|
||||
"err",
|
||||
"-n",
|
||||
"40",
|
||||
"--no-pager",
|
||||
"-o",
|
||||
"short-iso",
|
||||
],
|
||||
timeout=30.0,
|
||||
),
|
||||
),
|
||||
]
|
||||
# Comfy often logs under dlbackend
|
||||
for cand in (
|
||||
COMFY / "user" / "comfyui.log",
|
||||
COMFY / "comfyui.log",
|
||||
DATA / "Logs" / "recent.log",
|
||||
):
|
||||
if cand.is_file():
|
||||
try:
|
||||
text = cand.read_text(encoding="utf-8", errors="replace")
|
||||
parts.append(section(f"file {cand}", "\n".join(text.splitlines()[-60:])))
|
||||
except OSError:
|
||||
pass
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
report = collect()
|
||||
try:
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUT.write_text(report + "\n", encoding="utf-8")
|
||||
print(f"DIAG saved → {OUT}", flush=True)
|
||||
except OSError as exc:
|
||||
print(f"DIAG save fail: {exc}", flush=True)
|
||||
print(report, flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -161,11 +161,18 @@ def _bind_access(
|
||||
ensure_swarm_comfy_installed(cfg, ip, log)
|
||||
except Exception as exc:
|
||||
log(f"SwarmUI Comfy install: {exc}")
|
||||
try:
|
||||
from gpu_rent.ready import collect_swarm_diagnostics
|
||||
|
||||
collect_swarm_diagnostics(cfg, ip, log)
|
||||
except Exception as diag_exc:
|
||||
log(f"diag: {diag_exc}")
|
||||
raise
|
||||
clock.mark("comfy-install", log)
|
||||
try:
|
||||
wait_backend_idle(cfg, ip, log)
|
||||
except CloudError as exc:
|
||||
# wait_backend_idle already collected diag on errored/timeout
|
||||
log(f"ready: {exc}")
|
||||
raise
|
||||
clock.mark("Idle", log)
|
||||
|
||||
@@ -37,6 +37,47 @@ def test_install_swarm_comfy_script_payload():
|
||||
assert "modern_dark" in text
|
||||
assert "backends present (idle/suspended)" in text
|
||||
assert ".gpu-rent-comfy-installing" in text
|
||||
assert "RestartBackends" in text
|
||||
assert 'bstat == "errored"' in text
|
||||
|
||||
|
||||
def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent import ready
|
||||
|
||||
class Cfg:
|
||||
pass
|
||||
|
||||
def fake_ssh(*a, **k):
|
||||
return "BUSY backend=errored"
|
||||
|
||||
diag_calls = {"n": 0}
|
||||
|
||||
def fake_diag(*a, **k):
|
||||
diag_calls["n"] += 1
|
||||
return "DIAG ok"
|
||||
|
||||
monkeypatch.setattr(ready, "run_ssh", fake_ssh)
|
||||
monkeypatch.setattr(ready, "collect_swarm_diagnostics", fake_diag)
|
||||
try:
|
||||
ready.wait_backend_idle(
|
||||
Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.01, errored_fail_sec=0.05
|
||||
)
|
||||
assert False, "expected CloudError"
|
||||
except CloudError as exc:
|
||||
assert "errored" in str(exc).lower()
|
||||
assert "диагностик" in str(exc).lower() or "diag" in str(exc).lower()
|
||||
assert diag_calls["n"] == 1
|
||||
|
||||
|
||||
def test_swarm_diag_script_covers_api_and_journal():
|
||||
from importlib.resources import files
|
||||
|
||||
text = files("gpu_rent.remote").joinpath("swarm_diag.py").read_text(encoding="utf-8")
|
||||
assert "ListBackends" in text
|
||||
assert "journalctl" in text
|
||||
assert ".gpu-rent-last-diag.txt" in text
|
||||
assert "nvidia-smi" in text
|
||||
|
||||
|
||||
def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
|
||||
|
||||
Reference in New Issue
Block a user