Implement GPU environment verification in session management

- Added a new function `verify_gpu_env` to check GPU stack readiness, including nvidia-smi, CUDA, and torch in the Comfy virtual environment when SwarmUI is enabled.
- Updated the session management to call `verify_gpu_env`, capturing GPU environment status and errors in the state notes.
- Enhanced documentation in `cli.md` to reflect the new GPU environment verification process.
- Added tests for `verify_gpu_env` to ensure proper functionality and error handling during GPU checks.
This commit is contained in:
Leonid Pershin
2026-08-21 06:58:23 +03:00
parent 09b7c36f3b
commit 3c8225a69e
7 changed files with 488 additions and 14 deletions
+13 -1
View File
@@ -22,7 +22,7 @@ from gpu_rent.cloud import (
)
from gpu_rent.bootstrap import run_bootstrap
from gpu_rent.provision import provision_vm, tune_swarm_perf
from gpu_rent.ready import verify_stack_on_vm, wait_backend_idle
from gpu_rent.ready import verify_gpu_env, verify_stack_on_vm, wait_backend_idle
from gpu_rent.snapshot import ensure_boot_snapshot
from gpu_rent.notify import notify_ready
from gpu_rent.config import Config
@@ -140,6 +140,18 @@ def _bind_access(
save_state(state)
raise
try:
gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
state.notes = dict(state.notes or {})
state.notes["gpu_env"] = [
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
]
except CloudError as exc:
state.notes = dict(state.notes or {})
state.notes["gpu_env_error"] = str(exc)[:500]
save_state(state)
raise
try:
ensure_boot_snapshot(
conn,