Implement GPU environment verification in session management
- Added a new function `verify_gpu_env` to check GPU stack readiness, including nvidia-smi, CUDA, and torch in the Comfy virtual environment when SwarmUI is enabled. - Updated the session management to call `verify_gpu_env`, capturing GPU environment status and errors in the state notes. - Enhanced documentation in `cli.md` to reflect the new GPU environment verification process. - Added tests for `verify_gpu_env` to ensure proper functionality and error handling during GPU checks.
This commit is contained in:
+13
-1
@@ -22,7 +22,7 @@ from gpu_rent.cloud import (
|
||||
)
|
||||
from gpu_rent.bootstrap import run_bootstrap
|
||||
from gpu_rent.provision import provision_vm, tune_swarm_perf
|
||||
from gpu_rent.ready import verify_stack_on_vm, wait_backend_idle
|
||||
from gpu_rent.ready import verify_gpu_env, verify_stack_on_vm, wait_backend_idle
|
||||
from gpu_rent.snapshot import ensure_boot_snapshot
|
||||
from gpu_rent.notify import notify_ready
|
||||
from gpu_rent.config import Config
|
||||
@@ -140,6 +140,18 @@ def _bind_access(
|
||||
save_state(state)
|
||||
raise
|
||||
|
||||
try:
|
||||
gpu_checks = verify_gpu_env(cfg, ip, log, timeout=600.0)
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["gpu_env"] = [
|
||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in gpu_checks
|
||||
]
|
||||
except CloudError as exc:
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["gpu_env_error"] = str(exc)[:500]
|
||||
save_state(state)
|
||||
raise
|
||||
|
||||
try:
|
||||
ensure_boot_snapshot(
|
||||
conn,
|
||||
|
||||
Reference in New Issue
Block a user