Implement GPU environment verification in session management

- Added a new function `verify_gpu_env` to check GPU stack readiness, including nvidia-smi, CUDA, and torch in the Comfy virtual environment when SwarmUI is enabled.
- Updated the session management to call `verify_gpu_env`, capturing GPU environment status and errors in the state notes.
- Enhanced documentation in `cli.md` to reflect the new GPU environment verification process.
- Added tests for `verify_gpu_env` to ensure proper functionality and error handling during GPU checks.
This commit is contained in:
Leonid Pershin
2026-08-21 06:58:23 +03:00
parent 09b7c36f3b
commit 3c8225a69e
7 changed files with 488 additions and 14 deletions
+88 -2
View File
@@ -1,4 +1,5 @@
from gpu_rent.ready import ServiceCheck, _expected_services, verify_stack_local
from gpu_rent.errors import CloudError
from gpu_rent.ready import ServiceCheck, _expected_services, verify_gpu_env, verify_stack_local
class _Cfg:
@@ -41,7 +42,9 @@ def test_verify_stack_local_fails_closed_port(monkeypatch):
ollama_local_port = 17999
llamacpp_local_port = 17812
monkeypatch.setattr("gpu_rent.ready._tcp_ok", lambda port, host="127.0.0.1", timeout=0.8: False)
monkeypatch.setattr(
"gpu_rent.ready._tcp_ok", lambda port, host="127.0.0.1", timeout=0.8: False
)
logs: list[str] = []
try:
verify_stack_local(C(), logs.append, timeout=0.3, poll_every=0.1)
@@ -53,3 +56,86 @@ def test_verify_stack_local_fails_closed_port(monkeypatch):
def test_service_check_dataclass():
c = ServiceCheck("x", True, "ok", "vm")
assert c.ok and c.where == "vm"
def test_verify_gpu_env_ok(monkeypatch):
payload = {
"ok": True,
"checks": [
{"name": "nvidia-smi", "required": True, "ok": True, "detail": "A100"},
{"name": "cuda", "required": True, "ok": True, "detail": "libcuda"},
{
"name": "torch",
"required": True,
"ok": True,
"detail": "torch=2.0 cuda=12 available=True",
},
{
"name": "triton/sage",
"required": False,
"ok": True,
"detail": "есть: triton; нет: —",
},
],
}
def fake_run_python(cfg, host, script, **kw):
assert "GPU_RENT_CHECK_SWARM" in script
return json.dumps(payload) + "\n"
import json
monkeypatch.setattr("gpu_rent.ssh_ops.run_python", fake_run_python)
# patch where used
monkeypatch.setattr(
"gpu_rent.ssh_ops.run_python",
fake_run_python,
raising=False,
)
import gpu_rent.ready as ready_mod
monkeypatch.setattr(
ready_mod,
"run_python",
fake_run_python,
raising=False,
)
# verify_gpu_env imports run_python inside the function
import gpu_rent.ssh_ops as ssh_ops
monkeypatch.setattr(ssh_ops, "run_python", fake_run_python)
logs: list[str] = []
out = verify_gpu_env(_Cfg(), "1.2.3.4", logs.append, timeout=5.0, poll_every=0.1)
assert all(c.ok for c in out)
assert any(c.name == "torch" for c in out)
def test_verify_gpu_env_fails_without_cuda(monkeypatch):
import json
import gpu_rent.ssh_ops as ssh_ops
payload = {
"ok": False,
"checks": [
{"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"},
{"name": "cuda", "required": True, "ok": False, "detail": "нет libcuda"},
{"name": "torch", "required": True, "ok": False, "detail": "no venv"},
],
}
monkeypatch.setattr(
ssh_ops,
"run_python",
lambda *a, **k: json.dumps(payload),
)
logs: list[str] = []
try:
verify_gpu_env(
_Cfg(), "1.2.3.4", logs.append, timeout=0.4, poll_every=0.1
)
assert False, "expected CloudError"
except CloudError as exc:
assert "cuda" in str(exc).lower() or "GPU-стек" in str(exc)