Refactor GPU environment verification tests and update CLI documentation

- Rearranged the order of verification steps in the CLI documentation for clarity.
- Enhanced the `test_verify_gpu_env` tests to improve logging and error assertions, ensuring accurate feedback on GPU stack checks.
- Added a new test to skip the torch requirement when only LLM is enabled, reflecting updated behavior in GPU environment verification.
This commit is contained in:
Leonid Pershin
2026-08-21 07:00:19 +03:00
parent 3c8225a69e
commit d409e2e154
2 changed files with 44 additions and 30 deletions
+3 -3
View File
@@ -231,9 +231,9 @@ Application credential для idle-killer CLI создаёт на `up` (узко
1. Nova `ACTIVE` 1. Nova `ACTIVE`
2. TCP 22 / SSH 2. TCP 22 / SSH
3. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434` / llama.cpp `:8080`) — `verify_stack_on_vm` 3. Backend Idle (если SwarmUI) → toast (если `NOTIFY_READY`)
4. На VM: **nvidia-smi / CUDA** (+ **torch+cuda** в Comfy venv, если SwarmUI) — `verify_gpu_env` 4. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434` / llama.cpp `:8080`) — `verify_stack_on_vm`
5. Backend Idle (если SwarmUI) → toast (если `NOTIFY_READY`) 5. На VM: **nvidia-smi / CUDA** (+ **torch+cuda** в Comfy venv, если SwarmUI) — `verify_gpu_env`
6. Туннель + проверка **localhost** тех же сервисов → access-card 6. Туннель + проверка **localhost** тех же сервисов → access-card
Локальный порт UI: **17801** (на VM по-прежнему 7801 на loopback). Локальный порт UI: **17801** (на VM по-прежнему 7801 на loopback).
+41 -27
View File
@@ -59,6 +59,10 @@ def test_service_check_dataclass():
def test_verify_gpu_env_ok(monkeypatch): def test_verify_gpu_env_ok(monkeypatch):
import json
import gpu_rent.ssh_ops as ssh_ops
payload = { payload = {
"ok": True, "ok": True,
"checks": [ "checks": [
@@ -83,34 +87,12 @@ def test_verify_gpu_env_ok(monkeypatch):
assert "GPU_RENT_CHECK_SWARM" in script assert "GPU_RENT_CHECK_SWARM" in script
return json.dumps(payload) + "\n" return json.dumps(payload) + "\n"
import json
monkeypatch.setattr("gpu_rent.ssh_ops.run_python", fake_run_python)
# patch where used
monkeypatch.setattr(
"gpu_rent.ssh_ops.run_python",
fake_run_python,
raising=False,
)
import gpu_rent.ready as ready_mod
monkeypatch.setattr(
ready_mod,
"run_python",
fake_run_python,
raising=False,
)
# verify_gpu_env imports run_python inside the function
import gpu_rent.ssh_ops as ssh_ops
monkeypatch.setattr(ssh_ops, "run_python", fake_run_python) monkeypatch.setattr(ssh_ops, "run_python", fake_run_python)
logs: list[str] = [] logs: list[str] = []
out = verify_gpu_env(_Cfg(), "1.2.3.4", logs.append, timeout=5.0, poll_every=0.1) out = verify_gpu_env(_Cfg(), "1.2.3.4", logs.append, timeout=5.0, poll_every=0.1)
assert all(c.ok for c in out) assert all(c.ok for c in out)
assert any(c.name == "torch" for c in out) assert any(c.name == "torch" for c in out)
assert any("GPU-стека" in line for line in logs)
def test_verify_gpu_env_fails_without_cuda(monkeypatch): def test_verify_gpu_env_fails_without_cuda(monkeypatch):
@@ -133,9 +115,41 @@ def test_verify_gpu_env_fails_without_cuda(monkeypatch):
) )
logs: list[str] = [] logs: list[str] = []
try: try:
verify_gpu_env( verify_gpu_env(_Cfg(), "1.2.3.4", logs.append, timeout=0.4, poll_every=0.1)
_Cfg(), "1.2.3.4", logs.append, timeout=0.4, poll_every=0.1
)
assert False, "expected CloudError" assert False, "expected CloudError"
except CloudError as exc: except CloudError as exc:
assert "cuda" in str(exc).lower() or "GPU-стек" in str(exc) assert "GPU-стек" in str(exc) or "cuda" in str(exc).lower()
def test_verify_gpu_env_llm_only_skips_torch_requirement(monkeypatch):
import json
import gpu_rent.ssh_ops as ssh_ops
class C:
enable_swarmui = False
llm_runtime = "llamacpp"
payload = {
"ok": True,
"checks": [
{"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"},
{"name": "cuda", "required": True, "ok": True, "detail": "ok"},
{
"name": "torch",
"required": False,
"ok": True,
"detail": "skip (llm-only, без Comfy venv)",
},
],
}
def fake_run_python(cfg, host, script, **kw):
assert "GPU_RENT_CHECK_SWARM" in script
assert "os.environ['GPU_RENT_CHECK_SWARM']='0'" in script
return json.dumps(payload)
monkeypatch.setattr(ssh_ops, "run_python", fake_run_python)
logs: list[str] = []
out = verify_gpu_env(C(), "1.2.3.4", logs.append, timeout=5.0)
assert all(c.ok for c in out)