Refactor GPU environment verification tests and update CLI documentation
- Rearranged the order of verification steps in the CLI documentation for clarity. - Enhanced the `test_verify_gpu_env` tests to improve logging and error assertions, ensuring accurate feedback on GPU stack checks. - Added a new test to skip the torch requirement when only LLM is enabled, reflecting updated behavior in GPU environment verification.
This commit is contained in:
+3
-3
@@ -231,9 +231,9 @@ Application credential для idle-killer CLI создаёт на `up` (узко
|
|||||||
|
|
||||||
1. Nova `ACTIVE`
|
1. Nova `ACTIVE`
|
||||||
2. TCP 22 / SSH
|
2. TCP 22 / SSH
|
||||||
3. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434` / llama.cpp `:8080`) — `verify_stack_on_vm`
|
3. Backend Idle (если SwarmUI) → toast (если `NOTIFY_READY`)
|
||||||
4. На VM: **nvidia-smi / CUDA** (+ **torch+cuda** в Comfy venv, если SwarmUI) — `verify_gpu_env`
|
4. На VM: HTTP сервисов стека (SwarmUI `:7801` / Ollama `:11434` / llama.cpp `:8080`) — `verify_stack_on_vm`
|
||||||
5. Backend Idle (если SwarmUI) → toast (если `NOTIFY_READY`)
|
5. На VM: **nvidia-smi / CUDA** (+ **torch+cuda** в Comfy venv, если SwarmUI) — `verify_gpu_env`
|
||||||
6. Туннель + проверка **localhost** тех же сервисов → access-card
|
6. Туннель + проверка **localhost** тех же сервисов → access-card
|
||||||
|
|
||||||
Локальный порт UI: **17801** (на VM по-прежнему 7801 на loopback).
|
Локальный порт UI: **17801** (на VM по-прежнему 7801 на loopback).
|
||||||
|
|||||||
+41
-27
@@ -59,6 +59,10 @@ def test_service_check_dataclass():
|
|||||||
|
|
||||||
|
|
||||||
def test_verify_gpu_env_ok(monkeypatch):
|
def test_verify_gpu_env_ok(monkeypatch):
|
||||||
|
import json
|
||||||
|
|
||||||
|
import gpu_rent.ssh_ops as ssh_ops
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"checks": [
|
"checks": [
|
||||||
@@ -83,34 +87,12 @@ def test_verify_gpu_env_ok(monkeypatch):
|
|||||||
assert "GPU_RENT_CHECK_SWARM" in script
|
assert "GPU_RENT_CHECK_SWARM" in script
|
||||||
return json.dumps(payload) + "\n"
|
return json.dumps(payload) + "\n"
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
monkeypatch.setattr("gpu_rent.ssh_ops.run_python", fake_run_python)
|
|
||||||
# patch where used
|
|
||||||
monkeypatch.setattr(
|
|
||||||
"gpu_rent.ssh_ops.run_python",
|
|
||||||
fake_run_python,
|
|
||||||
raising=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
import gpu_rent.ready as ready_mod
|
|
||||||
|
|
||||||
monkeypatch.setattr(
|
|
||||||
ready_mod,
|
|
||||||
"run_python",
|
|
||||||
fake_run_python,
|
|
||||||
raising=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
# verify_gpu_env imports run_python inside the function
|
|
||||||
import gpu_rent.ssh_ops as ssh_ops
|
|
||||||
|
|
||||||
monkeypatch.setattr(ssh_ops, "run_python", fake_run_python)
|
monkeypatch.setattr(ssh_ops, "run_python", fake_run_python)
|
||||||
|
|
||||||
logs: list[str] = []
|
logs: list[str] = []
|
||||||
out = verify_gpu_env(_Cfg(), "1.2.3.4", logs.append, timeout=5.0, poll_every=0.1)
|
out = verify_gpu_env(_Cfg(), "1.2.3.4", logs.append, timeout=5.0, poll_every=0.1)
|
||||||
assert all(c.ok for c in out)
|
assert all(c.ok for c in out)
|
||||||
assert any(c.name == "torch" for c in out)
|
assert any(c.name == "torch" for c in out)
|
||||||
|
assert any("GPU-стека" in line for line in logs)
|
||||||
|
|
||||||
|
|
||||||
def test_verify_gpu_env_fails_without_cuda(monkeypatch):
|
def test_verify_gpu_env_fails_without_cuda(monkeypatch):
|
||||||
@@ -133,9 +115,41 @@ def test_verify_gpu_env_fails_without_cuda(monkeypatch):
|
|||||||
)
|
)
|
||||||
logs: list[str] = []
|
logs: list[str] = []
|
||||||
try:
|
try:
|
||||||
verify_gpu_env(
|
verify_gpu_env(_Cfg(), "1.2.3.4", logs.append, timeout=0.4, poll_every=0.1)
|
||||||
_Cfg(), "1.2.3.4", logs.append, timeout=0.4, poll_every=0.1
|
|
||||||
)
|
|
||||||
assert False, "expected CloudError"
|
assert False, "expected CloudError"
|
||||||
except CloudError as exc:
|
except CloudError as exc:
|
||||||
assert "cuda" in str(exc).lower() or "GPU-стек" in str(exc)
|
assert "GPU-стек" in str(exc) or "cuda" in str(exc).lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_verify_gpu_env_llm_only_skips_torch_requirement(monkeypatch):
|
||||||
|
import json
|
||||||
|
|
||||||
|
import gpu_rent.ssh_ops as ssh_ops
|
||||||
|
|
||||||
|
class C:
|
||||||
|
enable_swarmui = False
|
||||||
|
llm_runtime = "llamacpp"
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"ok": True,
|
||||||
|
"checks": [
|
||||||
|
{"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"},
|
||||||
|
{"name": "cuda", "required": True, "ok": True, "detail": "ok"},
|
||||||
|
{
|
||||||
|
"name": "torch",
|
||||||
|
"required": False,
|
||||||
|
"ok": True,
|
||||||
|
"detail": "skip (llm-only, без Comfy venv)",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
def fake_run_python(cfg, host, script, **kw):
|
||||||
|
assert "GPU_RENT_CHECK_SWARM" in script
|
||||||
|
assert "os.environ['GPU_RENT_CHECK_SWARM']='0'" in script
|
||||||
|
return json.dumps(payload)
|
||||||
|
|
||||||
|
monkeypatch.setattr(ssh_ops, "run_python", fake_run_python)
|
||||||
|
logs: list[str] = []
|
||||||
|
out = verify_gpu_env(C(), "1.2.3.4", logs.append, timeout=5.0)
|
||||||
|
assert all(c.ok for c in out)
|
||||||
|
|||||||
Reference in New Issue
Block a user