Refactor backend status handling and improve idle management

- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning.
- Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback.
- Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process.
- Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection.
- Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.

tests/test_hold_killer.py

@@ -147,6 +147,42 @@ def test_classify_loading_is_busy(monkeypatch):
     assert "loading" in detail
 
 
+def test_classify_empty_is_busy(monkeypatch):
+    """No backends yet (first Comfy install) must not start idle clock."""
+    mod = _load_remote()
+
+    class FakeResp:
+        def __init__(self, payload):
+            self._payload = payload
+
+        def read(self):
+            import json
+
+            return json.dumps(self._payload).encode()
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *args):
+            return False
+
+    def fake_urlopen(req, timeout=0, context=None):
+        url = getattr(req, "full_url", None) or req.get_full_url()
+        if "GetNewSession" in url:
+            return FakeResp({"session_id": "abc"})
+        return FakeResp(
+            {
+                "status": {"waiting_gens": 0, "live_gens": 0, "loading_models": 0},
+                "backend_status": {"status": "empty"},
+            }
+        )
+
+    monkeypatch.setattr(mod.urllib.request, "urlopen", fake_urlopen)
+    busy, detail = mod.swarm_busy("http://127.0.0.1:7801")
+    assert busy is True
+    assert "empty" in detail
+
+
 def test_classify_busy_queue(monkeypatch):
     mod = _load_remote()
 

tests/test_tunnel_watch.py

@@ -27,30 +27,34 @@ def test_decide_exit_missing():
     assert d.kind == "exit"
 
 
-def test_tunnel_forwards_swarm_only(monkeypatch):
+def test_decide_soft_fail_keeps_tunnel():
+    d = decide_watch("SOFT_FAIL", tunnel_alive=True)
+    assert d.kind == "ok"
+    assert "soft-fail" in d.detail
+
+
+def test_tunnel_forwards_swarm_only():
     class Cfg:
         swarmui_local_port = 17801
         llm_runtime = "none"
         ollama_local_port = 17811
+        enable_swarmui = True
 
-    monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})())
     assert tunnel_forwards(Cfg()) == [(17801, 7801)]
 
 
-def test_tunnel_forwards_prefers_cfg_over_stale_notes(monkeypatch):
+def test_tunnel_forwards_prefers_cfg_over_stale_notes():
+    """tunnel_forwards uses cfg only — notes must not add Ollama."""
     class Cfg:
         swarmui_local_port = 17801
         llm_runtime = "none"
         ollama_local_port = 17811
+        enable_swarmui = True
 
-    monkeypatch.setattr(
-        "gpu_rent.tunnel.load_state",
-        lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
-    )
     assert tunnel_forwards(Cfg()) == [(17801, 7801)]
 
 
-def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
+def test_resolve_llm_uses_cfg_only(monkeypatch):
     from gpu_rent.access_card import resolve_llm_runtime
 
     class Cfg:
@@ -60,13 +64,10 @@ def test_resolve_llm_notes_only_when_cfg_none(monkeypatch):
         "gpu_rent.access_card.load_state",
         lambda: type("S", (), {"notes": {"llm_runtime": "ollama"}})(),
     )
-    assert resolve_llm_runtime(Cfg()) == "ollama"
+    # Stale notes must not override live cfg=none
+    assert resolve_llm_runtime(Cfg()) == "none"
 
     class Cfg2:
         llm_runtime = "ollama"
 
-    monkeypatch.setattr(
-        "gpu_rent.access_card.load_state",
-        lambda: type("S", (), {"notes": {"llm_runtime": "none"}})(),
-    )
     assert resolve_llm_runtime(Cfg2()) == "ollama"

tests/test_wait_backend_idle.py

@@ -9,11 +9,15 @@ def test_remote_poll_empty_is_busy_not_ready():
 
 
 def test_remote_poll_running_is_ready():
-    """SwarmUI: running = healthy ready; idle = suspended (cannot generate)."""
+    """SwarmUI: running = healthy ready."""
     assert 'bstat == "running"' in _REMOTE_POLL
     assert "READY backend=running" in _REMOTE_POLL
-    assert "READY backend=idle" not in _REMOTE_POLL
-    assert "BUSY backend=idle" in _REMOTE_POLL
+
+
+def test_remote_poll_idle_suspended_is_ready():
+    """Suspended idle backends are installed — ready for up (wake on gen)."""
+    assert "READY backend=idle" in _REMOTE_POLL
+    assert "BUSY backend=idle" not in _REMOTE_POLL
 
 
 def test_remote_poll_loading_is_busy():
@@ -31,9 +35,8 @@ def test_install_swarm_comfy_script_payload():
     assert '"backend": "comfyui"' in text
     assert '"models": "none"' in text
     assert "modern_dark" in text
-    assert "detect_stage" in text
-    assert "dlbackend=" in text
-    assert 'end="\\r"' in text or 'end="\\r"' in text
+    assert "backends present (idle/suspended)" in text
+    assert ".gpu-rent-comfy-installing" in text
 
 
 def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
@@ -73,3 +76,41 @@ def test_verify_gpu_env_fail_fast_empty_dlbackend(monkeypatch):
     except CloudError as exc:
         assert "fail-fast" in str(exc).lower() or "torch" in str(exc).lower()
     assert calls["n"] == 1
+
+
+def test_verify_gpu_env_fail_fast_torch_no_cuda(monkeypatch):
+    import json
+
+    from gpu_rent.errors import CloudError
+    from gpu_rent.ready import verify_gpu_env
+    import gpu_rent.ssh_ops as ssh_ops
+
+    class Cfg:
+        enable_swarmui = True
+
+    payload = {
+        "ok": False,
+        "checks": [
+            {"name": "nvidia-smi", "required": True, "ok": True, "detail": "ok"},
+            {"name": "cuda", "required": True, "ok": True, "detail": "ok"},
+            {
+                "name": "torch",
+                "required": True,
+                "ok": False,
+                "detail": "torch=2.0 cuda=None available=false (CPU wheel / без cuda — не заживёт само)",
+            },
+        ],
+    }
+    calls = {"n": 0}
+
+    def fake(*a, **k):
+        calls["n"] += 1
+        return json.dumps(payload)
+
+    monkeypatch.setattr(ssh_ops, "run_python", fake)
+    try:
+        verify_gpu_env(Cfg(), "1.2.3.4", [].append, timeout=600.0, poll_every=0.1)
+        assert False, "expected CloudError"
+    except CloudError:
+        pass
+    assert calls["n"] == 1