Refactor backend status handling and improve idle management

- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning.
- Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback.
- Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process.
- Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection.
- Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.

src/gpu_rent/tunnel.py

@@ -55,11 +55,31 @@ class WatchDecision:
     detail: str = ""
 
 
+def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
+    """Return (status, detail). Refreshes IAM token via connect().
+
+    On soft auth/API failure return SOFT_FAIL (not fake ACTIVE) so we do not
+    mask DELETED/ERROR forever.
+    """
+    try:
+        conn = connect(cfg)
+        server = pick_existing_server(conn)
+        if not server:
+            return None, "нет сервера"
+        return server_status(server), server.id
+    except GpuRentError as exc:
+        log(f"watch: OpenStack временно недоступен ({exc})")
+        return "SOFT_FAIL", "auth-soft-fail"
+
+
 def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
     """Pure policy for tunnel watchdog (unit-tested)."""
     if not status:
         return WatchDecision("exit", "нет сервера gpu-rent")
     st = status.upper()
+    if st == "SOFT_FAIL":
+        # Transient OpenStack blip — keep tunnel, do not pretend ACTIVE forever.
+        return WatchDecision("ok", "openstack soft-fail")
     if st in EXIT_STATUSES:
         return WatchDecision("exit", f"Nova {st}")
     if st in SHELVED_STATUSES:
@@ -146,19 +166,6 @@ def _recover_unshelve(cfg: Config, log: Log) -> str:
     return ip
 
 
-def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]:
-    """Return (status, detail). Refreshes IAM token via connect()."""
-    try:
-        conn = connect(cfg)
-        server = pick_existing_server(conn)
-        if not server:
-            return None, "нет сервера"
-        return server_status(server), server.id
-    except GpuRentError as exc:
-        log(f"watch: OpenStack временно недоступен ({exc})")
-        return "ACTIVE", "auth-soft-fail"
-
-
 def run_tunnel(
     cfg: Config,
     host: str,