Refactor backend status handling and improve idle management

- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning.
- Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback.
- Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process.
- Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection.
- Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.

src/gpu_rent/remote/idle_killer.py

@@ -102,13 +102,29 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
     any_loading = bool(backend.get("any_loading"))
     if waiting or live or loading:
         return True, f"queue waiting={waiting} live={live} loading={loading}"
-    # SwarmUI "running" = healthy ready (no gens) — not busy for billing.
-    # "loading" / "some_loading" = still starting — keep GPU.
+    # First-install / no backends yet — do not start idle clock (Comfy install 15–40+ min).
+    if bstat in {"empty", "unknown"}:
+        return True, f"backend={bstat}"
     if bstat in {"loading", "some_loading"} or any_loading:
         return True, f"backend={bstat}"
     if bstat == "errored":
         return True, f"backend={bstat}"
-    # running / idle / disabled / empty / unknown with empty queue → allow idle clock
+    # Marker while InstallConfirmWS / comfy-install-linux.sh runs.
+    install_marker = DATA / ".gpu-rent-comfy-installing"
+    if install_marker.is_file():
+        try:
+            ts = float(install_marker.read_text(encoding="utf-8").strip().split()[0])
+            age = time.time() - ts
+        except (OSError, ValueError, IndexError):
+            age = 0.0
+        max_age = 3 * 60 * 60
+        if age <= max_age:
+            return True, f"comfy installing ({int(age)}s)"
+        try:
+            install_marker.unlink(missing_ok=True)
+        except OSError:
+            pass
+    # running / idle / disabled / all_disabled with empty queue → allow idle clock
     return False, f"idle backend={bstat}"
 
 
@@ -122,8 +138,8 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
         except (OSError, ValueError, IndexError):
             age = 0.0
             ts = 0.0
-        # Stale marker after SSH kill / crash — don't block billing forever.
-        max_age = 45 * 60
+        # Must be ≥ ollama_pull HTTP timeout (7200s); refresh keeps marker fresh during pull.
+        max_age = 3 * 60 * 60
         if age > max_age:
             try:
                 pull_marker.unlink(missing_ok=True)

src/gpu_rent/remote/install_swarm_comfy.py

@@ -452,11 +452,11 @@ def main() -> int:
     print(f"backend_status={bstat} venv={'yes' if comfy_venv_ok() else 'no'} "
           f"IsInstalled={settings_is_installed()}")
 
+    # Backends already registered (any non-empty status) + venv → skip InstallConfirmWS.
     if bstat == "idle":
-        print("Comfy backend already Idle — skip install")
+        print("backends present (idle/suspended) + skip install")
         return 0
     if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
-        # loading / running / etc. — installer not needed
         if comfy_venv_ok():
             print(f"backends present ({bstat}) + venv — skip install")
             return 0
@@ -471,19 +471,28 @@ def main() -> int:
         return 1
 
     if installed is True and comfy_venv_ok():
-        print("IsInstalled + venv — skip InstallConfirmWS (ждём Idle отдельно)")
+        print("IsInstalled + venv — skip InstallConfirmWS (ждём ready отдельно)")
         return 0
 
     if comfy_venv_ok() and bstat == "empty":
-        # Partial: script ran but backend never registered — still need install
-        # only if IsInstalled is false; otherwise user must add backend in UI.
         if installed is not True:
             print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
         else:
             return 1
 
-    sid = get_session(time.time() + 120)
-    run_install(sid)
+    marker = DATA / ".gpu-rent-comfy-installing"
+    try:
+        marker.write_text(f"{int(time.time())}\n", encoding="utf-8")
+    except OSError:
+        pass
+    try:
+        sid = get_session(time.time() + 120)
+        run_install(sid)
+    finally:
+        try:
+            marker.unlink(missing_ok=True)
+        except OSError:
+            pass
 
     # Confirm outcome
     time.sleep(3)

src/gpu_rent/remote/ollama_pull.py

@@ -77,6 +77,7 @@ def pull_stream(name: str, label: str) -> None:
     )
     t0 = time.monotonic()
     last_print = 0.0
+    last_touch = 0.0
     with urllib.request.urlopen(req, timeout=7200) as resp:
         while True:
             raw = resp.readline()
@@ -92,6 +93,13 @@ def pull_stream(name: str, label: str) -> None:
             total = int(ev.get("total") or 0) or None
             done = int(ev.get("completed") or 0)
             now = time.monotonic()
+            # Keep idle-killer marker fresh for long pulls (max-age 3h).
+            if now - last_touch >= 60.0:
+                try:
+                    MARKER.write_text(f"{int(time.time())}\n", encoding="utf-8")
+                except OSError:
+                    pass
+                last_touch = now
             if total and done:
                 if now - last_print >= 1.0 or done >= total:
                     elapsed = max(now - t0, 0.001)

src/gpu_rent/remote/stack_env_probe.py

@@ -99,13 +99,13 @@ def check_torch(py: Path) -> dict:
         " print(json.dumps({'ok':False,'detail':f'import torch: {e}'})); sys.exit(0)\n"
         "cuda=bool(torch.cuda.is_available())\n"
         "dev=torch.cuda.get_device_name(0) if cuda else ''\n"
-        "print(json.dumps({\n"
-        " 'ok': cuda,\n"
-        " 'detail': (\n"
-        "  f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
-        "  f'available={cuda} device={dev}'\n"
-        " ),\n"
-        "}))\n"
+        "if cuda:\n"
+        " detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
+        "         f'available=True device={dev}')\n"
+        "else:\n"
+        " detail=(f'torch={torch.__version__} cuda={torch.version.cuda} '\n"
+        "         f'available=false (CPU wheel / без cuda — не заживёт само)')\n"
+        "print(json.dumps({'ok': cuda, 'detail': detail}))\n"
     )
     code, out = run([str(py), "-c", script], timeout=120.0)
     line = ""