Refactor LLM runtime handling and enhance CLI documentation

- Updated `resolve_llm_runtime` to prioritize live configuration over legacy notes, ensuring accurate runtime resolution.
- Enhanced `tunnel_forwards` to prefer current configuration for LLM runtime, improving tunnel setup logic.
- Improved idle-killer logic to handle stale markers and provide clearer warnings in the status output.
- Updated CLI documentation in `cli.md` to reflect changes in command behavior and runtime handling.
- Enhanced tests to validate new runtime resolution logic and ensure proper handling of configuration states.

src/gpu_rent/remote/idle_killer.py

@@ -90,8 +90,24 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]:
 
 def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
     """Ollama pull / loaded models or llama.cpp with a model count as busy."""
-    if (DATA / ".gpu-rent-ollama-pulling").is_file():
-        return True, "ollama pulling"
+    pull_marker = DATA / ".gpu-rent-ollama-pulling"
+    if pull_marker.is_file():
+        try:
+            ts = float(pull_marker.read_text(encoding="utf-8").strip().split()[0])
+            age = time.time() - ts
+        except (OSError, ValueError, IndexError):
+            age = 0.0
+            ts = 0.0
+        # Stale marker after SSH kill / crash — don't block billing forever.
+        max_age = 45 * 60
+        if age > max_age:
+            try:
+                pull_marker.unlink(missing_ok=True)
+            except OSError:
+                pass
+            log(f"cleared stale ollama-pulling marker age={int(age)}s")
+        else:
+            return True, f"ollama pulling ({int(age)}s)"
     ctx = ssl.create_default_context()
     # Ollama: any running model
     try:
@@ -104,21 +120,18 @@ def llm_busy(timeout: float = 3.0) -> tuple[bool, str]:
             return True, f"ollama running {names}"
     except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, OSError):
         pass
-    # llama.cpp OpenAI models endpoint — if server up and lists a model, treat lightly:
-    # only busy if /health ok AND we recently had activity is hard; use loaded via props.
+    # llama.cpp: slots in use
     try:
         req = urllib.request.Request("http://127.0.0.1:8080/health", method="GET")
         with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
             if getattr(resp, "status", 200) == 200:
-                # Server alive with a model is OK for idle unless slots busy — skip kill only
-                # when props show n_slots_in_use if available.
                 try:
                     req2 = urllib.request.Request("http://127.0.0.1:8080/props", method="GET")
                     with urllib.request.urlopen(req2, timeout=timeout, context=ctx) as resp2:
                         props = json.loads(resp2.read().decode("utf-8"))
-                    in_use = int(props.get("total_slots") or 0) - int(
-                        props.get("available_slots") or props.get("total_slots") or 0
-                    )
+                    total = int(props.get("total_slots") or 0)
+                    avail = int(props.get("available_slots") or total)
+                    in_use = total - avail if total else 0
                     if in_use > 0:
                         return True, f"llamacpp slots_in_use={in_use}"
                 except Exception: