Refactor backend status handling and improve idle management

- Updated the idle-killer logic to treat SwarmUI `empty` and `disabled` states as busy, preventing unnecessary idle time during provisioning.
- Enhanced the `wait_backend_idle` function to recognize suspended backends as ready, improving resource utilization and user feedback.
- Refined the `install_swarm_comfy` script to skip installation when backends are already present, streamlining the setup process.
- Improved the `resolve_llm_runtime` function to prioritize live configuration over stale state notes, ensuring accurate runtime detection.
- Added tests to validate the new backend status handling and idle management logic, ensuring robustness and reliability.

src/gpu_rent/provision.py

@@ -234,18 +234,109 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool:
             applied = bool(json.loads(raw).get("settings_applied"))
         except json.JSONDecodeError:
             applied = False
-    if not remote_exists(cfg, host, settings) or not applied:
-        fds = (
-            "DefaultUser:\n"
-            "    AutoComplete:\n"
-            f"        Source: {cfg.autocomplete_filename}\n"
-            "        EscapeParens: true\n"
-        )
-        put_text(cfg, host, settings, fds)
-        log(f"Settings.fds AutoComplete.Source = {cfg.autocomplete_filename}")
+    if not applied:
+        _merge_autocomplete_into_settings(cfg, host, settings, cfg.autocomplete_filename, log)
+        if remote_exists(cfg, host, meta_path):
+            try:
+                meta_obj = json.loads(
+                    run_ssh(cfg, host, f"cat {meta_path}", check=False)
+                )
+            except json.JSONDecodeError:
+                meta_obj = {}
+            if isinstance(meta_obj, dict):
+                meta_obj["settings_applied"] = True
+                put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n")
     return changed
 
 
+_AUTOCOMPLETE_MERGE_PY = r'''
+#!/usr/bin/env python3
+"""Merge AutoComplete.Source into Settings.fds without wiping other keys."""
+from __future__ import annotations
+
+import os
+import re
+import sys
+from pathlib import Path
+
+p = Path(os.environ.get("GPU_RENT_SETTINGS_FDS") or "/mnt/swarm_data/Data/Settings.fds")
+fname = (os.environ.get("GPU_RENT_AUTOCOMPLETE_FILE") or "").strip()
+if not fname:
+    print("no GPU_RENT_AUTOCOMPLETE_FILE", file=sys.stderr)
+    raise SystemExit(1)
+
+block = (
+    "DefaultUser:\n"
+    "    AutoComplete:\n"
+    f"        Source: {fname}\n"
+    "        EscapeParens: true\n"
+)
+
+if not p.is_file():
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(block, encoding="utf-8")
+    print(f"created Settings.fds AutoComplete.Source={fname}")
+    raise SystemExit(0)
+
+text = p.read_text(encoding="utf-8", errors="replace")
+if re.search(rf"^\s*Source:\s*{re.escape(fname)}\s*$", text, re.M):
+    print(f"AutoComplete.Source already {fname}")
+    raise SystemExit(0)
+
+# Replace Source line if AutoComplete section exists
+new, n = re.subn(
+    r"(^[ \t]*Source:\s*).*$",
+    rf"\1{fname}",
+    text,
+    count=1,
+    flags=re.M,
+)
+if n and "AutoComplete" in text:
+    p.write_text(new, encoding="utf-8")
+    print(f"patched AutoComplete.Source={fname}")
+    raise SystemExit(0)
+
+if re.search(r"^DefaultUser:\s*$", text, re.M):
+    new = re.sub(
+        r"^(DefaultUser:\s*\n)",
+        (
+            r"\1    AutoComplete:\n"
+            f"        Source: {fname}\n"
+            "        EscapeParens: true\n"
+        ),
+        text,
+        count=1,
+        flags=re.M,
+    )
+    p.write_text(new, encoding="utf-8")
+    print(f"inserted AutoComplete under DefaultUser Source={fname}")
+else:
+    p.write_text(text.rstrip() + "\n\n" + block, encoding="utf-8")
+    print(f"appended DefaultUser.AutoComplete Source={fname}")
+'''
+
+
+def _merge_autocomplete_into_settings(
+    cfg: Config, host: str, settings_path: str, filename: str, log: Log
+) -> None:
+    """Patch AutoComplete.Source in Settings.fds without wiping the rest."""
+    out = run_python(
+        cfg,
+        host,
+        _AUTOCOMPLETE_MERGE_PY,
+        remote_path="/tmp/gpu-rent-patch_autocomplete.py",
+        timeout=60,
+        log=None,
+        env={
+            "GPU_RENT_SETTINGS_FDS": settings_path,
+            "GPU_RENT_AUTOCOMPLETE_FILE": filename,
+        },
+    )
+    for line in (out or "").splitlines():
+        if line.strip():
+            log(line.strip())
+
+
 def _download_url(host: str, version_id: int, file_info: dict) -> str:
     raw = str(file_info.get("downloadUrl") or "")
     if "civitai." in raw and "/api/download/" in raw: