Enhance backend recovery and diagnostics in installation and tuning scripts

- Improved the `install_swarm_comfy` function to handle empty backend states more effectively, introducing recovery mechanisms and enhanced logging for better visibility.
- Updated the `tune_swarm_perf` function to always sanitize backend FDS corruption, ensuring consistent performance tuning.
- Added new tests to validate the functionality of backend recovery and FDS sanitization, ensuring robustness in backend management.

src/gpu_rent/remote/install_swarm_comfy.py

@@ -657,11 +657,21 @@ def main() -> int:
         return 0
 
     if comfy_venv_ok() and bstat == "empty":
-        if installed is not True:
+        print(
-            print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
+            "backend empty при venv/IsInstalled — recover (sanitize FDS / AddNewBackend)…",
+            flush=True,
+        )
+        bstat = recover_empty_backends()
+        if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
+            print(f"recovered empty → {bstat}")
+            return 0
+        if installed is True:
+            print(
+                "WARN: всё ещё empty после recover — пробую InstallConfirmWS",
+                flush=True,
+            )
         else:
-            run_diagnostics()
+            print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
-            return 1
 
     marker = DATA / ".gpu-rent-comfy-installing"
     try:

src/gpu_rent/remote/tune_swarm_perf.py

@@ -206,17 +206,21 @@ def main() -> int:
         except json.JSONDecodeError:
             prev = {}
     same_gpu = bool(prev.get("uuid") and prev.get("uuid") == plan["uuid"])
+    # Always repair FDS corruption even when tune is otherwise a no-op.
+    fds_fixed = sanitize_backends_fds()
     if same_gpu and prev.get("extra_args") == plan["extra_args"]:
         if prev.get("pip_ok") or not plan["use_sage"]:
             print(f"perf tune already applied for {plan['name']} ({plan['tier']})")
+            if fds_fixed:
+                print("RESTART_SWARMUI=1")
             return 0
 
     if same_gpu and plan["use_sage"] and prev.get("pip_ok") is False:
         print("perf tune: retry (previous pip_ok=false — sage/triton ещё не встали)")
 
     print(f"perf tune: {plan['name']} tier={plan['tier']} sage={plan['use_sage']}")
+    restarted_needed = bool(fds_fixed)
     pip_ok = not plan["use_sage"]
-    restarted_needed = False
     if plan["use_sage"]:
         py = find_comfy_python()
         if py:

tests/test_tune_swarm_perf.py

@@ -154,14 +154,25 @@ def test_pip_install_uses_python_dash_m(tmp_path, monkeypatch):
     assert seen and seen[0][:4] == [str(py), "-m", "pip", "install"]
 
 
-def test_pip_install_oserror_returns_false(tmp_path, monkeypatch):
+def test_patch_extra_args_fds_empty_x(tmp_path, monkeypatch):
+    """FDS empty ExtraArgs is '\\x' — must replace, not append."""
     mod = _load()
-    py = tmp_path / "python"
+    backends = tmp_path / "Backends.fds"
-    py.write_text("", encoding="utf-8")
+    backends.write_text("0:\n\ttype: comfyui_selfstart\n\tExtraArgs: \\x\n", encoding="utf-8")
+    monkeypatch.setattr(mod, "BACKENDS", backends)
+    assert mod.patch_backends_extra_args("--use-sage-attention") is True
+    text = backends.read_text(encoding="utf-8")
+    assert "ExtraArgs: --use-sage-attention" in text
+    assert "\\x --use-sage" not in text
 
-    def boom(*_a, **_k):
-        raise OSError(8, "Exec format error", str(py))
 
-    monkeypatch.setattr(mod, "sage_already_importable", lambda _p: False)
+def test_sanitize_backends_fds_corruption(tmp_path, monkeypatch):
-    monkeypatch.setattr(mod.subprocess, "check_call", boom)
+    mod = _load()
-    assert mod.pip_install_sage(py) is False
+    backends = tmp_path / "Backends.fds"
+    backends.write_text(
+        "ExtraArgs: \\x --use-sage-attention\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(mod, "BACKENDS", backends)
+    assert mod.sanitize_backends_fds() is True
+    assert backends.read_text(encoding="utf-8") == "ExtraArgs: --use-sage-attention\n"

tests/test_wait_backend_idle.py

@@ -39,6 +39,8 @@ def test_install_swarm_comfy_script_payload():
     assert ".gpu-rent-comfy-installing" in text
     assert "RestartBackends" in text
     assert 'bstat == "errored"' in text
+    assert "sanitize_backends_fds" in text or "recover_empty_backends" in text
+    assert "AddNewBackend" in text
 
 
 def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):