From f8bbde8ac0549d6c53f239031e561acf294810a1 Mon Sep 17 00:00:00 2001
From: Leonid Pershin <lpershin@syntellect.ru>
Date: Fri, 21 Aug 2026 10:54:02 +0300
Subject: [PATCH] Enhance backend recovery and diagnostics in installation and
 tuning scripts

- Improved the `install_swarm_comfy` function to handle empty backend states more effectively, introducing recovery mechanisms and enhanced logging for better visibility.
- Updated the `tune_swarm_perf` function to always sanitize backend FDS corruption, ensuring consistent performance tuning.
- Added new tests to validate the functionality of backend recovery and FDS sanitization, ensuring robustness in backend management.
---
 src/gpu_rent/remote/install_swarm_comfy.py | 18 +++++++++++----
 src/gpu_rent/remote/tune_swarm_perf.py     |  6 ++++-
 tests/test_tune_swarm_perf.py              | 27 +++++++++++++++-------
 tests/test_wait_backend_idle.py            |  2 ++
 4 files changed, 40 insertions(+), 13 deletions(-)

diff --git a/src/gpu_rent/remote/install_swarm_comfy.py b/src/gpu_rent/remote/install_swarm_comfy.py
index b5c9711..677dbb9 100644
--- a/src/gpu_rent/remote/install_swarm_comfy.py
+++ b/src/gpu_rent/remote/install_swarm_comfy.py
@@ -657,11 +657,21 @@ def main() -> int:
         return 0
 
     if comfy_venv_ok() and bstat == "empty":
-        if installed is not True:
-            print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
+        print(
+            "backend empty при venv/IsInstalled — recover (sanitize FDS / AddNewBackend)…",
+            flush=True,
+        )
+        bstat = recover_empty_backends()
+        if bstat not in ("empty", "unknown") and not bstat.startswith("error:"):
+            print(f"recovered empty → {bstat}")
+            return 0
+        if installed is True:
+            print(
+                "WARN: всё ещё empty после recover — пробую InstallConfirmWS",
+                flush=True,
+            )
         else:
-            run_diagnostics()
-            return 1
+            print("venv есть, IsInstalled=false — запускаю InstallConfirmWS")
 
     marker = DATA / ".gpu-rent-comfy-installing"
     try:
diff --git a/src/gpu_rent/remote/tune_swarm_perf.py b/src/gpu_rent/remote/tune_swarm_perf.py
index f2d6d02..534704c 100644
--- a/src/gpu_rent/remote/tune_swarm_perf.py
+++ b/src/gpu_rent/remote/tune_swarm_perf.py
@@ -206,17 +206,21 @@ def main() -> int:
         except json.JSONDecodeError:
             prev = {}
     same_gpu = bool(prev.get("uuid") and prev.get("uuid") == plan["uuid"])
+    # Always repair FDS corruption even when tune is otherwise a no-op.
+    fds_fixed = sanitize_backends_fds()
     if same_gpu and prev.get("extra_args") == plan["extra_args"]:
         if prev.get("pip_ok") or not plan["use_sage"]:
             print(f"perf tune already applied for {plan['name']} ({plan['tier']})")
+            if fds_fixed:
+                print("RESTART_SWARMUI=1")
             return 0
 
     if same_gpu and plan["use_sage"] and prev.get("pip_ok") is False:
         print("perf tune: retry (previous pip_ok=false — sage/triton ещё не встали)")
 
     print(f"perf tune: {plan['name']} tier={plan['tier']} sage={plan['use_sage']}")
+    restarted_needed = bool(fds_fixed)
     pip_ok = not plan["use_sage"]
-    restarted_needed = False
     if plan["use_sage"]:
         py = find_comfy_python()
         if py:
diff --git a/tests/test_tune_swarm_perf.py b/tests/test_tune_swarm_perf.py
index 423ce65..15765e6 100644
--- a/tests/test_tune_swarm_perf.py
+++ b/tests/test_tune_swarm_perf.py
@@ -154,14 +154,25 @@ def test_pip_install_uses_python_dash_m(tmp_path, monkeypatch):
     assert seen and seen[0][:4] == [str(py), "-m", "pip", "install"]
 
 
-def test_pip_install_oserror_returns_false(tmp_path, monkeypatch):
+def test_patch_extra_args_fds_empty_x(tmp_path, monkeypatch):
+    """FDS empty ExtraArgs is '\\x' — must replace, not append."""
     mod = _load()
-    py = tmp_path / "python"
-    py.write_text("", encoding="utf-8")
+    backends = tmp_path / "Backends.fds"
+    backends.write_text("0:\n\ttype: comfyui_selfstart\n\tExtraArgs: \\x\n", encoding="utf-8")
+    monkeypatch.setattr(mod, "BACKENDS", backends)
+    assert mod.patch_backends_extra_args("--use-sage-attention") is True
+    text = backends.read_text(encoding="utf-8")
+    assert "ExtraArgs: --use-sage-attention" in text
+    assert "\\x --use-sage" not in text
 
-    def boom(*_a, **_k):
-        raise OSError(8, "Exec format error", str(py))
 
-    monkeypatch.setattr(mod, "sage_already_importable", lambda _p: False)
-    monkeypatch.setattr(mod.subprocess, "check_call", boom)
-    assert mod.pip_install_sage(py) is False
+def test_sanitize_backends_fds_corruption(tmp_path, monkeypatch):
+    mod = _load()
+    backends = tmp_path / "Backends.fds"
+    backends.write_text(
+        "ExtraArgs: \\x --use-sage-attention\n",
+        encoding="utf-8",
+    )
+    monkeypatch.setattr(mod, "BACKENDS", backends)
+    assert mod.sanitize_backends_fds() is True
+    assert backends.read_text(encoding="utf-8") == "ExtraArgs: --use-sage-attention\n"
diff --git a/tests/test_wait_backend_idle.py b/tests/test_wait_backend_idle.py
index e9b360a..d7ecab3 100644
--- a/tests/test_wait_backend_idle.py
+++ b/tests/test_wait_backend_idle.py
@@ -39,6 +39,8 @@ def test_install_swarm_comfy_script_payload():
     assert ".gpu-rent-comfy-installing" in text
     assert "RestartBackends" in text
     assert 'bstat == "errored"' in text
+    assert "sanitize_backends_fds" in text or "recover_empty_backends" in text
+    assert "AddNewBackend" in text
 
 
 def test_wait_backend_idle_fail_fast_on_errored(monkeypatch):