Refactor ComfyUI installation script to improve Python environment handling

- Replaced direct calls to `venv/bin/pip` with `python -m pip` to avoid execution errors on network mounts and improve compatibility.
- Introduced a new function `find_comfy_python` to locate the appropriate Python executable for ComfyUI installations.
- Updated the `pip_install_sage` function to utilize the new Python handling, enhancing installation reliability.
- Added tests to verify the new behavior of Python environment detection and installation processes, ensuring robustness.

src/gpu_rent/remote/tune_swarm_perf.py

@@ -18,13 +18,15 @@ DATA = Path("/mnt/swarm_data")
 GPU_JSON = DATA / ".gpu-rent-gpu.json"
 MARKER = DATA / ".gpu-rent-perf-tuned"
 BACKENDS = DATA / "Data" / "Backends.fds"
-COMFY_VENV_CANDIDATES = [
-    DATA / "dlbackend" / "ComfyUI" / "venv" / "bin" / "pip",
-    DATA / "dlbackend" / "comfy" / "venv" / "bin" / "pip",
-    DATA / "dlbackend" / "comfy" / "ComfyUI" / "venv" / "bin" / "pip",
-    Path("/opt/swarmui/dlbackend/ComfyUI/venv/bin/pip"),
-    Path("/opt/swarmui/dlbackend/comfy/venv/bin/pip"),
-    Path("/opt/swarmui/dlbackend/comfy/ComfyUI/venv/bin/pip"),
+# Prefer python then run `python -m pip` — venv/bin/pip often hits
+# OSError Errno 8 Exec format error on network mounts / bad shebangs.
+COMFY_PYTHON_CANDIDATES = [
+    DATA / "dlbackend" / "ComfyUI" / "venv" / "bin" / "python",
+    DATA / "dlbackend" / "comfy" / "venv" / "bin" / "python",
+    DATA / "dlbackend" / "comfy" / "ComfyUI" / "venv" / "bin" / "python",
+    Path("/opt/swarmui/dlbackend/ComfyUI/venv/bin/python"),
+    Path("/opt/swarmui/dlbackend/comfy/venv/bin/python"),
+    Path("/opt/swarmui/dlbackend/comfy/ComfyUI/venv/bin/python"),
 ]
 
 
@@ -66,16 +68,21 @@ def tier_notes(gpu: dict) -> dict:
     }
 
 
-def find_pip() -> Path | None:
-    for p in COMFY_VENV_CANDIDATES:
-        if p.is_file():
+def find_comfy_python() -> Path | None:
+    for p in COMFY_PYTHON_CANDIDATES:
+        if p.is_file() and os.access(p, os.X_OK):
+            return p
+    for p in sorted(DATA.glob("dlbackend/**/venv/bin/python")):
+        if p.is_file() and os.access(p, os.X_OK):
             return p
-    # glob
-    for p in DATA.glob("dlbackend/**/venv/bin/pip"):
-        return p
     return None
 
 
+# Back-compat alias for tests that monkeypatch find_pip
+def find_pip() -> Path | None:
+    return find_comfy_python()
+
+
 def patch_backends_extra_args(extra: str) -> bool:
     """Ensure ExtraArgs contains sage flag for Comfy self-start backends."""
     if not extra:
@@ -108,18 +115,50 @@ def patch_backends_extra_args(extra: str) -> bool:
     return True
 
 
-def pip_install_sage(pip: Path) -> bool:
-    print(f"pip install triton sageattention via {pip}")
+def sage_already_importable(py: Path) -> bool:
+    code, _ = _run(
+        [str(py), "-c", "import triton, sageattention"],
+        timeout=60,
+    )
+    return code == 0
+
+
+def _run(cmd: list[str], *, timeout: float = 900) -> tuple[int, str]:
+    try:
+        out = subprocess.check_output(
+            cmd, text=True, stderr=subprocess.STDOUT, timeout=timeout, env=_pip_env()
+        )
+        return 0, out
+    except subprocess.CalledProcessError as exc:
+        return exc.returncode, (exc.output or str(exc))
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        return 1, str(exc)
+
+
+def _pip_env() -> dict[str, str]:
     env = dict(os.environ)
     env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
-    cmd = [str(pip), "install", "-U", "triton", "sageattention"]
+    return env
+
+
+def pip_install_sage(py: Path) -> bool:
+    """Install via ``python -m pip`` (never exec venv/bin/pip directly)."""
+    if sage_already_importable(py):
+        print(f"triton+sageattention already importable via {py}")
+        return True
+    print(f"python -m pip install triton sageattention via {py}")
+    cmd = [str(py), "-m", "pip", "install", "-U", "triton", "sageattention"]
     try:
-        subprocess.check_call(cmd, env=env)
+        subprocess.check_call(cmd, env=_pip_env())
         print("triton + sageattention installed")
         return True
     except subprocess.CalledProcessError as exc:
         print(f"WARN: pip install failed ({exc}) — ExtraArgs NOT applied")
         return False
+    except OSError as exc:
+        # Errno 8 Exec format error on bad wrappers — should not happen with -m pip
+        print(f"WARN: cannot run pip ({exc}) — ExtraArgs NOT applied")
+        return False
 
 
 def main() -> int:
@@ -144,11 +183,11 @@ def main() -> int:
     pip_ok = not plan["use_sage"]
     restarted_needed = False
     if plan["use_sage"]:
-        pip = find_pip()
-        if pip:
-            pip_ok = pip_install_sage(pip)
+        py = find_comfy_python()
+        if py:
+            pip_ok = pip_install_sage(py)
         else:
-            print("Comfy venv pip not found yet — will retry next up")
+            print("Comfy venv python not found yet — will retry next up")
             pip_ok = False
         # Only patch ExtraArgs when wheels installed — otherwise Comfy may break.
         if pip_ok and patch_backends_extra_args(plan["extra_args"]):