Enhance Ollama model management and performance tuning

- Updated the `provision_llm` function to utilize the `/api/tags` endpoint for verifying available models, improving accuracy in model management.
- Introduced a new `already_have_ollama_tag` function to ensure exact tag matching, preventing mismatches during model checks.
- Enhanced the `pull_stream` function to require a successful status from the API before proceeding, ensuring reliable model downloads.
- Added logic to handle unwritten blob files, improving the robustness of the model pulling process.
- Updated documentation and tests to reflect these changes, ensuring clarity and reliability in Ollama model operations.

tests/test_tune_swarm_perf.py

@@ -82,6 +82,7 @@ def test_pip_ok_patches_extra_args(tmp_path, monkeypatch):
     monkeypatch.setattr(mod, "find_pip", lambda: pip)
     monkeypatch.setattr(mod, "find_comfy_python", lambda: pip)
     monkeypatch.setattr(mod, "pip_install_sage", lambda _p: True)
+    monkeypatch.setattr(mod, "triton_jit_ok", lambda _p: True)
 
     assert mod.main() == 0
     marker = json.loads((data / ".gpu-rent-perf-tuned").read_text(encoding="utf-8"))
@@ -131,6 +132,7 @@ def test_pip_fail_retries_next_run(tmp_path, monkeypatch):
     monkeypatch.setattr(mod, "find_pip", lambda: pip)
     monkeypatch.setattr(mod, "find_comfy_python", lambda: pip)
     monkeypatch.setattr(mod, "pip_install_sage", lambda _p: True)
+    monkeypatch.setattr(mod, "triton_jit_ok", lambda _p: True)
 
     assert mod.main() == 0
     new_m = json.loads(marker.read_text(encoding="utf-8"))
@@ -223,3 +225,37 @@ def test_ensure_absolute_start_script(tmp_path, monkeypatch):
     text = backends.read_text(encoding="utf-8")
     assert str(main_py.resolve()) in text
     assert mod.ensure_absolute_start_script() is False
+
+
+def test_jit_fail_strips_sage_extra_args(tmp_path, monkeypatch):
+    mod = _load()
+    data = tmp_path
+    backends = data / "Data" / "Backends.fds"
+    backends.parent.mkdir(parents=True)
+    backends.write_text("ExtraArgs: --use-sage-attention\n", encoding="utf-8")
+    gpu_json = data / ".gpu-rent-gpu.json"
+    gpu_json.write_text(
+        json.dumps(
+            {
+                "vram_mib": 24576,
+                "compute_cap": "8.9",
+                "uuid": "gpu-1",
+                "name": "RTX",
+            }
+        ),
+        encoding="utf-8",
+    )
+    pip = data / "fake-pip"
+    pip.write_text("#!/bin/sh\n", encoding="utf-8")
+    monkeypatch.setattr(mod, "DATA", data)
+    monkeypatch.setattr(mod, "GPU_JSON", gpu_json)
+    monkeypatch.setattr(mod, "MARKER", data / ".gpu-rent-perf-tuned")
+    monkeypatch.setattr(mod, "BACKENDS", backends)
+    monkeypatch.setattr(mod, "find_comfy_python", lambda: pip)
+    monkeypatch.setattr(mod, "pip_install_sage", lambda _p: True)
+    monkeypatch.setattr(mod, "triton_jit_ok", lambda _p: False)
+    assert mod.main() == 0
+    marker = json.loads((data / ".gpu-rent-perf-tuned").read_text(encoding="utf-8"))
+    assert marker["pip_ok"] is True
+    assert marker["jit_ok"] is False
+    assert "--use-sage-attention" not in backends.read_text(encoding="utf-8")