Enhance Ollama model management and performance tuning

- Updated the `provision_llm` function to utilize the `/api/tags` endpoint for verifying available models, improving accuracy in model management.
- Introduced a new `already_have_ollama_tag` function to ensure exact tag matching, preventing mismatches during model checks.
- Enhanced the `pull_stream` function to require a successful status from the API before proceeding, ensuring reliable model downloads.
- Added logic to handle unwritten blob files, improving the robustness of the model pulling process.
- Updated documentation and tests to reflect these changes, ensuring clarity and reliability in Ollama model operations.

src/gpu_rent/provision.py

@@ -923,8 +923,49 @@ def ensure_swarmui_running(cfg: Config, host: str, log: Log, restart: bool) -> N
     run_ssh(cfg, host, "sudo -n systemctl enable swarmui", check=False)
 
 
+_OLLAMA_TAGS_PY = r"""
+import json, urllib.request
+try:
+    with urllib.request.urlopen("http://127.0.0.1:11434/api/tags", timeout=8) as r:
+        data = json.loads(r.read().decode())
+except Exception as exc:
+    print("ERR " + str(exc)[:200])
+    raise SystemExit(0)
+for m in data.get("models") or []:
+    if isinstance(m, dict):
+        for key in ("name", "model"):
+            n = m.get(key)
+            if n:
+                print(n)
+    elif isinstance(m, str) and m.strip():
+        print(m.strip())
+"""
+
+
+def _ollama_api_tags(cfg: Config, host: str) -> set[str]:
+    """Names from Ollama /api/tags (same source as Assistent / verify)."""
+    out = run_ssh(
+        cfg,
+        host,
+        "python3 - <<'PY'\n" + _OLLAMA_TAGS_PY + "\nPY",
+        check=False,
+        timeout=20,
+    )
+    names: set[str] = set()
+    for ln in out.splitlines():
+        s = ln.strip()
+        if not s or s.startswith("ERR "):
+            continue
+        names.add(s)
+    return names
+
+
 def provision_llm(cfg: Config, host: str, log: Log) -> None:
-    from gpu_rent.llm_runtime import normalize_runtime, parse_ollama_models
+    from gpu_rent.llm_runtime import (
+        already_have_ollama_tag,
+        normalize_runtime,
+        parse_ollama_models,
+    )
     from gpu_rent.ssh_ops import run_script_sudo
     from gpu_rent.state import load_state, save_state
 
@@ -950,6 +991,7 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
         st.notes.pop("llm_error", None)
         save_state(st)
         return
+    still: list[str] = []
     if runtime == "ollama":
         _stop_units("gpu-rent-llamacpp")
         log("LLM: ставим/запускаем Ollama")
@@ -967,48 +1009,53 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
         if defaults:
             log(f"Ollama preferred: {defaults[0]}")
         names = [e.name for e in entries]
+        still: list[str] = []
         if not names:
             log("ollama-models.yaml пуст — pull skip")
         else:
-            # Fast path: all tags already present — skip upload/pull script.
-            listed = run_ssh(
-                cfg,
-                host,
-                "ollama list 2>/dev/null | awk 'NR>1 {print $1}' || true",
-                check=False,
-                timeout=30,
-            )
-            have = {ln.strip() for ln in listed.splitlines() if ln.strip()}
-            missing = []
-            for name in names:
-                if name in have or (
-                    ":" not in name and f"{name}:latest" in have
-                ) or (
-                    name.endswith(":latest") and name.rsplit(":", 1)[0] in have
-                ):
-                    continue
-                missing.append(name)
+            have = _ollama_api_tags(cfg, host)
+            missing = [n for n in names if not already_have_ollama_tag(have, n)]
             if not missing:
-                log(f"ollama pull: skip — все {len(names)} уже есть")
+                log(f"ollama pull: skip — /api/tags уже {sorted(have)}")
             else:
                 put_text(
                     cfg, host, "/tmp/gpu-rent-ollama-models.json", json.dumps(missing, indent=2)
                 )
-                log(f"Ollama: pull {len(missing)} из манифеста (нет: {len(missing)})")
-                run_python(
-                    cfg,
-                    host,
-                    _pkg_text("ollama_pull.py"),
-                    remote_path="/tmp/gpu-rent-ollama_pull.py",
-                    timeout=7200,
-                    log=log,
+                log(
+                    f"Ollama: pull {len(missing)} из манифеста "
+                    f"(/api/tags={len(have)})"
+                )
+                try:
+                    run_python(
+                        cfg,
+                        host,
+                        _pkg_text("ollama_pull.py"),
+                        remote_path="/tmp/gpu-rent-ollama_pull.py",
+                        timeout=7200,
+                        log=log,
+                    )
+                except Exception as exc:
+                    log(f"⚠ Ollama pull: {exc}")
+            have = _ollama_api_tags(cfg, host)
+            still = [n for n in names if not already_have_ollama_tag(have, n)]
+            if still:
+                log(
+                    "⚠ Ollama /api/tags без "
+                    + ", ".join(still[:5])
+                    + f" (есть: {sorted(have) or 'пусто'}). "
+                    "SwarmUI ок — GPU не гасим; Assistent будет пустой."
                 )
     else:
         raise CloudError(f"неизвестный LLM_RUNTIME={runtime!r}")
     st = load_state()
     st.notes = dict(st.notes or {})
     st.notes["llm_runtime"] = runtime
-    st.notes.pop("llm_error", None)
+    if still:
+        st.notes["llm_error"] = (
+            "нет в /api/tags: " + ", ".join(still[:5])
+        )[:500]
+    else:
+        st.notes.pop("llm_error", None)
     save_state(st)