Enhance Ollama model management and performance tuning

- Updated the `provision_llm` function to utilize the `/api/tags` endpoint for verifying available models, improving accuracy in model management.
- Introduced a new `already_have_ollama_tag` function to ensure exact tag matching, preventing mismatches during model checks.
- Enhanced the `pull_stream` function to require a successful status from the API before proceeding, ensuring reliable model downloads.
- Added logic to handle unwritten blob files, improving the robustness of the model pulling process.
- Updated documentation and tests to reflect these changes, ensuring clarity and reliability in Ollama model operations.

src/gpu_rent/ready.py

@@ -84,7 +84,8 @@ print("WAIT timeout-slice")
 
 # One-shot probe of configured stack endpoints on the VM (JSON line).
 _REMOTE_STACK_PROBE = r'''
-import json, urllib.error, urllib.request, subprocess
+import json, time, urllib.error, urllib.request, subprocess
+from pathlib import Path
 
 def http_ok(url, timeout=4.0):
     try:
@@ -107,9 +108,19 @@ def unit_active(name):
     except Exception:
         return "unknown"
 
+def pulling_age():
+    p = Path("/mnt/swarm_data/.gpu-rent-ollama-pulling")
+    try:
+        if p.is_file():
+            return max(0.0, time.time() - p.stat().st_mtime)
+    except OSError:
+        return None
+    return None
+
 checks = []
 want_swarm = WANT_SWARM
 want_ollama = WANT_OLLAMA
+want_ollama_models = WANT_OLLAMA_MODELS
 
 if want_swarm:
     ok, detail = http_ok("http://127.0.0.1:7801/")
@@ -134,9 +145,11 @@ if want_swarm:
         "ok": ok,
         "detail": detail,
         "unit": unit_active("swarmui"),
+        "retry": not ok,
     })
 
 if want_ollama:
+    retry = True
     try:
         req = urllib.request.Request("http://127.0.0.1:11434/api/tags", method="GET")
         with urllib.request.urlopen(req, timeout=5) as resp:
@@ -154,15 +167,27 @@ if want_ollama:
             preview = ", ".join(names[:3])
             extra = "" if len(names) <= 3 else f" +{len(names) - 3}"
             ok, detail = True, f"{len(names)} models ({preview}{extra})"
+            retry = False
+        elif want_ollama_models:
+            age = pulling_age()
+            if age is not None and age < 2700:
+                ok, detail = False, f"Ollama up, 0 models — pull идёт ({int(age)}s)"
+                retry = True
+            else:
+                ok, detail = True, "WARN 0 models — Assistent empty (GPU не гасим)"
+                retry = False
         else:
-            ok, detail = False, "Ollama up, 0 models — Assistent dropdown empty; ollama pull"
+            ok, detail = True, "0 models (манифест пуст)"
+            retry = False
     except Exception as exc:
         ok, detail = False, str(exc)[:160]
+        retry = True
     checks.append({
         "name": "ollama",
         "ok": ok,
         "detail": detail,
         "unit": unit_active("gpu-rent-ollama"),
+        "retry": retry,
     })
 
 print(json.dumps({"checks": checks}, ensure_ascii=False))
@@ -175,6 +200,7 @@ class ServiceCheck:
     ok: bool
     detail: str
     where: str = "vm"  # vm | local
+    retry: bool = True
 
 
 Log = Callable[[str], None]
@@ -343,10 +369,29 @@ def _expected_services(cfg: Config) -> tuple[bool, bool]:
     return swarm, rt == "ollama"
 
 
+def _want_ollama_models(cfg: Config) -> bool:
+    """True when ollama-models.yaml lists tags that must appear in /api/tags."""
+    if normalize_runtime(getattr(cfg, "llm_runtime", "none")) != "ollama":
+        return False
+    from gpu_rent.llm_runtime import parse_ollama_models
+
+    path = getattr(cfg, "ollama_models_manifest", None)
+    if path is None:
+        return False
+    try:
+        return bool(parse_ollama_models(path))
+    except (OSError, ValueError):
+        return False
+
+
 def _probe_vm_once(cfg: Config, host: str) -> list[ServiceCheck]:
     want_swarm, want_ollama = _expected_services(cfg)
     script = (
         _REMOTE_STACK_PROBE.replace("WANT_SWARM", "True" if want_swarm else "False")
+        .replace(
+            "WANT_OLLAMA_MODELS",
+            "True" if (want_ollama and _want_ollama_models(cfg)) else "False",
+        )
         .replace("WANT_OLLAMA", "True" if want_ollama else "False")
     )
     out = run_ssh(
@@ -380,7 +425,13 @@ def _probe_vm_once(cfg: Config, host: str) -> list[ServiceCheck]:
         if unit and unit != "unknown":
             detail = f"{detail}; unit={unit}"
         checks.append(
-            ServiceCheck(name=name, ok=bool(item.get("ok")), detail=detail, where="vm")
+            ServiceCheck(
+                name=name,
+                ok=bool(item.get("ok")),
+                detail=detail,
+                where="vm",
+                retry=bool(item.get("retry", True)),
+            )
         )
     return checks
 
@@ -417,9 +468,22 @@ def verify_stack_on_vm(
             last = [ServiceCheck("ssh", False, str(exc)[:200], "vm")]
         if last and all(c.ok for c in last):
             for c in last:
-                log(f"  [ok] {c.name}: {c.detail}")
+                mark = "warn" if c.detail.startswith("WARN") else "ok"
+                log(f"  [{mark}] {c.name}: {c.detail}")
             log("проверка VM: всё отвечает")
             return last
+        stuck = [c for c in last if not c.ok and not c.retry]
+        if stuck:
+            for c in last:
+                mark = "ok" if c.ok else "FAIL"
+                log(f"  [{mark}] {c.name}: {c.detail}")
+            if raise_on_fail:
+                failed = [c.name for c in stuck]
+                raise CloudError(
+                    f"{', '.join(failed)} не готов и ждать бесполезно: "
+                    f"{stuck[0].detail}. GPU жив — gpu-rent logs / повторный up (pull)"
+                )
+            return last
         bad = ", ".join(f"{c.name}={c.detail}" for c in last if not c.ok) or "пусто"
         wait.tick(f"  … ещё нет: {bad}")
         time.sleep(poll_every)