Enhance GPU probing and performance tuning in provisioning

- Introduced GPU probing functionality to gather and store GPU specifications in `/mnt/swarm_data/.gpu-rent-gpu.json`, aiding in performance tuning.
- Updated `install_ollama.sh` and `install_llamacpp.sh` to utilize GPU information for configuring optimal runtime parameters.
- Enhanced `provision.py` to include GPU probing and performance tuning logic, ensuring better resource allocation for LLM operations.
- Improved documentation in `decisions.md`, `llm.md`, and `swarmui.md` to reflect changes in GPU handling and performance tuning processes.
- Added new tests to validate the GPU probing and model resolution logic, ensuring robustness in handling various GPU configurations.

src/gpu_rent/civitai.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import time
 from dataclasses import dataclass
 
 import httpx
@@ -147,6 +148,133 @@ def fetch_model_version_by_hash(
     raise CloudError(f"Civitai by-hash {digest[:12]}…: {last_error} (хосты {', '.join(seen)})")
 
 
+def _sha_from_version(version: dict) -> list[str]:
+    """All SHA256 hashes listed on a version's files (lower)."""
+    out: list[str] = []
+    for f in version.get("files") or []:
+        if not isinstance(f, dict):
+            continue
+        hashes = f.get("hashes") or {}
+        if not isinstance(hashes, dict):
+            continue
+        sha = hashes.get("SHA256") or hashes.get("sha256")
+        if sha:
+            out.append(str(sha).strip().lower())
+    return out
+
+
+def fetch_model_versions_by_hashes(
+    token: str | None,
+    host: str,
+    sha256_list: list[str],
+    *,
+    timeout: float = 60.0,
+    chunk_size: int = 100,
+) -> dict[str, dict]:
+    """POST /api/v1/model-versions/by-hash (≤100). Map sha256(lower) → version.
+
+    Unmatched hashes are omitted (caller treats as unknown). Retries once on 429.
+    If response versions lack files[].hashes, falls back to /by-hash/ids + GET version.
+    """
+    cleaned: list[str] = []
+    seen_h: set[str] = set()
+    for raw in sha256_list:
+        digest = str(raw).strip().lower()
+        if len(digest) == 64 and all(c in "0123456789abcdef" for c in digest):
+            if digest not in seen_h:
+                cleaned.append(digest)
+                seen_h.add(digest)
+    if not cleaned:
+        return {}
+
+    first = _normalize_host(host)
+    order = [first, other_host(first)]
+    headers: dict[str, str] = {"Content-Type": "application/json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    result: dict[str, dict] = {}
+
+    def _post(candidate: str, path: str, body: list[str]) -> httpx.Response | None:
+        url = f"https://{candidate}/api/v1/{path}"
+        last: httpx.Response | None = None
+        for attempt in range(2):
+            try:
+                with httpx.Client(timeout=timeout, follow_redirects=True) as client:
+                    last = client.post(url, headers=headers, json=body)
+            except httpx.HTTPError:
+                return None
+            if last.status_code == 429 and attempt == 0:
+                time.sleep(2.0)
+                continue
+            return last
+        return last
+
+    for i in range(0, len(cleaned), chunk_size):
+        chunk = cleaned[i : i + chunk_size]
+        last_error = "нет ответа"
+        ok = False
+        for candidate in order:
+            if candidate not in ALLOWED_HOSTS:
+                continue
+            response = _post(candidate, "model-versions/by-hash", chunk)
+            if response is None:
+                last_error = "network"
+                continue
+            if response.status_code == 200:
+                data = response.json()
+                if isinstance(data, list):
+                    for version in data:
+                        if not isinstance(version, dict) or version.get("id") is None:
+                            continue
+                        for sha in _sha_from_version(version):
+                            if sha in seen_h:
+                                result.setdefault(sha, version)
+                    ok = True
+                    break
+                last_error = "не list"
+                continue
+            last_error = f"HTTP {response.status_code}"
+            if response.status_code not in {404, 400}:
+                continue
+        if not ok:
+            raise CloudError(
+                f"Civitai by-hash batch ({len(chunk)}): {last_error} "
+                f"(хосты {', '.join(order)})"
+            )
+
+        missing = [h for h in chunk if h not in result]
+        if not missing:
+            continue
+        # Fallback: ids endpoint maps hash → versionId, then GET version for modelId.
+        for candidate in order:
+            if candidate not in ALLOWED_HOSTS:
+                continue
+            response = _post(candidate, "model-versions/by-hash/ids", missing)
+            if response is None or response.status_code != 200:
+                continue
+            pairs = response.json()
+            if not isinstance(pairs, list):
+                continue
+            for pair in pairs:
+                if not isinstance(pair, dict):
+                    continue
+                h = str(pair.get("hash") or "").strip().lower()
+                try:
+                    vid = int(pair.get("modelVersionId"))
+                except (TypeError, ValueError):
+                    continue
+                if h not in seen_h:
+                    continue
+                try:
+                    _host, version = fetch_model_version(token or "", candidate, vid)
+                except CloudError:
+                    continue
+                result.setdefault(h, version)
+            break
+
+    return result
+
+
 def civitai_model_url(model_id: int, version_id: int, host: str = "civitai.red") -> str:
     """Canonical manifest URL (links only — no download)."""
     h = _normalize_host(host)