Enhance GPU probing and performance tuning in provisioning

- Introduced GPU probing functionality to gather and store GPU specifications in `/mnt/swarm_data/.gpu-rent-gpu.json`, aiding in performance tuning.
- Updated `install_ollama.sh` and `install_llamacpp.sh` to utilize GPU information for configuring optimal runtime parameters.
- Enhanced `provision.py` to include GPU probing and performance tuning logic, ensuring better resource allocation for LLM operations.
- Improved documentation in `decisions.md`, `llm.md`, and `swarmui.md` to reflect changes in GPU handling and performance tuning processes.
- Added new tests to validate the GPU probing and model resolution logic, ensuring robustness in handling various GPU configurations.

src/gpu_rent/capture.py

@@ -15,7 +15,9 @@ import yaml
 
 from gpu_rent.civitai import (
     civitai_model_url,
+    fetch_model_version,
     fetch_model_version_by_hash,
+    fetch_model_versions_by_hashes,
     version_ids_from_payload,
 )
 from gpu_rent.config import Config
@@ -62,12 +64,23 @@ class ExtCaptureItem:
     directory: str
 
 
+@dataclass
+class ResolveOutcome:
+    """ok → item; unknown → not on Civitai; api_error → network/HTTP (retry later)."""
+
+    item: ModelCaptureItem | None = None
+    status: str = "unknown"  # ok | unknown | api_error
+    detail: str = ""
+
+
 @dataclass
 class CaptureReport:
     models_new: list[ModelCaptureItem] = field(default_factory=list)
     models_skip: list[str] = field(default_factory=list)
     models_unknown: list[str] = field(default_factory=list)
+    models_api_errors: list[str] = field(default_factory=list)
     ext_new: list[ExtCaptureItem] = field(default_factory=list)
+    ext_updated: list[ExtCaptureItem] = field(default_factory=list)
     ext_skip: list[str] = field(default_factory=list)
     ext_unknown: list[str] = field(default_factory=list)
     models_path: Path | None = None
@@ -102,10 +115,10 @@ def resolve_model_item(
     token: str,
     api_host: str,
     link_host: str,
-) -> ModelCaptureItem | None:
+) -> ResolveOutcome:
     kind = str(raw.get("kind") or "")
     if kind not in MODEL_TYPES:
-        return None
+        return ResolveOutcome(status="unknown", detail="bad kind")
     rel = str(raw.get("rel") or raw.get("name") or "?")
     version_id = raw.get("version_id")
     model_id = raw.get("model_id")
@@ -120,35 +133,74 @@ def resolve_model_item(
     except (TypeError, ValueError):
         mid = None
 
+    def _ok(v: int, m: int, name: str = title) -> ResolveOutcome:
+        return ResolveOutcome(
+            item=ModelCaptureItem(
+                kind=kind,
+                version_id=v,
+                model_id=m,
+                url=civitai_model_url(m, v, link_host),
+                title=name,
+                rel=rel,
+            ),
+            status="ok",
+        )
+
     if vid is not None and mid is not None:
-        return ModelCaptureItem(
-            kind=kind,
-            version_id=vid,
-            model_id=mid,
-            url=civitai_model_url(mid, vid, link_host),
-            title=title,
-            rel=rel,
+        return _ok(vid, mid)
+
+    # Partial sidecar: have version id → GET /model-versions/{id} for modelId.
+    if vid is not None and mid is None:
+        try:
+            _h, version = fetch_model_version(token or "", api_host, vid)
+        except CloudError as exc:
+            msg = str(exc)
+            if "404" in msg or "пустой files" in msg:
+                return ResolveOutcome(
+                    status="unknown",
+                    detail=f"{rel}  version_id={vid} ({msg})",
+                )
+            return ResolveOutcome(
+                status="api_error",
+                detail=f"{rel}  version_id={vid}  {msg}",
+            )
+        vid2, mid2 = version_ids_from_payload(version)
+        if vid2 is not None and mid2 is not None:
+            name = str(version.get("name") or title)
+            return _ok(vid2, mid2, name)
+        return ResolveOutcome(
+            status="unknown",
+            detail=f"{rel}  version_id={vid} (нет modelId в ответе)",
         )
 
     sha = raw.get("sha256")
     if not sha:
-        return None
+        return ResolveOutcome(
+            status="unknown",
+            detail=f"{rel}  (нет sidecar и нет sha256)",
+        )
     try:
         _host, version = fetch_model_version_by_hash(token or None, api_host, str(sha))
-    except CloudError:
-        return None
+    except CloudError as exc:
+        msg = str(exc)
+        # 404 = genuinely not on Civitai; other → api_error
+        if "HTTP 404" in msg or msg.rstrip().endswith("404"):
+            return ResolveOutcome(
+                status="unknown",
+                detail=f"{rel}  sha={str(sha)[:12]}…",
+            )
+        return ResolveOutcome(
+            status="api_error",
+            detail=f"{rel}  sha={str(sha)[:12]}…  {msg}",
+        )
     vid2, mid2 = version_ids_from_payload(version)
     if vid2 is None or mid2 is None:
-        return None
-    name = version.get("name") or title
-    return ModelCaptureItem(
-        kind=kind,
-        version_id=vid2,
-        model_id=mid2,
-        url=civitai_model_url(mid2, vid2, link_host),
-        title=str(name),
-        rel=rel,
-    )
+        return ResolveOutcome(
+            status="unknown",
+            detail=f"{rel}  sha={str(sha)[:12]}… (пустой payload)",
+        )
+    name = str(version.get("name") or title)
+    return _ok(vid2, mid2, name)
 
 
 def _backup(path: Path) -> None:
@@ -156,37 +208,46 @@ def _backup(path: Path) -> None:
         shutil.copy2(path, path.with_suffix(path.suffix + ".bak"))
 
 
+def _keep_model_entry(it: dict) -> bool:
+    if not isinstance(it, dict):
+        return False
+    url = it.get("url")
+    if url:
+        return True
+    vid = it.get("version_id")
+    return vid not in (None, "", 0, "0")
+
+
 def merge_models_yaml(
     path: Path,
     new_items: list[ModelCaptureItem],
     *,
     dry_run: bool,
 ) -> tuple[list[ModelCaptureItem], list[str]]:
-    """Return (actually_new, skip_msgs). Writes path unless dry_run."""
+    """Return (actually_new, skip_msgs). Dedupe by (kind, version_id)."""
     existing = parse_models(path) if path.is_file() else []
-    have: set[int] = set()
+    have: set[tuple[str, int]] = set()
     for e in existing:
-        if e.version_id is not None:
-            have.add(e.version_id)
-        elif e.url:
+        vid = e.version_id
+        if vid is None and e.url:
             vid = extract_version_id(e.url)
-            if vid is not None:
-                have.add(vid)
+        if vid is not None:
+            have.add((e.kind, vid))
 
     added: list[ModelCaptureItem] = []
     skipped: list[str] = []
-    seen_new: set[int] = set()
+    seen_new: set[tuple[str, int]] = set()
     for item in new_items:
-        if item.version_id in have or item.version_id in seen_new:
+        key = (item.kind, item.version_id)
+        if key in have or key in seen_new:
             skipped.append(f"{item.kind} {item.title} modelVersionId={item.version_id}")
             continue
-        seen_new.add(item.version_id)
+        seen_new.add(key)
         added.append(item)
 
     if dry_run or not added:
         return added, skipped
 
-    # Rebuild full mapping: keep existing entries, append new urls.
     data: dict[str, list[dict[str, str]]] = {k: [] for k in MODEL_TYPES}
     if path.is_file():
         raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
@@ -195,13 +256,12 @@ def merge_models_yaml(
                 items = raw.get(kind) or []
                 if isinstance(items, list):
                     for it in items:
-                        if isinstance(it, dict) and (it.get("url") or it.get("version_id") not in (None, 0, "0")):
+                        if _keep_model_entry(it):
                             data[kind].append(dict(it))
 
     for item in added:
         data[item.kind].append({"url": item.url})
 
-    # Drop empty kinds for cleaner file
     out = {k: v for k, v in data.items() if v}
     _backup(path)
     path.parent.mkdir(parents=True, exist_ok=True)
@@ -217,32 +277,9 @@ def merge_extensions_yaml(
     new_items: list[ExtCaptureItem],
     *,
     dry_run: bool,
-) -> tuple[list[ExtCaptureItem], list[str]]:
-    existing = parse_extensions(path) if path.is_file() else []
-    have_urls: set[tuple[str, str]] = set()
-    have_dirs: set[tuple[str, str]] = set()
-    for e in existing:
-        have_urls.add((e.kind, strip_git_auth(e.url).rstrip("/").lower()))
-        dirname = e.directory or ""
-        if dirname:
-            have_dirs.add((e.kind, dirname.lower()))
-
-    added: list[ExtCaptureItem] = []
-    skipped: list[str] = []
-    for item in new_items:
-        key_url = (item.kind, strip_git_auth(item.url).rstrip("/").lower())
-        key_dir = (item.kind, item.directory.lower())
-        if key_url in have_urls or key_dir in have_dirs:
-            skipped.append(f"{item.kind} {item.directory} {item.url}")
-            continue
-        have_urls.add(key_url)
-        have_dirs.add(key_dir)
-        added.append(item)
-
-    if dry_run or not added:
-        return added, skipped
-
-    data: dict[str, list[dict[str, str]]] = {"swarmui": [], "comfy": []}
+) -> tuple[list[ExtCaptureItem], list[ExtCaptureItem], list[str]]:
+    """Return (added, updated, skipped). Same dir + different URL → update url/ref."""
+    data: dict[str, list[dict[str, Any]]] = {"swarmui": [], "comfy": []}
     if path.is_file():
         raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
         if isinstance(raw, dict):
@@ -253,13 +290,59 @@ def merge_extensions_yaml(
                         if isinstance(it, dict) and it.get("url"):
                             data[kind].append(dict(it))
 
-    for item in added:
-        entry: dict[str, str] = {
-            "url": strip_git_auth(item.url),
-            "ref": item.ref,
-            "dir": item.directory,
-        }
-        data[item.kind].append(entry)
+    def _dir_of(it: dict) -> str:
+        d = it.get("dir")
+        if d:
+            return str(d).lower()
+        url = strip_git_auth(str(it.get("url") or ""))
+        name = url.rstrip("/").rsplit("/", 1)[-1]
+        if name.endswith(".git"):
+            name = name[:-4]
+        return name.lower()
+
+    added: list[ExtCaptureItem] = []
+    updated: list[ExtCaptureItem] = []
+    skipped: list[str] = []
+
+    for item in new_items:
+        clean = strip_git_auth(item.url)
+        key_url = (item.kind, clean.rstrip("/").lower())
+        key_dir = (item.kind, item.directory.lower())
+        bucket = data.setdefault(item.kind, [])
+
+        matched_url = False
+        matched_dir_idx: int | None = None
+        for idx, it in enumerate(bucket):
+            it_url = strip_git_auth(str(it.get("url") or "")).rstrip("/").lower()
+            if (item.kind, it_url) == key_url:
+                matched_url = True
+                break
+            if (item.kind, _dir_of(it)) == key_dir:
+                matched_dir_idx = idx
+
+        if matched_url:
+            skipped.append(f"{item.kind} {item.directory} {clean}")
+            continue
+
+        if matched_dir_idx is not None:
+            old = bucket[matched_dir_idx]
+            old_url = strip_git_auth(str(old.get("url") or ""))
+            old_ref = str(old.get("ref") or "main")
+            if old_url.rstrip("/").lower() == clean.rstrip("/").lower() and old_ref == item.ref:
+                skipped.append(f"{item.kind} {item.directory} {clean}")
+                continue
+            old["url"] = clean
+            old["ref"] = item.ref
+            if not old.get("dir"):
+                old["dir"] = item.directory
+            updated.append(item)
+            continue
+
+        bucket.append({"url": clean, "ref": item.ref, "dir": item.directory})
+        added.append(item)
+
+    if dry_run or (not added and not updated):
+        return added, updated, skipped
 
     out = {k: v for k, v in data.items() if v}
     _backup(path)
@@ -268,7 +351,7 @@ def merge_extensions_yaml(
         yaml.safe_dump(out, allow_unicode=True, default_flow_style=False, sort_keys=False),
         encoding="utf-8",
     )
-    return added, skipped
+    return added, updated, skipped
 
 
 def capture_models(
@@ -287,24 +370,91 @@ def capture_models(
         raise CloudError("inventory.models: ожидался list")
 
     resolved: list[ModelCaptureItem] = []
+    need_hash: list[dict] = []
     link_host = cfg.civitai_api_host or "civitai.red"
+    token = cfg.civitai_api_token
+    api_host = cfg.civitai_api_host
+
     for raw in raw_models:
         if not isinstance(raw, dict):
             continue
         if kind_filter and str(raw.get("kind")) != kind_filter:
             continue
-        item = resolve_model_item(
-            raw,
-            token=cfg.civitai_api_token,
-            api_host=cfg.civitai_api_host,
-            link_host=link_host,
-        )
-        if item is None:
-            rel = str(raw.get("rel") or raw.get("name") or "?")
-            sha = str(raw.get("sha256") or "")[:12]
-            report.models_unknown.append(f"{rel}  sha={sha}…")
+        vid = raw.get("version_id")
+        mid = raw.get("model_id")
+        try:
+            vid_i = int(vid) if vid is not None else None
+        except (TypeError, ValueError):
+            vid_i = None
+        try:
+            mid_i = int(mid) if mid is not None else None
+        except (TypeError, ValueError):
+            mid_i = None
+        sha = raw.get("sha256")
+
+        # Full sidecar / ids → no API. Partial vid → GET version. Else batch by-hash.
+        if vid_i is not None and mid_i is not None:
+            outcome = resolve_model_item(
+                raw, token=token, api_host=api_host, link_host=link_host
+            )
+        elif vid_i is not None:
+            outcome = resolve_model_item(
+                raw, token=token, api_host=api_host, link_host=link_host
+            )
+        elif sha:
+            need_hash.append(raw)
             continue
-        resolved.append(item)
+        else:
+            outcome = resolve_model_item(
+                raw, token=token, api_host=api_host, link_host=link_host
+            )
+
+        if outcome.status == "ok" and outcome.item is not None:
+            resolved.append(outcome.item)
+        elif outcome.status == "api_error":
+            report.models_api_errors.append(outcome.detail or "?")
+        else:
+            report.models_unknown.append(outcome.detail or str(raw.get("rel") or "?"))
+
+    if need_hash:
+        digests = [str(r["sha256"]).strip().lower() for r in need_hash if r.get("sha256")]
+        log(f"capture: by-hash batch {len(digests)} файл(ов)…")
+        try:
+            by_hash = fetch_model_versions_by_hashes(token or None, api_host, digests)
+        except CloudError as exc:
+            report.models_api_errors.append(f"by-hash batch: {exc}")
+            by_hash = {}
+            for raw in need_hash:
+                rel = str(raw.get("rel") or "?")
+                sha = str(raw.get("sha256") or "")[:12]
+                report.models_api_errors.append(f"{rel}  sha={sha}… (batch failed)")
+            need_hash = []
+
+        for raw in need_hash:
+            sha = str(raw.get("sha256") or "").strip().lower()
+            rel = str(raw.get("rel") or raw.get("name") or "?")
+            version = by_hash.get(sha)
+            if not version:
+                report.models_unknown.append(f"{rel}  sha={sha[:12]}…")
+                continue
+            vid2, mid2 = version_ids_from_payload(version)
+            if vid2 is None or mid2 is None:
+                report.models_unknown.append(f"{rel}  sha={sha[:12]}… (пустой payload)")
+                continue
+            kind = str(raw.get("kind") or "")
+            if kind not in MODEL_TYPES:
+                continue
+            title = str(version.get("name") or Path(str(raw.get("name") or rel)).stem)
+            resolved.append(
+                ModelCaptureItem(
+                    kind=kind,
+                    version_id=vid2,
+                    model_id=mid2,
+                    url=civitai_model_url(mid2, vid2, link_host),
+                    title=title,
+                    rel=rel,
+                )
+            )
 
     added, skipped = merge_models_yaml(
         cfg.models_manifest, resolved, dry_run=dry_run
@@ -350,12 +500,13 @@ def capture_extensions(
             )
         )
 
-    added, skipped = merge_extensions_yaml(
+    added, updated, skipped = merge_extensions_yaml(
         cfg.extensions_manifest, resolved, dry_run=dry_run
     )
     report.ext_new = added
+    report.ext_updated = updated
     report.ext_skip = skipped
-    report.wrote_extensions = bool(added) and not dry_run
+    report.wrote_extensions = (bool(added) or bool(updated)) and not dry_run
     return report
 
 
@@ -364,7 +515,9 @@ def merge_reports(a: CaptureReport, b: CaptureReport) -> CaptureReport:
         models_new=a.models_new + b.models_new,
         models_skip=a.models_skip + b.models_skip,
         models_unknown=a.models_unknown + b.models_unknown,
+        models_api_errors=a.models_api_errors + b.models_api_errors,
         ext_new=a.ext_new + b.ext_new,
+        ext_updated=a.ext_updated + b.ext_updated,
         ext_skip=a.ext_skip + b.ext_skip,
         ext_unknown=a.ext_unknown + b.ext_unknown,
         models_path=a.models_path or b.models_path,
@@ -390,27 +543,42 @@ def capture_all(
     return merge_reports(m, e)
 
 
-def print_report(report: CaptureReport, log: Log, *, dry_run: bool) -> None:
+def print_report(
+    report: CaptureReport,
+    log: Log,
+    *,
+    dry_run: bool,
+    show_models: bool = True,
+    show_extensions: bool = True,
+) -> None:
     prefix = "[dry-run] " if dry_run else ""
-    log(
-        f"{prefix}models: +{len(report.models_new)} new, "
-        f"{len(report.models_skip)} already in yaml, "
-        f"{len(report.models_unknown)} unknown (no Civitai)"
-    )
-    for item in report.models_new:
-        log(f"  + {item.kind}  {item.title}  modelVersionId={item.version_id}")
-    for line in report.models_unknown:
-        log(f"  ? {line}")
+    if show_models:
+        log(
+            f"{prefix}models: +{len(report.models_new)} new, "
+            f"{len(report.models_skip)} already in yaml, "
+            f"{len(report.models_unknown)} unknown (no Civitai), "
+            f"{len(report.models_api_errors)} api errors"
+        )
+        for item in report.models_new:
+            log(f"  + {item.kind}  {item.title}  modelVersionId={item.version_id}")
+        for line in report.models_unknown:
+            log(f"  ? {line}")
+        for line in report.models_api_errors:
+            log(f"  ! {line}")
 
-    log(
-        f"{prefix}extensions: +{len(report.ext_new)} new, "
-        f"{len(report.ext_skip)} already, "
-        f"{len(report.ext_unknown)} unknown"
-    )
-    for item in report.ext_new:
-        log(f"  + {item.kind}  {item.directory}  {item.url} @{item.ref}")
-    for line in report.ext_unknown:
-        log(f"  ? {line}")
+    if show_extensions:
+        log(
+            f"{prefix}extensions: +{len(report.ext_new)} new, "
+            f"~{len(report.ext_updated)} updated, "
+            f"{len(report.ext_skip)} already, "
+            f"{len(report.ext_unknown)} unknown"
+        )
+        for item in report.ext_new:
+            log(f"  + {item.kind}  {item.directory}  {item.url} @{item.ref}")
+        for item in report.ext_updated:
+            log(f"  ~ {item.kind}  {item.directory}  {item.url} @{item.ref}")
+        for line in report.ext_unknown:
+            log(f"  ? {line}")
 
     if dry_run:
         log("dry-run — файлы не записаны")
@@ -423,7 +591,4 @@ def print_report(report: CaptureReport, log: Log, *, dry_run: bool) -> None:
             f"(backup {report.extensions_path.name}.bak)"
         )
     if not report.wrote_models and not report.wrote_extensions:
-        if report.models_new or report.ext_new:
-            pass
-        else:
-            log("нечего добавлять")
+        log("нечего добавлять")