From 242f5b7c898cfadaea5a5f4214952492189960a3 Mon Sep 17 00:00:00 2001
From: Leonid Pershin <lpershin@syntellect.ru>
Date: Fri, 21 Aug 2026 07:51:17 +0300
Subject: [PATCH] Enhance LLM runtime and installation script for Linux support

- Added a new function `pick_llamacpp_linux_asset_url` to select appropriate Linux release assets, prioritizing Ubuntu CUDA and Vulkan options while excluding Windows and macOS binaries.
- Updated the installation script to build `llama-server` from source when Linux CUDA binaries are unavailable, improving compatibility and flexibility.
- Revised documentation to reflect changes in asset handling and installation procedures.
- Added tests to validate the new asset selection logic, ensuring correct behavior in various scenarios.
---
 docs/llm.md                             |   4 +-
 src/gpu_rent/llm_runtime.py             |  36 ++++
 src/gpu_rent/remote/install_llamacpp.sh | 269 ++++++++++++++++++------
 tests/test_llm_runtime.py               |  37 ++++
 4 files changed, 283 insertions(+), 63 deletions(-)

diff --git a/docs/llm.md b/docs/llm.md
index 0a19c20..8e39d18 100644
--- a/docs/llm.md
+++ b/docs/llm.md
@@ -120,6 +120,8 @@ Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`:
 
 На `up`: скачать GGUF → `/mnt/swarm_data/llamacpp/models` → `llama-server` + systemd. Уже скачанные крупные файлы не трогает.
 
+**Бинарник:** у upstream нет Linux CUDA в GitHub Releases (только Windows). `install_llamacpp.sh` собирает `llama-server` из исходников (`GGML_CUDA=ON`, arch из GPU probe). Если `nvcc` недоступен — fallback на Ubuntu Vulkan/CPU asset (без Windows). Pin: `LLAMACPP_TAG=b10545`. Override: `LLAMACPP_ASSET_URL` + `LLAMACPP_SHA256`.
+
 ### Пресеты (меню)
 
 | # | ключ | что |
@@ -154,6 +156,6 @@ Busy (не гасить GPU):
 ```bash
 OLLAMA_VERSION=0.6.5
 OLLAMA_SHA256=<sha256 of ollama-linux-amd64.tgz>
-LLAMACPP_TAG=b4690
+LLAMACPP_TAG=b10545
 # или LLAMACPP_ASSET_URL=... + LLAMACPP_SHA256=...
 ```
diff --git a/src/gpu_rent/llm_runtime.py b/src/gpu_rent/llm_runtime.py
index 5db2885..30e0942 100644
--- a/src/gpu_rent/llm_runtime.py
+++ b/src/gpu_rent/llm_runtime.py
@@ -357,6 +357,42 @@ def llm_remote_port(runtime: str) -> int | None:
     return None
 
 
+def pick_llamacpp_linux_asset_url(assets: list[dict[str, Any]]) -> str:
+    """Choose a Linux llama.cpp release asset URL.
+
+    Upstream ships Windows CUDA zips first; never pick win/macos/cudart-only.
+    Prefer ubuntu+cuda → linux+cuda → ubuntu vulkan x64 → ubuntu x64 CPU.
+    Mirrored in remote/install_llamacpp.sh (pick_linux_asset_url).
+    """
+    cands: list[tuple[int, str]] = []
+    for a in assets:
+        name = str(a.get("name") or "").lower()
+        url = str(a.get("browser_download_url") or "")
+        if not (url.endswith(".zip") or url.endswith(".tar.gz")):
+            continue
+        if any(x in name for x in ("win", "macos", "android", "darwin", "xcframework", "-ui.")):
+            continue
+        if "cudart" in name:
+            continue
+        score = 0
+        if "ubuntu" in name and "x64" in name and "cuda" in name:
+            score = 100
+        elif "linux" in name and "cuda" in name:
+            score = 90
+        elif "ubuntu" in name and "vulkan" in name and "x64" in name:
+            score = 50
+        elif "ubuntu" in name and "x64" in name and not any(
+            x in name for x in ("sycl", "openvino", "arm", "s390", "rocm")
+        ):
+            score = 30
+        elif "ubuntu" in name or "linux" in name:
+            score = 10
+        if score:
+            cands.append((score, url))
+    cands.sort(key=lambda t: t[0], reverse=True)
+    return cands[0][1] if cands else ""
+
+
 def append_vars_llm_runtime(vars_file: Path, runtime: str) -> None:
     from gpu_rent.varsfile import upsert_vars
 
diff --git a/src/gpu_rent/remote/install_llamacpp.sh b/src/gpu_rent/remote/install_llamacpp.sh
index 4fc5ae6..c8da691 100644
--- a/src/gpu_rent/remote/install_llamacpp.sh
+++ b/src/gpu_rent/remote/install_llamacpp.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 # Install llama-server (CUDA) for OpenAI-compatible API on loopback :8080.
+# Official GitHub releases ship Windows CUDA only — on Linux we build from source.
 set -euo pipefail
 
 SWARM_USER="${SWARM_USER:-ubuntu}"
@@ -7,7 +8,11 @@ DATA_ROOT="/mnt/swarm_data"
 LLAMA_ROOT="${DATA_ROOT}/llamacpp"
 MODELS_DIR="${LLAMA_ROOT}/models"
 BIN_DIR="${LLAMA_ROOT}/bin"
+SRC_DIR="${LLAMA_ROOT}/src"
+STAMP="${BIN_DIR}/.build-id"
 UNIT="gpu-rent-llamacpp"
+REPO="https://github.com/ggml-org/llama.cpp.git"
+API_BASE="https://api.github.com/repos/ggml-org/llama.cpp"
 
 log() { echo "[gpu-rent-llamacpp] $*"; }
 
@@ -20,84 +25,224 @@ mkdir -p "$MODELS_DIR" "$BIN_DIR"
 chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT"
 
 SERVER_BIN="${BIN_DIR}/llama-server"
-if [[ ! -x "$SERVER_BIN" ]]; then
-  log "скачиваю llama-server (cuda) release…"
-  # Pin: LLAMACPP_TAG=b4690  LLAMACPP_ASSET_URL=...  LLAMACPP_SHA256=...
-  # Без pin — latest release (supply-chain risk; docs/llm.md).
-  TMP="$(mktemp -d)"
-  cd "$TMP"
-  LLAMACPP_TAG="${LLAMACPP_TAG:-}"
-  LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
-  LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
-  if [[ -n "$LLAMACPP_ASSET_URL" ]]; then
-    URL="$LLAMACPP_ASSET_URL"
-  elif [[ -n "$LLAMACPP_TAG" ]]; then
-    API="https://api.github.com/repos/ggerganov/llama.cpp/releases/tags/${LLAMACPP_TAG}"
-    URL="$(curl -fsSL "$API" | python3 -c '
+LLAMACPP_TAG="${LLAMACPP_TAG:-}"
+LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
+LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
+
+# Pick a Linux release asset (never Windows/macOS). Prefer ubuntu CUDA → vulkan → cpu.
+pick_linux_asset_url() {
+  python3 -c '
 import json,sys
 data=json.load(sys.stdin)
 assets=data.get("assets") or []
-prefer=[]
+cands=[]
 for a in assets:
     n=(a.get("name") or "").lower()
     u=a.get("browser_download_url") or ""
     if not (u.endswith(".zip") or u.endswith(".tar.gz")):
         continue
-    if "cuda" in n or "cu12" in n or "cu11" in n:
-        prefer.append(u)
-    elif "ubuntu" in n or "linux" in n:
-        prefer.append(u)
-print(prefer[0] if prefer else "")
-')"
-  else
-    log "WARN: LLAMACPP_TAG/ASSET_URL не заданы — берём latest (нет pin). См. docs/llm.md"
-    API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
-    URL="$(curl -fsSL "$API" | python3 -c '
-import json,sys
-data=json.load(sys.stdin)
-assets=data.get("assets") or []
-prefer=[]
-for a in assets:
-    n=(a.get("name") or "").lower()
-    u=a.get("browser_download_url") or ""
-    if not (u.endswith(".zip") or u.endswith(".tar.gz")):
+    if any(x in n for x in ("win","macos","android","darwin","xcframework","-ui.")):
         continue
-    if "cuda" in n or "cu12" in n or "cu11" in n:
-        prefer.append(u)
+    if "cudart" in n:
+        continue
+    score=0
+    if "ubuntu" in n and "x64" in n and "cuda" in n:
+        score=100
+    elif "linux" in n and "cuda" in n:
+        score=90
+    elif "ubuntu" in n and "vulkan" in n and "x64" in n:
+        score=50
+    elif "ubuntu" in n and "x64" in n and not any(
+        x in n for x in ("sycl","openvino","arm","s390","rocm")
+    ):
+        score=30
     elif "ubuntu" in n or "linux" in n:
-        prefer.append(u)
-print(prefer[0] if prefer else "")
-')"
+        score=10
+    if score:
+        cands.append((score, u, n))
+cands.sort(reverse=True)
+print(cands[0][1] if cands else "")
+'
+}
+
+resolve_release_tag() {
+  if [[ -n "$LLAMACPP_TAG" ]]; then
+    echo "$LLAMACPP_TAG"
+    return
   fi
-  if [[ -z "$URL" ]]; then
-    log "не нашёл бинарь в release — поставь llama-server вручную в ${SERVER_BIN}"
-    exit 1
+  log "WARN: LLAMACPP_TAG/ASSET_URL не заданы — берём latest (нет pin). См. docs/llm.md"
+  curl -fsSL "${API_BASE}/releases/latest" | python3 -c \
+    'import json,sys; print(json.load(sys.stdin).get("tag_name") or "")'
+}
+
+cuda_architectures() {
+  python3 - <<'PY'
+import json
+from pathlib import Path
+p = Path("/mnt/swarm_data/.gpu-rent-gpu.json")
+cap = "8.9"
+if p.is_file():
+    try:
+        cap = str(json.loads(p.read_text()).get("compute_cap") or cap)
+    except Exception:
+        pass
+parts = cap.split(".")
+try:
+    maj, mnr = int(parts[0]), int(parts[1]) if len(parts) > 1 else 0
+    print(f"{maj}{mnr}")
+except ValueError:
+    print("89")
+PY
+}
+
+ensure_build_deps() {
+  export DEBIAN_FRONTEND=noninteractive
+  apt-get install -y -qq \
+    cmake build-essential git curl ca-certificates \
+    libcurl4-openssl-dev >/dev/null
+  if command -v nvcc >/dev/null 2>&1; then
+    return 0
   fi
-  log "asset $URL"
-  curl -fL "$URL" -o pkg.bin
-  if [[ -n "$LLAMACPP_SHA256" ]]; then
-    echo "${LLAMACPP_SHA256}  pkg.bin" | sha256sum -c -
+  if [[ -x /usr/local/cuda/bin/nvcc ]]; then
+    export PATH="/usr/local/cuda/bin:${PATH}"
+    return 0
+  fi
+  log "ставлю nvidia-cuda-toolkit (нужен nvcc для сборки)…"
+  apt-get install -y -qq nvidia-cuda-toolkit >/dev/null
+  if command -v nvcc >/dev/null 2>&1; then
+    return 0
+  fi
+  if [[ -x /usr/local/cuda/bin/nvcc ]]; then
+    export PATH="/usr/local/cuda/bin:${PATH}"
+    return 0
+  fi
+  return 1
+}
+
+install_from_archive_url() {
+  local url="$1"
+  local tmp
+  tmp="$(mktemp -d)"
+  (
+    cd "$tmp"
+    log "asset $url"
+    curl -fL "$url" -o pkg.bin
+    if [[ -n "$LLAMACPP_SHA256" ]]; then
+      echo "${LLAMACPP_SHA256}  pkg.bin" | sha256sum -c -
+    else
+      log "WARN: LLAMACPP_SHA256 не задан — checksum skip"
+    fi
+    if file pkg.bin | grep -qi zip; then
+      apt-get install -y -qq unzip >/dev/null 2>&1 || true
+      unzip -qo pkg.bin -d out
+    else
+      mkdir -p out
+      tar -xaf pkg.bin -C out 2>/dev/null || tar -xzf pkg.bin -C out
+    fi
+    local found
+    found="$(find out -type f -name 'llama-server' | head -n1 || true)"
+    if [[ -z "$found" ]]; then
+      found="$(find out -type f -name 'server' | head -n1 || true)"
+    fi
+    if [[ -z "$found" ]]; then
+      log "в архиве нет llama-server"
+      exit 1
+    fi
+    install -m 755 "$found" "$SERVER_BIN"
+    chown "${SWARM_USER}:${SWARM_USER}" "$SERVER_BIN"
+  )
+  local rc=$?
+  rm -rf "$tmp"
+  return "$rc"
+}
+
+build_cuda_from_source() {
+  local tag="$1"
+  local arch
+  arch="$(cuda_architectures)"
+  log "собираю llama-server CUDA из исходников (tag=${tag}, arch=${arch})…"
+  if ! ensure_build_deps; then
+    log "нет nvcc — CUDA-сборку пропускаем"
+    return 1
+  fi
+  log "nvcc $(nvcc --version 2>/dev/null | tail -n1 || echo '?')"
+
+  mkdir -p "$SRC_DIR"
+  if [[ -d "${SRC_DIR}/.git" ]]; then
+    git -C "$SRC_DIR" fetch --depth 1 origin tag "$tag" 2>/dev/null || true
+    if ! git -C "$SRC_DIR" checkout -f "$tag" 2>/dev/null; then
+      rm -rf "$SRC_DIR"
+      git clone --depth 1 --branch "$tag" "$REPO" "$SRC_DIR"
+    fi
   else
-    log "WARN: LLAMACPP_SHA256 не задан — checksum skip"
+    rm -rf "$SRC_DIR"
+    git clone --depth 1 --branch "$tag" "$REPO" "$SRC_DIR"
   fi
-  if file pkg.bin | grep -qi zip; then
-    apt-get install -y -qq unzip >/dev/null 2>&1 || true
-    unzip -qo pkg.bin -d out
-  else
-    mkdir -p out
-    tar -xaf pkg.bin -C out 2>/dev/null || tar -xzf pkg.bin -C out
+
+  cmake -S "$SRC_DIR" -B "${SRC_DIR}/build" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DGGML_CUDA=ON \
+    -DCMAKE_CUDA_ARCHITECTURES="${arch}" \
+    -DLLAMA_CURL=ON \
+    -DLLAMA_BUILD_SERVER=ON
+  cmake --build "${SRC_DIR}/build" -j"$(nproc)" --target llama-server
+
+  local built="${SRC_DIR}/build/bin/llama-server"
+  if [[ ! -x "$built" ]]; then
+    log "сборка не дала ${built}"
+    return 1
   fi
-  FOUND="$(find out -type f -name 'llama-server' | head -n1 || true)"
-  if [[ -z "$FOUND" ]]; then
-    FOUND="$(find out -type f -name 'server' | head -n1 || true)"
-  fi
-  if [[ -z "$FOUND" ]]; then
-    log "в архиве нет llama-server"
-    exit 1
-  fi
-  install -m 755 "$FOUND" "$SERVER_BIN"
+  install -m 755 "$built" "$SERVER_BIN"
   chown "${SWARM_USER}:${SWARM_USER}" "$SERVER_BIN"
-  rm -rf "$TMP"
+  echo "cuda:${tag}:${arch}" >"$STAMP"
+  chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
+  log "CUDA binary → ${SERVER_BIN}"
+  return 0
+}
+
+install_linux_release_fallback() {
+  local tag="$1"
+  local api url
+  api="${API_BASE}/releases/tags/${tag}"
+  url="$(curl -fsSL "$api" | pick_linux_asset_url)"
+  if [[ -z "$url" ]]; then
+    log "в release ${tag} нет Linux-ассета"
+    return 1
+  fi
+  if [[ "$url" == *vulkan* ]]; then
+    log "WARN: официального Linux CUDA нет — беру Ubuntu Vulkan prebuilt"
+  elif [[ "$url" != *cuda* ]]; then
+    log "WARN: беру Linux prebuilt без CUDA (CPU) — лучше собрать с nvcc"
+  fi
+  install_from_archive_url "$url"
+  echo "asset:${tag}" >"$STAMP"
+  chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
+}
+
+if [[ -x "$SERVER_BIN" ]]; then
+  log "llama-server уже есть: ${SERVER_BIN}"
+else
+  if [[ -n "$LLAMACPP_ASSET_URL" ]]; then
+    log "скачиваю llama-server по LLAMACPP_ASSET_URL…"
+    install_from_archive_url "$LLAMACPP_ASSET_URL"
+    echo "asset-url" >"$STAMP"
+    chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
+  else
+    tag="$(resolve_release_tag)"
+    if [[ -z "$tag" ]]; then
+      log "не удалось определить release tag"
+      exit 1
+    fi
+    if ! build_cuda_from_source "$tag"; then
+      log "CUDA-сборка не удалась — fallback на Linux release asset"
+      install_linux_release_fallback "$tag"
+    fi
+  fi
+fi
+
+if [[ ! -x "$SERVER_BIN" ]]; then
+  log "нет исполняемого ${SERVER_BIN}"
+  exit 1
 fi
 
 # Prefer a weights GGUF (skip mmproj), then attach --mmproj if present.
diff --git a/tests/test_llm_runtime.py b/tests/test_llm_runtime.py
index 4324206..3dde913 100644
--- a/tests/test_llm_runtime.py
+++ b/tests/test_llm_runtime.py
@@ -7,6 +7,7 @@ from gpu_rent.llm_runtime import (
     normalize_runtime,
     parse_llamacpp_models,
     parse_ollama_models,
+    pick_llamacpp_linux_asset_url,
     remap_llamacpp_url,
     write_ollama_models_preset,
 )
@@ -86,3 +87,39 @@ def test_remap_dead_bartowski_abliterate_url(tmp_path: Path):
     entries = parse_llamacpp_models(path)
     assert len(entries) == 1
     assert entries[0].url == fixed
+
+
+def test_pick_llamacpp_linux_asset_skips_windows_cuda():
+    assets = [
+        {
+            "name": "cudart-llama-bin-win-cuda-12.4-x64.zip",
+            "browser_download_url": "https://example/cudart-win.zip",
+        },
+        {
+            "name": "llama-b10545-bin-win-cuda-12.4-x64.zip",
+            "browser_download_url": "https://example/win-cuda.zip",
+        },
+        {
+            "name": "llama-b10545-bin-ubuntu-x64.tar.gz",
+            "browser_download_url": "https://example/ubuntu-cpu.tar.gz",
+        },
+        {
+            "name": "llama-b10545-bin-ubuntu-vulkan-x64.tar.gz",
+            "browser_download_url": "https://example/ubuntu-vulkan.tar.gz",
+        },
+    ]
+    assert pick_llamacpp_linux_asset_url(assets) == "https://example/ubuntu-vulkan.tar.gz"
+
+
+def test_pick_llamacpp_linux_asset_prefers_ubuntu_cuda():
+    assets = [
+        {
+            "name": "llama-b1-bin-ubuntu-vulkan-x64.tar.gz",
+            "browser_download_url": "https://example/vulkan.tar.gz",
+        },
+        {
+            "name": "llama-b1-bin-ubuntu-cuda-12.4-x64.tar.gz",
+            "browser_download_url": "https://example/cuda.tar.gz",
+        },
+    ]
+    assert pick_llamacpp_linux_asset_url(assets) == "https://example/cuda.tar.gz"