Enhance LLM runtime and installation script for Linux support
- Added a new function `pick_llamacpp_linux_asset_url` to select appropriate Linux release assets, prioritizing Ubuntu CUDA and Vulkan options while excluding Windows and macOS binaries. - Updated the installation script to build `llama-server` from source when Linux CUDA binaries are unavailable, improving compatibility and flexibility. - Revised documentation to reflect changes in asset handling and installation procedures. - Added tests to validate the new asset selection logic, ensuring correct behavior in various scenarios.
This commit is contained in:
Leonid Pershin
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install llama-server (CUDA) for OpenAI-compatible API on loopback :8080.
|
||||
# Official GitHub releases ship Windows CUDA only — on Linux we build from source.
|
||||
set -euo pipefail
|
||||
|
||||
SWARM_USER="${SWARM_USER:-ubuntu}"
|
||||
@@ -7,7 +8,11 @@ DATA_ROOT="/mnt/swarm_data"
|
||||
LLAMA_ROOT="${DATA_ROOT}/llamacpp"
|
||||
MODELS_DIR="${LLAMA_ROOT}/models"
|
||||
BIN_DIR="${LLAMA_ROOT}/bin"
|
||||
SRC_DIR="${LLAMA_ROOT}/src"
|
||||
STAMP="${BIN_DIR}/.build-id"
|
||||
UNIT="gpu-rent-llamacpp"
|
||||
REPO="https://github.com/ggml-org/llama.cpp.git"
|
||||
API_BASE="https://api.github.com/repos/ggml-org/llama.cpp"
|
||||
|
||||
log() { echo "[gpu-rent-llamacpp] $*"; }
|
||||
|
||||
@@ -20,84 +25,224 @@ mkdir -p "$MODELS_DIR" "$BIN_DIR"
|
||||
chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT"
|
||||
|
||||
SERVER_BIN="${BIN_DIR}/llama-server"
|
||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
log "скачиваю llama-server (cuda) release…"
|
||||
# Pin: LLAMACPP_TAG=b4690 LLAMACPP_ASSET_URL=... LLAMACPP_SHA256=...
|
||||
# Без pin — latest release (supply-chain risk; docs/llm.md).
|
||||
TMP="$(mktemp -d)"
|
||||
cd "$TMP"
|
||||
LLAMACPP_TAG="${LLAMACPP_TAG:-}"
|
||||
LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
|
||||
LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
|
||||
if [[ -n "$LLAMACPP_ASSET_URL" ]]; then
|
||||
URL="$LLAMACPP_ASSET_URL"
|
||||
elif [[ -n "$LLAMACPP_TAG" ]]; then
|
||||
API="https://api.github.com/repos/ggerganov/llama.cpp/releases/tags/${LLAMACPP_TAG}"
|
||||
URL="$(curl -fsSL "$API" | python3 -c '
|
||||
LLAMACPP_TAG="${LLAMACPP_TAG:-}"
|
||||
LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
|
||||
LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
|
||||
|
||||
# Pick a Linux release asset (never Windows/macOS). Prefer ubuntu CUDA → vulkan → cpu.
|
||||
pick_linux_asset_url() {
|
||||
python3 -c '
|
||||
import json,sys
|
||||
data=json.load(sys.stdin)
|
||||
assets=data.get("assets") or []
|
||||
prefer=[]
|
||||
cands=[]
|
||||
for a in assets:
|
||||
n=(a.get("name") or "").lower()
|
||||
u=a.get("browser_download_url") or ""
|
||||
if not (u.endswith(".zip") or u.endswith(".tar.gz")):
|
||||
continue
|
||||
if "cuda" in n or "cu12" in n or "cu11" in n:
|
||||
prefer.append(u)
|
||||
elif "ubuntu" in n or "linux" in n:
|
||||
prefer.append(u)
|
||||
print(prefer[0] if prefer else "")
|
||||
')"
|
||||
else
|
||||
log "WARN: LLAMACPP_TAG/ASSET_URL не заданы — берём latest (нет pin). См. docs/llm.md"
|
||||
API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest"
|
||||
URL="$(curl -fsSL "$API" | python3 -c '
|
||||
import json,sys
|
||||
data=json.load(sys.stdin)
|
||||
assets=data.get("assets") or []
|
||||
prefer=[]
|
||||
for a in assets:
|
||||
n=(a.get("name") or "").lower()
|
||||
u=a.get("browser_download_url") or ""
|
||||
if not (u.endswith(".zip") or u.endswith(".tar.gz")):
|
||||
if any(x in n for x in ("win","macos","android","darwin","xcframework","-ui.")):
|
||||
continue
|
||||
if "cuda" in n or "cu12" in n or "cu11" in n:
|
||||
prefer.append(u)
|
||||
if "cudart" in n:
|
||||
continue
|
||||
score=0
|
||||
if "ubuntu" in n and "x64" in n and "cuda" in n:
|
||||
score=100
|
||||
elif "linux" in n and "cuda" in n:
|
||||
score=90
|
||||
elif "ubuntu" in n and "vulkan" in n and "x64" in n:
|
||||
score=50
|
||||
elif "ubuntu" in n and "x64" in n and not any(
|
||||
x in n for x in ("sycl","openvino","arm","s390","rocm")
|
||||
):
|
||||
score=30
|
||||
elif "ubuntu" in n or "linux" in n:
|
||||
prefer.append(u)
|
||||
print(prefer[0] if prefer else "")
|
||||
')"
|
||||
score=10
|
||||
if score:
|
||||
cands.append((score, u, n))
|
||||
cands.sort(reverse=True)
|
||||
print(cands[0][1] if cands else "")
|
||||
'
|
||||
}
|
||||
|
||||
resolve_release_tag() {
|
||||
if [[ -n "$LLAMACPP_TAG" ]]; then
|
||||
echo "$LLAMACPP_TAG"
|
||||
return
|
||||
fi
|
||||
if [[ -z "$URL" ]]; then
|
||||
log "не нашёл бинарь в release — поставь llama-server вручную в ${SERVER_BIN}"
|
||||
exit 1
|
||||
log "WARN: LLAMACPP_TAG/ASSET_URL не заданы — берём latest (нет pin). См. docs/llm.md"
|
||||
curl -fsSL "${API_BASE}/releases/latest" | python3 -c \
|
||||
'import json,sys; print(json.load(sys.stdin).get("tag_name") or "")'
|
||||
}
|
||||
|
||||
cuda_architectures() {
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
p = Path("/mnt/swarm_data/.gpu-rent-gpu.json")
|
||||
cap = "8.9"
|
||||
if p.is_file():
|
||||
try:
|
||||
cap = str(json.loads(p.read_text()).get("compute_cap") or cap)
|
||||
except Exception:
|
||||
pass
|
||||
parts = cap.split(".")
|
||||
try:
|
||||
maj, mnr = int(parts[0]), int(parts[1]) if len(parts) > 1 else 0
|
||||
print(f"{maj}{mnr}")
|
||||
except ValueError:
|
||||
print("89")
|
||||
PY
|
||||
}
|
||||
|
||||
ensure_build_deps() {
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get install -y -qq \
|
||||
cmake build-essential git curl ca-certificates \
|
||||
libcurl4-openssl-dev >/dev/null
|
||||
if command -v nvcc >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
log "asset $URL"
|
||||
curl -fL "$URL" -o pkg.bin
|
||||
if [[ -n "$LLAMACPP_SHA256" ]]; then
|
||||
echo "${LLAMACPP_SHA256} pkg.bin" | sha256sum -c -
|
||||
if [[ -x /usr/local/cuda/bin/nvcc ]]; then
|
||||
export PATH="/usr/local/cuda/bin:${PATH}"
|
||||
return 0
|
||||
fi
|
||||
log "ставлю nvidia-cuda-toolkit (нужен nvcc для сборки)…"
|
||||
apt-get install -y -qq nvidia-cuda-toolkit >/dev/null
|
||||
if command -v nvcc >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
if [[ -x /usr/local/cuda/bin/nvcc ]]; then
|
||||
export PATH="/usr/local/cuda/bin:${PATH}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
install_from_archive_url() {
|
||||
local url="$1"
|
||||
local tmp
|
||||
tmp="$(mktemp -d)"
|
||||
(
|
||||
cd "$tmp"
|
||||
log "asset $url"
|
||||
curl -fL "$url" -o pkg.bin
|
||||
if [[ -n "$LLAMACPP_SHA256" ]]; then
|
||||
echo "${LLAMACPP_SHA256} pkg.bin" | sha256sum -c -
|
||||
else
|
||||
log "WARN: LLAMACPP_SHA256 не задан — checksum skip"
|
||||
fi
|
||||
if file pkg.bin | grep -qi zip; then
|
||||
apt-get install -y -qq unzip >/dev/null 2>&1 || true
|
||||
unzip -qo pkg.bin -d out
|
||||
else
|
||||
mkdir -p out
|
||||
tar -xaf pkg.bin -C out 2>/dev/null || tar -xzf pkg.bin -C out
|
||||
fi
|
||||
local found
|
||||
found="$(find out -type f -name 'llama-server' | head -n1 || true)"
|
||||
if [[ -z "$found" ]]; then
|
||||
found="$(find out -type f -name 'server' | head -n1 || true)"
|
||||
fi
|
||||
if [[ -z "$found" ]]; then
|
||||
log "в архиве нет llama-server"
|
||||
exit 1
|
||||
fi
|
||||
install -m 755 "$found" "$SERVER_BIN"
|
||||
chown "${SWARM_USER}:${SWARM_USER}" "$SERVER_BIN"
|
||||
)
|
||||
local rc=$?
|
||||
rm -rf "$tmp"
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
build_cuda_from_source() {
|
||||
local tag="$1"
|
||||
local arch
|
||||
arch="$(cuda_architectures)"
|
||||
log "собираю llama-server CUDA из исходников (tag=${tag}, arch=${arch})…"
|
||||
if ! ensure_build_deps; then
|
||||
log "нет nvcc — CUDA-сборку пропускаем"
|
||||
return 1
|
||||
fi
|
||||
log "nvcc $(nvcc --version 2>/dev/null | tail -n1 || echo '?')"
|
||||
|
||||
mkdir -p "$SRC_DIR"
|
||||
if [[ -d "${SRC_DIR}/.git" ]]; then
|
||||
git -C "$SRC_DIR" fetch --depth 1 origin tag "$tag" 2>/dev/null || true
|
||||
if ! git -C "$SRC_DIR" checkout -f "$tag" 2>/dev/null; then
|
||||
rm -rf "$SRC_DIR"
|
||||
git clone --depth 1 --branch "$tag" "$REPO" "$SRC_DIR"
|
||||
fi
|
||||
else
|
||||
log "WARN: LLAMACPP_SHA256 не задан — checksum skip"
|
||||
rm -rf "$SRC_DIR"
|
||||
git clone --depth 1 --branch "$tag" "$REPO" "$SRC_DIR"
|
||||
fi
|
||||
if file pkg.bin | grep -qi zip; then
|
||||
apt-get install -y -qq unzip >/dev/null 2>&1 || true
|
||||
unzip -qo pkg.bin -d out
|
||||
else
|
||||
mkdir -p out
|
||||
tar -xaf pkg.bin -C out 2>/dev/null || tar -xzf pkg.bin -C out
|
||||
|
||||
cmake -S "$SRC_DIR" -B "${SRC_DIR}/build" \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_CUDA=ON \
|
||||
-DCMAKE_CUDA_ARCHITECTURES="${arch}" \
|
||||
-DLLAMA_CURL=ON \
|
||||
-DLLAMA_BUILD_SERVER=ON
|
||||
cmake --build "${SRC_DIR}/build" -j"$(nproc)" --target llama-server
|
||||
|
||||
local built="${SRC_DIR}/build/bin/llama-server"
|
||||
if [[ ! -x "$built" ]]; then
|
||||
log "сборка не дала ${built}"
|
||||
return 1
|
||||
fi
|
||||
FOUND="$(find out -type f -name 'llama-server' | head -n1 || true)"
|
||||
if [[ -z "$FOUND" ]]; then
|
||||
FOUND="$(find out -type f -name 'server' | head -n1 || true)"
|
||||
fi
|
||||
if [[ -z "$FOUND" ]]; then
|
||||
log "в архиве нет llama-server"
|
||||
exit 1
|
||||
fi
|
||||
install -m 755 "$FOUND" "$SERVER_BIN"
|
||||
install -m 755 "$built" "$SERVER_BIN"
|
||||
chown "${SWARM_USER}:${SWARM_USER}" "$SERVER_BIN"
|
||||
rm -rf "$TMP"
|
||||
echo "cuda:${tag}:${arch}" >"$STAMP"
|
||||
chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
|
||||
log "CUDA binary → ${SERVER_BIN}"
|
||||
return 0
|
||||
}
|
||||
|
||||
install_linux_release_fallback() {
|
||||
local tag="$1"
|
||||
local api url
|
||||
api="${API_BASE}/releases/tags/${tag}"
|
||||
url="$(curl -fsSL "$api" | pick_linux_asset_url)"
|
||||
if [[ -z "$url" ]]; then
|
||||
log "в release ${tag} нет Linux-ассета"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$url" == *vulkan* ]]; then
|
||||
log "WARN: официального Linux CUDA нет — беру Ubuntu Vulkan prebuilt"
|
||||
elif [[ "$url" != *cuda* ]]; then
|
||||
log "WARN: беру Linux prebuilt без CUDA (CPU) — лучше собрать с nvcc"
|
||||
fi
|
||||
install_from_archive_url "$url"
|
||||
echo "asset:${tag}" >"$STAMP"
|
||||
chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
|
||||
}
|
||||
|
||||
if [[ -x "$SERVER_BIN" ]]; then
|
||||
log "llama-server уже есть: ${SERVER_BIN}"
|
||||
else
|
||||
if [[ -n "$LLAMACPP_ASSET_URL" ]]; then
|
||||
log "скачиваю llama-server по LLAMACPP_ASSET_URL…"
|
||||
install_from_archive_url "$LLAMACPP_ASSET_URL"
|
||||
echo "asset-url" >"$STAMP"
|
||||
chown "${SWARM_USER}:${SWARM_USER}" "$STAMP"
|
||||
else
|
||||
tag="$(resolve_release_tag)"
|
||||
if [[ -z "$tag" ]]; then
|
||||
log "не удалось определить release tag"
|
||||
exit 1
|
||||
fi
|
||||
if ! build_cuda_from_source "$tag"; then
|
||||
log "CUDA-сборка не удалась — fallback на Linux release asset"
|
||||
install_linux_release_fallback "$tag"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ! -x "$SERVER_BIN" ]]; then
|
||||
log "нет исполняемого ${SERVER_BIN}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Prefer a weights GGUF (skip mmproj), then attach --mmproj if present.
|
||||
|
||||
Reference in New Issue
Block a user