Implement UP_STOP_ON_FAIL option to manage GPU state on installation failure

- Added a new configuration option `UP_STOP_ON_FAIL` to control whether the GPU should be stopped automatically if the `up` command fails, enhancing user control over resource management.
- Updated the CLI to include a `--keep-on-fail` flag, allowing users to prevent GPU shutdown during installation errors.
- Enhanced the installation scripts and documentation to reflect these changes, providing clearer guidance on the new behavior and configuration options.
- Improved error handling in the CLI to ensure proper cleanup of resources in case of failure, preventing unexpected billing for unused GPU resources.

src/gpu_rent/remote/install_llamacpp.sh

@@ -31,8 +31,10 @@ SERVER_BIN="${BIN_DIR}/llama-server"
 LLAMACPP_TAG="${LLAMACPP_TAG:-}"
 LLAMACPP_ASSET_URL="${LLAMACPP_ASSET_URL:-}"
 LLAMACPP_SHA256="${LLAMACPP_SHA256:-}"
-# 1 = force CUDA compile; 0/empty = prebuilt first, build only if prebuilt fails
+# 1 = force CUDA compile; 0 = never compile (Vulkan/CPU prebuilt only)
 LLAMACPP_BUILD_CUDA="${LLAMACPP_BUILD_CUDA:-}"
+# auto | cuda | vulkan — default auto: CUDA if nvcc already on VM, else Vulkan prebuilt
+LLAMACPP_BACKEND="${LLAMACPP_BACKEND:-auto}"
 LLAMACPP_FORCE_REINSTALL="${LLAMACPP_FORCE_REINSTALL:-}"
 LLAMACPP_NGL="${LLAMACPP_NGL:-}"
 LLAMACPP_CTX="${LLAMACPP_CTX:-}"
@@ -40,11 +42,48 @@ LLAMACPP_HOST="${LLAMACPP_HOST:-127.0.0.1}"
 LLAMACPP_PORT="${LLAMACPP_PORT:-8080}"
 LLAMACPP_EXTRA_ARGS="${LLAMACPP_EXTRA_ARGS:-}"
 
+have_nvcc() {
+  if command -v nvcc >/dev/null 2>&1; then
+    return 0
+  fi
+  if [[ -x /usr/local/cuda/bin/nvcc ]]; then
+    export PATH="/usr/local/cuda/bin:${PATH}"
+    return 0
+  fi
+  return 1
+}
+
+# Prefer CUDA when toolkit already present (GPU images / prior up). Vulkan = fast no-compile.
+want_cuda_build() {
+  case "${LLAMACPP_BUILD_CUDA}" in
+    1|yes|true) return 0 ;;
+    0|no|false) return 1 ;;
+  esac
+  case "${LLAMACPP_BACKEND}" in
+    cuda) return 0 ;;
+    vulkan) return 1 ;;
+    *)
+      if have_nvcc; then
+        return 0
+      fi
+      return 1
+      ;;
+  esac
+}
+
 if [[ "${LLAMACPP_FORCE_REINSTALL}" == "1" ]]; then
   log "LLAMACPP_FORCE_REINSTALL=1 — удаляю старый бинарь"
   rm -f "$SERVER_BIN" "$STAMP"
 fi
 
+# Upgrade path: previous default was Vulkan prebuilt; if nvcc is here, prefer CUDA.
+if [[ -x "$SERVER_BIN" && -f "$STAMP" && "${LLAMACPP_BACKEND}" != "vulkan" && "${LLAMACPP_BUILD_CUDA}" != "0" ]]; then
+  if grep -q '^asset:' "$STAMP" 2>/dev/null && want_cuda_build; then
+    log "был Vulkan/CPU prebuilt, nvcc есть — пересобираю CUDA (лучше на NVIDIA)"
+    rm -f "$SERVER_BIN" "$STAMP"
+  fi
+fi
+
 # Prefer ubuntu CUDA (rare) → vulkan → cpu. Never Windows/macOS/cudart-only.
 pick_linux_asset_url() {
   python3 -c '
@@ -321,12 +360,18 @@ else
     fi
 
     installed=0
-    if [[ "${LLAMACPP_BUILD_CUDA}" == "1" ]]; then
-      log "LLAMACPP_BUILD_CUDA=1 — сразу CUDA-сборка"
+    if want_cuda_build; then
+      log "backend: CUDA (nvcc есть или LLAMACPP_BACKEND/BUILD_CUDA) — сборка, Vulkan только если упадёт"
       if build_cuda_from_source "$tag"; then
         installed=1
+      else
+        log "CUDA-сборка не вышла — fallback на Linux prebuilt (Vulkan/CPU)"
+        if install_linux_release "$tag"; then
+          installed=1
+        fi
       fi
     else
+      log "backend: Linux prebuilt (нет nvcc / LLAMACPP_BACKEND=vulkan) — без compile"
       if install_linux_release "$tag"; then
         installed=1
       else