Implement UP_STOP_ON_FAIL option to manage GPU state on installation failure

- Added a new configuration option `UP_STOP_ON_FAIL` to control whether the GPU should be stopped automatically if the `up` command fails, enhancing user control over resource management.
- Updated the CLI to include a `--keep-on-fail` flag, allowing users to prevent GPU shutdown during installation errors.
- Enhanced the installation scripts and documentation to reflect these changes, providing clearer guidance on the new behavior and configuration options.
- Improved error handling in the CLI to ensure proper cleanup of resources in case of failure, preventing unexpected billing for unused GPU resources.

src/gpu_rent/provision.py

@@ -36,6 +36,7 @@ _LLAMACPP_INSTALL_ENV = (
     "LLAMACPP_ASSET_URL",
     "LLAMACPP_SHA256",
     "LLAMACPP_BUILD_CUDA",
+    "LLAMACPP_BACKEND",
     "LLAMACPP_FORCE_REINSTALL",
     "LLAMACPP_NGL",
     "LLAMACPP_CTX",
@@ -544,13 +545,13 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None:
         log("LLM: ставим/запускаем llama.cpp server")
         import os
 
-        build_cuda = (os.environ.get("LLAMACPP_BUILD_CUDA") or "").strip() == "1"
+        # Vulkan finishes in seconds; CUDA compile needs up to ~15–20 min.
         run_script_sudo(
             cfg,
             host,
             _pkg_text("install_llamacpp.sh"),
             remote_path="/tmp/gpu-rent-install_llamacpp.sh",
-            timeout=3600 if build_cuda else 1200,
+            timeout=3600,
             env=_remote_llm_env(cfg, *_LLAMACPP_INSTALL_ENV),
             log=log,
         )