Implement UP_STOP_ON_FAIL option to manage GPU state on installation failure

- Added a new configuration option `UP_STOP_ON_FAIL` to control whether the GPU should be stopped automatically if the `up` command fails, enhancing user control over resource management.
- Updated the CLI to include a `--keep-on-fail` flag, allowing users to prevent GPU shutdown during installation errors.
- Enhanced the installation scripts and documentation to reflect these changes, providing clearer guidance on the new behavior and configuration options.
- Improved error handling in the CLI to ensure proper cleanup of resources in case of failure, preventing unexpected billing for unused GPU resources.
This commit is contained in:
Leonid Pershin
2026-08-21 08:32:33 +03:00
parent 5081a6bc1e
commit 6871c511c4
9 changed files with 179 additions and 14 deletions
+3
View File
@@ -121,6 +121,7 @@ class Config:
idle_grace_minutes: int
pull_output: bool
notify_ready: bool
up_stop_on_fail: bool
selectel_api_token: str
balance_notify_step_rub: float
balance_notify_low_rub: float
@@ -258,6 +259,8 @@ def load_config(*, require_auth: bool = True) -> Config:
idle_grace_minutes=_as_int(os.environ.get("IDLE_GRACE_MINUTES"), 90),
pull_output=_as_bool(os.environ.get("PULL_OUTPUT"), False),
notify_ready=_as_bool(os.environ.get("NOTIFY_READY"), True),
# Default on: failed up must not leave a billing GPU running.
up_stop_on_fail=_as_bool(os.environ.get("UP_STOP_ON_FAIL"), True),
selectel_api_token=(os.environ.get("SELECTEL_API_TOKEN") or "").strip(),
balance_notify_step_rub=_as_float(os.environ.get("BALANCE_NOTIFY_STEP_RUB"), 200.0),
balance_notify_low_rub=_as_float(os.environ.get("BALANCE_NOTIFY_LOW_RUB"), 0.0),