Implement UP_STOP_ON_FAIL option to manage GPU state on installation failure

- Added a new configuration option `UP_STOP_ON_FAIL` to control whether the GPU should be stopped automatically if the `up` command fails, enhancing user control over resource management.
- Updated the CLI to include a `--keep-on-fail` flag, allowing users to prevent GPU shutdown during installation errors.
- Enhanced the installation scripts and documentation to reflect these changes, providing clearer guidance on the new behavior and configuration options.
- Improved error handling in the CLI to ensure proper cleanup of resources in case of failure, preventing unexpected billing for unused GPU resources.
This commit is contained in:
Leonid Pershin
2026-08-21 08:32:33 +03:00
parent 5081a6bc1e
commit 6871c511c4
9 changed files with 179 additions and 14 deletions
+40
View File
@@ -57,6 +57,21 @@ def _die(exc: BaseException) -> None:
raise typer.Exit(1)
def _stop_after_failed_up(cfg, cause: BaseException) -> None:
"""Delete compute after a failed up so billing does not continue unnoticed."""
warn(
"up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)"
)
try:
cmd_stop(cfg, no_pull=True, log=log)
ok("compute остановлен, диски на месте")
except Exception as stop_exc:
err(
f"auto-stop не удался: {stop_exc} — срочно: gpu-rent stop "
f"(причина up: {cause})"
)
def _print_checks(checks, *, quiet: bool = False) -> int:
failed = blocking_failed(checks)
if quiet:
@@ -439,6 +454,11 @@ def up(
"--no-update",
help="Не делать git pull SwarmUI и установленных extensions",
),
keep_on_fail: bool = typer.Option(
False,
"--keep-on-fail",
help="Не гасить GPU если up упал (по умолчанию stop; UP_STOP_ON_FAIL=false)",
),
verbose: bool = typer.Option(
False,
"--verbose",
@@ -458,6 +478,8 @@ def up(
),
) -> None:
"""Create/unshelve GPU; SwarmUI и/или LLM; по умолчанию туннель."""
cfg = None
up_ok = False
try:
from dataclasses import replace
@@ -680,6 +702,7 @@ def up(
ask=None if yes else ask,
log=log,
)
up_ok = True
if no_tunnel:
from gpu_rent.access_card import print_access_card
@@ -705,7 +728,24 @@ def up(
open_browser=open_browser,
log=log,
)
except KeyboardInterrupt as exc:
if (
not up_ok
and cfg is not None
and not keep_on_fail
and bool(getattr(cfg, "up_stop_on_fail", True))
):
_stop_after_failed_up(cfg, exc)
err("прервано (Ctrl+C)")
raise typer.Exit(130) from exc
except GpuRentError as exc:
if (
not up_ok
and cfg is not None
and not keep_on_fail
and bool(getattr(cfg, "up_stop_on_fail", True))
):
_stop_after_failed_up(cfg, exc)
_die(exc)