Implement UP_STOP_ON_FAIL option to manage GPU state on installation failure
- Added a new configuration option `UP_STOP_ON_FAIL` to control whether the GPU should be stopped automatically if the `up` command fails, enhancing user control over resource management. - Updated the CLI to include a `--keep-on-fail` flag, allowing users to prevent GPU shutdown during installation errors. - Enhanced the installation scripts and documentation to reflect these changes, providing clearer guidance on the new behavior and configuration options. - Improved error handling in the CLI to ensure proper cleanup of resources in case of failure, preventing unexpected billing for unused GPU resources.
This commit is contained in:
@@ -57,6 +57,21 @@ def _die(exc: BaseException) -> None:
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def _stop_after_failed_up(cfg, cause: BaseException) -> None:
|
||||
"""Delete compute after a failed up so billing does not continue unnoticed."""
|
||||
warn(
|
||||
"up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)"
|
||||
)
|
||||
try:
|
||||
cmd_stop(cfg, no_pull=True, log=log)
|
||||
ok("compute остановлен, диски на месте")
|
||||
except Exception as stop_exc:
|
||||
err(
|
||||
f"auto-stop не удался: {stop_exc} — срочно: gpu-rent stop "
|
||||
f"(причина up: {cause})"
|
||||
)
|
||||
|
||||
|
||||
def _print_checks(checks, *, quiet: bool = False) -> int:
|
||||
failed = blocking_failed(checks)
|
||||
if quiet:
|
||||
@@ -439,6 +454,11 @@ def up(
|
||||
"--no-update",
|
||||
help="Не делать git pull SwarmUI и установленных extensions",
|
||||
),
|
||||
keep_on_fail: bool = typer.Option(
|
||||
False,
|
||||
"--keep-on-fail",
|
||||
help="Не гасить GPU если up упал (по умолчанию stop; UP_STOP_ON_FAIL=false)",
|
||||
),
|
||||
verbose: bool = typer.Option(
|
||||
False,
|
||||
"--verbose",
|
||||
@@ -458,6 +478,8 @@ def up(
|
||||
),
|
||||
) -> None:
|
||||
"""Create/unshelve GPU; SwarmUI и/или LLM; по умолчанию туннель."""
|
||||
cfg = None
|
||||
up_ok = False
|
||||
try:
|
||||
from dataclasses import replace
|
||||
|
||||
@@ -680,6 +702,7 @@ def up(
|
||||
ask=None if yes else ask,
|
||||
log=log,
|
||||
)
|
||||
up_ok = True
|
||||
if no_tunnel:
|
||||
from gpu_rent.access_card import print_access_card
|
||||
|
||||
@@ -705,7 +728,24 @@ def up(
|
||||
open_browser=open_browser,
|
||||
log=log,
|
||||
)
|
||||
except KeyboardInterrupt as exc:
|
||||
if (
|
||||
not up_ok
|
||||
and cfg is not None
|
||||
and not keep_on_fail
|
||||
and bool(getattr(cfg, "up_stop_on_fail", True))
|
||||
):
|
||||
_stop_after_failed_up(cfg, exc)
|
||||
err("прервано (Ctrl+C)")
|
||||
raise typer.Exit(130) from exc
|
||||
except GpuRentError as exc:
|
||||
if (
|
||||
not up_ok
|
||||
and cfg is not None
|
||||
and not keep_on_fail
|
||||
and bool(getattr(cfg, "up_stop_on_fail", True))
|
||||
):
|
||||
_stop_after_failed_up(cfg, exc)
|
||||
_die(exc)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user