Implement UP_STOP_ON_FAIL option to manage GPU state on installation failure

- Added a new configuration option `UP_STOP_ON_FAIL` to control whether the GPU should be stopped automatically if the `up` command fails, enhancing user control over resource management. - Updated the CLI to include a `--keep-on-fail` flag, allowing users to prevent GPU shutdown during installation errors. - Enhanced the installation scripts and documentation to reflect these changes, providing clearer guidance on the new behavior and configuration options. - Improved error handling in the CLI to ensure proper cleanup of resources in case of failure, preventing unexpected billing for unused GPU resources.
2026-08-21 08:32:33 +03:00
parent 5081a6bc1e
commit 6871c511c4
9 changed files with 179 additions and 14 deletions
@@ -0,0 +1,60 @@
+from types import SimpleNamespace
+
+from gpu_rent.cli import _stop_after_failed_up
+
+
+def test_stop_after_failed_up_calls_cmd_stop(monkeypatch):
+    calls: list[dict] = []
+
+    def fake_stop(cfg, *, no_pull=False, log=None, destroy_disks=False):
+        calls.append({"no_pull": no_pull, "destroy_disks": destroy_disks})
+        return SimpleNamespace()
+
+    monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop)
+    monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None)
+
+    _stop_after_failed_up(SimpleNamespace(), RuntimeError("boom"))
+    assert calls == [{"no_pull": True, "destroy_disks": False}]
+
+
+def test_up_stop_on_fail_default_true(monkeypatch, tmp_path):
+    monkeypatch.setenv("HOME", str(tmp_path))
+    monkeypatch.setenv("USERPROFILE", str(tmp_path))
+    for key in (
+        "OS_AUTH_URL",
+        "OS_USER_DOMAIN_NAME",
+        "OS_USERNAME",
+        "OS_PASSWORD",
+        "OS_PROJECT_ID",
+        "OS_REGION_NAME",
+        "GPU_RENT_AZ",
+    ):
+        monkeypatch.setenv(key, "x")
+    monkeypatch.delenv("UP_STOP_ON_FAIL", raising=False)
+    from gpu_rent.config import load_config
+
+    cfg = load_config(require_auth=False)
+    assert cfg.up_stop_on_fail is True
+
+
+def test_up_stop_on_fail_can_disable(monkeypatch, tmp_path):
+    monkeypatch.setenv("HOME", str(tmp_path))
+    monkeypatch.setenv("USERPROFILE", str(tmp_path))
+    for key in (
+        "OS_AUTH_URL",
+        "OS_USER_DOMAIN_NAME",
+        "OS_USERNAME",
+        "OS_PASSWORD",
+        "OS_PROJECT_ID",
+        "OS_REGION_NAME",
+        "GPU_RENT_AZ",
+    ):
+        monkeypatch.setenv(key, "x")
+    monkeypatch.setenv("UP_STOP_ON_FAIL", "false")
+    from gpu_rent.config import load_config
+
+    cfg = load_config(require_auth=False)
+    assert cfg.up_stop_on_fail is False