From 0bad693b121b72641d29712e669d8c923ee82541 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Sun, 23 Aug 2026 17:54:11 +0300 Subject: [PATCH] Enhance Selectel balance handling and error messaging in GPU rental system - Updated Selectel documentation to clarify error responses related to balance issues. - Introduced `peek_balance_rub` function for best-effort balance checks, handling API failures gracefully. - Improved error messages for insufficient funds and quota issues, specifying Selectel's 403 policy response. - Added balance checks in the doctor command to ensure users are informed about their balance status before attempting GPU creation. - Refactored exception handling in cloud operations to provide clearer feedback on balance-related errors. Co-authored-by: Cursor --- docs/selectel.md | 2 +- src/gpu_rent/balance.py | 13 ++++ src/gpu_rent/cli.py | 21 ++++++- src/gpu_rent/cloud.py | 113 ++++++++++++++++++++++++++++++---- src/gpu_rent/doctor.py | 43 +++++++++++++ src/gpu_rent/session.py | 1 + tests/test_balance.py | 16 +++++ tests/test_create_errors.py | 111 +++++++++++++++++++++++++++++++++ tests/test_doctor_balance.py | 51 +++++++++++++++ tests/test_up_stop_on_fail.py | 46 ++++++++++---- 10 files changed, 389 insertions(+), 28 deletions(-) create mode 100644 tests/test_create_errors.py create mode 100644 tests/test_doctor_balance.py diff --git a/docs/selectel.md b/docs/selectel.md index 455eb39..1fdaeb8 100644 --- a/docs/selectel.md +++ b/docs/selectel.md @@ -142,7 +142,7 @@ Ingress SG: TCP/22 с адреса оператора. 7801 снаружи не - квота compute/GPU; - наличие flavor в сегменте (`OS-FLV-DISABLED:disabled`, extra specs); -- баланс / возможность создать ресурс (ошибка 402/403 — понятное сообщение, не traceback). +- баланс / возможность создать ресурс: Selectel при нулевом балансе часто отвечает **403 policy** (`os_compute_api:servers:create`), не 402. CLI пишет «нет средств», не «квота GPU=0». Письмо в поддержку — ручной шаг один раз, не часть CLI. diff --git a/src/gpu_rent/balance.py b/src/gpu_rent/balance.py index 05fee21..5a67cfa 100644 --- a/src/gpu_rent/balance.py +++ b/src/gpu_rent/balance.py @@ -18,6 +18,8 @@ Log = Callable[[str], None] BALANCES_URL = "https://api.selectel.ru/v3/balances" DEFAULT_STEP_RUB = 200.0 +# Selectel member role often cannot create compute below this (docs/setup.md). +MEMBER_MIN_RUB = 100.0 @dataclass @@ -124,6 +126,17 @@ def fetch_balance_rub(api_token: str, *, timeout: float = 20.0) -> float: return balance_rub_from_payload(payload) +def peek_balance_rub(api_token: str | None, *, timeout: float = 8.0) -> float | None: + """Best-effort balance; None if token missing or API fails.""" + token = (api_token or "").strip() + if not token: + return None + try: + return fetch_balance_rub(token, timeout=timeout) + except Exception: + return None + + def spend_steps(baseline_rub: float, current_rub: float, step_rub: float) -> int: """How many full step intervals have been spent since baseline.""" if step_rub <= 0: diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 7900f0b..bf3c6a8 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -52,21 +52,36 @@ def _die(exc: BaseException) -> None: "(Ctrl+C / Ctrl+D на туннеле гасят GPU, диски остаются)" ) msg = str(exc) - if "gpu-rent status" not in msg and "Дальше:" not in msg: + skip = any( + s in msg + for s in ( + "gpu-rent status", + "Дальше:", + "Пополни", + "нет средств", + "gpu-rent up", + ) + ) + if not skip: err(hint) raise typer.Exit(1) def _stop_after_failed_up(cfg, cause: BaseException) -> None: """Delete compute after a failed up so billing does not continue unnoticed.""" + from gpu_rent.state import load_state + + state = load_state() + if not getattr(state, "server_id", None) and not getattr(state, "floating_ip", None): + # Create never succeeded — nothing to bill, don't print "гашу GPU". + return warn( "up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)" ) try: - from gpu_rent.state import load_state from gpu_rent.vm_logs import print_log_digest - fip = load_state().floating_ip + fip = getattr(state, "floating_ip", None) if fip: print_log_digest(cfg, fip, console=console, log=log) except Exception as dig_exc: diff --git a/src/gpu_rent/cloud.py b/src/gpu_rent/cloud.py index 0cd0ae8..7f7d8f1 100644 --- a/src/gpu_rent/cloud.py +++ b/src/gpu_rent/cloud.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re import time from typing import Any, Callable @@ -27,16 +28,96 @@ from gpu_rent.os_client import ( ) -def _wrap(exc: BaseException, what: str) -> CloudError: +def _http_status(exc: BaseException) -> int | None: + for attr in ("http_status", "status_code"): + val = getattr(exc, attr, None) + if isinstance(val, int) and val >= 400: + return val + match = re.search(r"\b([45]\d{2})\b", str(exc)[:200]) + return int(match.group(1)) if match else None + + +def _exc_brief(exc: BaseException, *, limit: int = 160) -> str: + """One-line exception for logs — no URLs, no user_data / SSH keys.""" text = str(exc) - low = text.lower() - if "402" in text or "payment" in low: - return CloudError(f"{what}: нет средств / 402. {text}") - if "403" in text or "forbidden" in low or "quota" in low: - return CloudError( - f"{what}: квота или запрет (403). Часто GPU=0 — тикет в поддержку, docs/setup.md. {text}" + text = re.split(r"\nValue:|\bValue:|\n#cloud-config|\nusers:", text, maxsplit=1)[0] + text = re.sub(r"https?://\S+", "", text) + text = re.sub(r"ssh-(?:ed25519|rsa|dss) \S+", "ssh-…", text, flags=re.I) + text = re.sub(r"\s+", " ", text).strip(" :.,") + if len(text) > limit: + text = text[: limit - 1] + "…" + return text or exc.__class__.__name__ + + +def _is_create_policy_deny(exc: BaseException) -> bool: + low = str(exc).lower() + return "os_compute_api:servers:create" in low or ( + "policy" in low and "servers:create" in low + ) + + +def _funds_message(what: str, balance_rub: float | None) -> str: + from gpu_rent.balance import MEMBER_MIN_RUB + + if balance_rub is not None and balance_rub <= 0: + return ( + f"{what}: нет средств на балансе ({balance_rub:.0f} ₽). " + "Selectel отвечает 403 policy, не 402 и не квота GPU. " + "Пополни в панели Selectel, затем gpu-rent up" ) - return CloudError(f"{what}: {text}") + if balance_rub is not None and balance_rub < MEMBER_MIN_RUB: + return ( + f"{what}: Selectel запретил create (403 policy) при балансе " + f"{balance_rub:.0f} ₽ — для member обычно нужно ≥{MEMBER_MIN_RUB:.0f} ₽. " + "Пополни в панели, затем gpu-rent up" + ) + if balance_rub is not None: + return ( + f"{what}: запрет create (403 policy) при балансе {balance_rub:.0f} ₽ — " + "это IAM/роль сервисного пользователя, не квота GPU. docs/setup.md" + ) + return ( + f"{what}: Selectel запретил create (403 policy). " + "Обычно кончились деньги, не квота GPU. " + "Пополни баланс в панели; SELECTEL_API_TOKEN покажет сумму. " + "Затем gpu-rent up" + ) + + +def _wrap( + exc: BaseException, + what: str, + *, + api_token: str | None = None, +) -> CloudError: + status = _http_status(exc) + low = str(exc).lower() + brief = _exc_brief(exc) + + if status == 402 or "payment required" in low: + return CloudError( + f"{what}: нет средств. Пополни баланс в панели Selectel, затем gpu-rent up" + ) + + if "quota" in low or "overlimit" in low or "over limit" in low: + return CloudError( + f"{what}: квота исчерпана. Панель IAM → проект → квоты; " + f"GPU=0 — тикет, docs/setup.md. {brief}" + ) + + create_deny = _is_create_policy_deny(exc) or ( + status == 403 and ("create" in what.lower() or what == "сервер") + ) + if create_deny: + from gpu_rent.balance import peek_balance_rub + + return CloudError(_funds_message(what, peek_balance_rub(api_token))) + + if status == 403 or "forbidden" in low: + return CloudError( + f"{what}: запрет (403). Часто нет денег на балансе, не квота GPU. {brief}" + ) + return CloudError(f"{what}: {brief}") def _oid(obj: Any) -> str: @@ -385,6 +466,7 @@ def create_gpu_server( spot: bool, public_key: str, log: Callable[[str], None], + api_token: str | None = None, ) -> Any: bdm = [ { @@ -415,13 +497,12 @@ def create_gpu_server( "security_groups": [{"name": sg_name}], } ud_b64 = _user_data_b64(public_key) - ud_plain = _ssh_user_data(public_key) # Selectel: OpenStack API expects Base64 user_data; config_drive often unsupported. + # Do not send plaintext user_data — Nova 400 dumps the whole cloud-config (SSH keys). attempts: list[tuple[str, dict[str, Any]]] = [ ("tags+user_data_b64", {**base, "tags": tags, "user_data": ud_b64}), ("user_data_b64", {**base, "user_data": ud_b64}), - ("user_data_plain", {**base, "user_data": ud_plain}), ("key_name_only", {**base, "tags": tags}), ("minimal", dict(base)), ] @@ -435,9 +516,17 @@ def create_gpu_server( break except Exception as exc: last_exc = exc - log(f"create {label}: {exc}") + status = _http_status(exc) + # 401/402/403 / policy deny — payload variants cannot help; stop. + if status in {401, 402, 403} or _is_create_policy_deny(exc): + break + log(f"create {label}: {_exc_brief(exc)}") if server is None: - raise _wrap(last_exc or RuntimeError("create failed"), "create server") + raise _wrap( + last_exc or RuntimeError("create failed"), + "сервер", + api_token=api_token, + ) if "user_data" not in used: log( diff --git a/src/gpu_rent/doctor.py b/src/gpu_rent/doctor.py index d778545..8698273 100644 --- a/src/gpu_rent/doctor.py +++ b/src/gpu_rent/doctor.py @@ -40,6 +40,47 @@ class Check: detail: str +def _check_balance(cfg: Config, checks: list[Check]) -> None: + from gpu_rent.balance import MEMBER_MIN_RUB, fetch_balance_rub + + token = (cfg.selectel_api_token or "").strip() + if not token: + checks.append( + Check( + "баланс ₽", + True, + False, + "нет SELECTEL_API_TOKEN — ₽ не проверяем (403 create часто = нет денег)", + ) + ) + return + try: + rub = fetch_balance_rub(token) + except Exception as exc: + checks.append(Check("баланс ₽", True, False, f"не прочитать: {exc}")) + return + if rub <= 0: + checks.append( + Check( + "баланс ₽", + False, + True, + f"{rub:.0f} ₽ — GPU не создать. Пополни в панели Selectel.", + ) + ) + elif rub < MEMBER_MIN_RUB: + checks.append( + Check( + "баланс ₽", + True, + False, + f"{rub:.0f} ₽ — мало (member обычно ≥{MEMBER_MIN_RUB:.0f} ₽)", + ) + ) + else: + checks.append(Check("баланс ₽", True, False, f"{rub:.0f} ₽")) + + def run_doctor() -> list[Check]: checks: list[Check] = [] runtime_dir().mkdir(parents=True, exist_ok=True) @@ -123,6 +164,8 @@ def run_doctor() -> list[Check]: except CloudError as exc: checks.append(Check("квота GPU", False, True, str(exc))) + _check_balance(cfg, checks) + flavors = list(iter_flavors(conn)) gpu_flavors = [f for f in flavors if looks_like_gpu(f)] ranked = rank_flavors( diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py index 2bcca70..cde3ed0 100644 --- a/src/gpu_rent/session.py +++ b/src/gpu_rent/session.py @@ -613,6 +613,7 @@ def cmd_up( spot=spot, public_key=public_key, log=log, + api_token=cfg.selectel_api_token, ) state.server_id = server.id state.server_name = getattr(server, "name", None) diff --git a/tests/test_balance.py b/tests/test_balance.py index 42a21e6..47515b9 100644 --- a/tests/test_balance.py +++ b/tests/test_balance.py @@ -1,9 +1,11 @@ from gpu_rent.balance import ( BalanceWatchState, balance_rub_from_payload, + peek_balance_rub, plan_balance_notices, spend_steps, ) +from gpu_rent.errors import CloudError def test_balance_kopecks(): @@ -54,3 +56,17 @@ def test_plan_low_once(): assert any("низкий" in m for m in msgs) state, msgs2 = plan_balance_notices(state, 100.0, step_rub=200.0, low_rub=200.0) assert not any("низкий" in m for m in msgs2) + + +def test_peek_balance_rub_empty_token(): + assert peek_balance_rub("") is None + assert peek_balance_rub(None) is None + + +def test_peek_balance_rub_swallows_errors(monkeypatch): + def boom(*_a, **_k): + raise CloudError("сеть") + + monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", boom) + assert peek_balance_rub("tok") is None + diff --git a/tests/test_create_errors.py b/tests/test_create_errors.py new file mode 100644 index 0000000..2b6e883 --- /dev/null +++ b/tests/test_create_errors.py @@ -0,0 +1,111 @@ +from types import SimpleNamespace + +import pytest + +from gpu_rent.cloud import ( + _exc_brief, + _http_status, + _is_create_policy_deny, + _wrap, + create_gpu_server, +) +from gpu_rent.errors import CloudError + +POLICY_403 = ( + "ForbiddenException: 403: Client Error for url: " + "https://ru-6.cloud.api.selcloud.ru/compute/v2.1/servers, " + "Policy doesn't allow os_compute_api:servers:create to be performed." +) + + +class Forbidden(Exception): + def __init__(self, msg: str, http_status: int = 403): + super().__init__(msg) + self.http_status = http_status + + +def test_http_status_from_attr_and_text(): + assert _http_status(Forbidden(POLICY_403)) == 403 + assert _http_status(RuntimeError("no status")) is None + + +def test_is_create_policy_deny(): + assert _is_create_policy_deny(Forbidden(POLICY_403)) + assert not _is_create_policy_deny(RuntimeError("quota exceeded for cores")) + + +def test_exc_brief_strips_url_and_user_data(): + dump = ( + "BadRequestException: 400: Invalid input for field/attribute user_data. " + "Value: #cloud-config\nssh_authorized_keys:\n" + " - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFake gpu-rent\n" + ) + brief = _exc_brief(RuntimeError(dump)) + assert "ssh-ed25519" not in brief + assert "AAAAC3" not in brief + assert "http" not in brief.lower() or "https://" not in brief + + +def test_wrap_403_create_is_funds_not_gpu_quota(): + err = _wrap(Forbidden(POLICY_403), "create server") + msg = str(err) + assert isinstance(err, CloudError) + assert "GPU=0" not in msg + assert "тикет" not in msg + assert "ssh-ed25519" not in msg + assert "https://" not in msg + assert "Пополни" in msg or "нет средств" in msg or "кончились деньги" in msg + + +def test_wrap_403_create_with_zero_balance(monkeypatch): + monkeypatch.setattr("gpu_rent.balance.peek_balance_rub", lambda *_a, **_k: 0.0) + msg = str(_wrap(Forbidden(POLICY_403), "create server", api_token="tok")) + assert "0 ₽" in msg + assert "нет средств" in msg + assert "Пополни" in msg + assert "GPU=0" not in msg + + +def test_wrap_403_create_with_healthy_balance_is_iam(monkeypatch): + monkeypatch.setattr("gpu_rent.balance.peek_balance_rub", lambda *_a, **_k: 1500.0) + msg = str(_wrap(Forbidden(POLICY_403), "create server", api_token="tok")) + assert "1500 ₽" in msg + assert "IAM" in msg + assert "GPU=0" not in msg + + +def test_wrap_quota_message_only_when_quota_in_error(): + exc = RuntimeError("403 OverLimit: quota exceeded for instances") + msg = str(_wrap(exc, "create server")) + assert "квота" in msg.lower() + + +def test_create_gpu_server_stops_on_first_403(): + calls: list[int] = [] + + class Compute: + def create_server(self, **kw): + calls.append(1) + raise Forbidden(POLICY_403) + + logs: list[str] = [] + with pytest.raises(CloudError) as caught: + create_gpu_server( + SimpleNamespace(compute=Compute()), + flavor_id="1", + net_id="n", + sg_name="g", + boot_volume_id="b", + data_volume_id="d", + az="ru-6a", + spot=True, + public_key="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFake gpu-rent", + log=logs.append, + ) + assert len(calls) == 1 + assert logs == [] + msg = str(caught.value) + assert "GPU=0" not in msg + assert "тикет" not in msg + assert "ssh-ed25519" not in msg + assert "Пополни" in msg or "нет средств" in msg or "кончились деньги" in msg diff --git a/tests/test_doctor_balance.py b/tests/test_doctor_balance.py new file mode 100644 index 0000000..d8f17bc --- /dev/null +++ b/tests/test_doctor_balance.py @@ -0,0 +1,51 @@ +from types import SimpleNamespace + +from gpu_rent.doctor import Check, _check_balance +from gpu_rent.errors import CloudError + + +def test_check_balance_skips_without_token(): + checks: list[Check] = [] + _check_balance(SimpleNamespace(selectel_api_token=""), checks) + assert checks[0].name == "баланс ₽" + assert checks[0].ok is True + assert checks[0].blocking is False + + +def test_check_balance_blocks_when_empty(monkeypatch): + monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", lambda *_a, **_k: 0.0) + checks: list[Check] = [] + _check_balance(SimpleNamespace(selectel_api_token="tok"), checks) + assert checks[0].ok is False + assert checks[0].blocking is True + assert "0 ₽" in checks[0].detail + assert "Пополни" in checks[0].detail + + +def test_check_balance_warns_when_low(monkeypatch): + monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", lambda *_a, **_k: 40.0) + checks: list[Check] = [] + _check_balance(SimpleNamespace(selectel_api_token="tok"), checks) + assert checks[0].ok is True + assert checks[0].blocking is False + assert "40 ₽" in checks[0].detail + + +def test_check_balance_ok_when_healthy(monkeypatch): + monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", lambda *_a, **_k: 1200.0) + checks: list[Check] = [] + _check_balance(SimpleNamespace(selectel_api_token="tok"), checks) + assert checks[0].ok is True + assert "1200 ₽" in checks[0].detail + + +def test_check_balance_fetch_error_is_warning(monkeypatch): + def boom(*_a, **_k): + raise CloudError("сеть") + + monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", boom) + checks: list[Check] = [] + _check_balance(SimpleNamespace(selectel_api_token="tok"), checks) + assert checks[0].ok is True + assert checks[0].blocking is False + assert "не прочитать" in checks[0].detail diff --git a/tests/test_up_stop_on_fail.py b/tests/test_up_stop_on_fail.py index fc21ce4..f115b99 100644 --- a/tests/test_up_stop_on_fail.py +++ b/tests/test_up_stop_on_fail.py @@ -3,7 +3,14 @@ from types import SimpleNamespace from gpu_rent.cli import _stop_after_failed_up -def test_stop_after_failed_up_calls_cmd_stop(monkeypatch): +def _quiet(monkeypatch): + monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None) + monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None) + monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None) + monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None) + + +def test_stop_after_failed_up_skips_when_no_server(monkeypatch): calls: list[dict] = [] def fake_stop(cfg, *, no_pull=False, log=None, destroy_disks=False): @@ -11,13 +18,28 @@ def test_stop_after_failed_up_calls_cmd_stop(monkeypatch): return SimpleNamespace() monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop) - monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None) - monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None) - monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None) - monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None) + _quiet(monkeypatch) monkeypatch.setattr( "gpu_rent.state.load_state", - lambda: SimpleNamespace(floating_ip=None), + lambda: SimpleNamespace(server_id=None, floating_ip=None), + ) + + _stop_after_failed_up(SimpleNamespace(), RuntimeError("boom")) + assert calls == [] + + +def test_stop_after_failed_up_runs_when_server_exists(monkeypatch): + calls: list[dict] = [] + + def fake_stop(cfg, *, no_pull=False, log=None, destroy_disks=False): + calls.append({"no_pull": no_pull, "destroy_disks": destroy_disks}) + return SimpleNamespace() + + monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop) + _quiet(monkeypatch) + monkeypatch.setattr( + "gpu_rent.state.load_state", + lambda: SimpleNamespace(server_id="s1", floating_ip=None), ) _stop_after_failed_up(SimpleNamespace(), RuntimeError("boom")) @@ -33,20 +55,20 @@ def test_stop_after_failed_up_prints_digest_when_fip(monkeypatch): return SimpleNamespace() monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop) - monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None) - monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None) - monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None) - monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None) + _quiet(monkeypatch) monkeypatch.setattr( "gpu_rent.state.load_state", - lambda: SimpleNamespace(floating_ip="10.0.0.1"), + lambda: SimpleNamespace(server_id="s1", floating_ip="10.0.0.1"), ) monkeypatch.setattr( "gpu_rent.vm_logs.print_log_digest", lambda cfg, host, **kw: digests.append(host), ) - _stop_after_failed_up(SimpleNamespace(enable_swarmui=True, llm_runtime="none"), RuntimeError("boom")) + _stop_after_failed_up( + SimpleNamespace(enable_swarmui=True, llm_runtime="none"), + RuntimeError("boom"), + ) assert digests == ["10.0.0.1"] assert calls == [{"no_pull": True}]