Enhance Selectel balance handling and error messaging in GPU rental system

- Updated Selectel documentation to clarify error responses related to balance issues.
- Introduced `peek_balance_rub` function for best-effort balance checks, handling API failures gracefully.
- Improved error messages for insufficient funds and quota issues, specifying Selectel's 403 policy response.
- Added balance checks in the doctor command to ensure users are informed about their balance status before attempting GPU creation.
- Refactored exception handling in cloud operations to provide clearer feedback on balance-related errors.

Co-authored-by: Cursor <cursoragent@cursor.com>

docs/selectel.md

@@ -142,7 +142,7 @@ Ingress SG: TCP/22 с адреса оператора. 7801 снаружи не
 
 - квота compute/GPU;
 - наличие flavor в сегменте (`OS-FLV-DISABLED:disabled`, extra specs);
-- баланс / возможность создать ресурс (ошибка 402/403 — понятное сообщение, не traceback).
+- баланс / возможность создать ресурс: Selectel при нулевом балансе часто отвечает **403 policy** (`os_compute_api:servers:create`), не 402. CLI пишет «нет средств», не «квота GPU=0».
 
 Письмо в поддержку — ручной шаг один раз, не часть CLI.
 

src/gpu_rent/balance.py

@@ -18,6 +18,8 @@ Log = Callable[[str], None]
 
 BALANCES_URL = "https://api.selectel.ru/v3/balances"
 DEFAULT_STEP_RUB = 200.0
+# Selectel member role often cannot create compute below this (docs/setup.md).
+MEMBER_MIN_RUB = 100.0
 
 
 @dataclass
@@ -124,6 +126,17 @@ def fetch_balance_rub(api_token: str, *, timeout: float = 20.0) -> float:
     return balance_rub_from_payload(payload)
 
 
+def peek_balance_rub(api_token: str | None, *, timeout: float = 8.0) -> float | None:
+    """Best-effort balance; None if token missing or API fails."""
+    token = (api_token or "").strip()
+    if not token:
+        return None
+    try:
+        return fetch_balance_rub(token, timeout=timeout)
+    except Exception:
+        return None
+
+
 def spend_steps(baseline_rub: float, current_rub: float, step_rub: float) -> int:
     """How many full step intervals have been spent since baseline."""
     if step_rub <= 0:

src/gpu_rent/cli.py

@@ -52,21 +52,36 @@ def _die(exc: BaseException) -> None:
         "(Ctrl+C / Ctrl+D на туннеле гасят GPU, диски остаются)"
     )
     msg = str(exc)
-    if "gpu-rent status" not in msg and "Дальше:" not in msg:
+    skip = any(
+        s in msg
+        for s in (
+            "gpu-rent status",
+            "Дальше:",
+            "Пополни",
+            "нет средств",
+            "gpu-rent up",
+        )
+    )
+    if not skip:
         err(hint)
     raise typer.Exit(1)
 
 
 def _stop_after_failed_up(cfg, cause: BaseException) -> None:
     """Delete compute after a failed up so billing does not continue unnoticed."""
+    from gpu_rent.state import load_state
+
+    state = load_state()
+    if not getattr(state, "server_id", None) and not getattr(state, "floating_ip", None):
+        # Create never succeeded — nothing to bill, don't print "гашу GPU".
+        return
     warn(
         "up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)"
     )
     try:
-        from gpu_rent.state import load_state
         from gpu_rent.vm_logs import print_log_digest
 
-        fip = load_state().floating_ip
+        fip = getattr(state, "floating_ip", None)
         if fip:
             print_log_digest(cfg, fip, console=console, log=log)
     except Exception as dig_exc:

src/gpu_rent/cloud.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import re
 import time
 from typing import Any, Callable
 
@@ -27,16 +28,96 @@ from gpu_rent.os_client import (
 )
 
 
-def _wrap(exc: BaseException, what: str) -> CloudError:
+def _http_status(exc: BaseException) -> int | None:
+    for attr in ("http_status", "status_code"):
+        val = getattr(exc, attr, None)
+        if isinstance(val, int) and val >= 400:
+            return val
+    match = re.search(r"\b([45]\d{2})\b", str(exc)[:200])
+    return int(match.group(1)) if match else None
+
+
+def _exc_brief(exc: BaseException, *, limit: int = 160) -> str:
+    """One-line exception for logs — no URLs, no user_data / SSH keys."""
     text = str(exc)
-    low = text.lower()
+    text = re.split(r"\nValue:|\bValue:|\n#cloud-config|\nusers:", text, maxsplit=1)[0]
-    if "402" in text or "payment" in low:
+    text = re.sub(r"https?://\S+", "", text)
-        return CloudError(f"{what}: нет средств / 402. {text}")
+    text = re.sub(r"ssh-(?:ed25519|rsa|dss) \S+", "ssh-…", text, flags=re.I)
-    if "403" in text or "forbidden" in low or "quota" in low:
+    text = re.sub(r"\s+", " ", text).strip(" :.,")
-        return CloudError(
+    if len(text) > limit:
-            f"{what}: квота или запрет (403). Часто GPU=0 — тикет в поддержку, docs/setup.md. {text}"
+        text = text[: limit - 1] + "…"
+    return text or exc.__class__.__name__
+
+
+def _is_create_policy_deny(exc: BaseException) -> bool:
+    low = str(exc).lower()
+    return "os_compute_api:servers:create" in low or (
+        "policy" in low and "servers:create" in low
     )
-    return CloudError(f"{what}: {text}")
+
+
+def _funds_message(what: str, balance_rub: float | None) -> str:
+    from gpu_rent.balance import MEMBER_MIN_RUB
+
+    if balance_rub is not None and balance_rub <= 0:
+        return (
+            f"{what}: нет средств на балансе ({balance_rub:.0f} ₽). "
+            "Selectel отвечает 403 policy, не 402 и не квота GPU. "
+            "Пополни в панели Selectel, затем gpu-rent up"
+        )
+    if balance_rub is not None and balance_rub < MEMBER_MIN_RUB:
+        return (
+            f"{what}: Selectel запретил create (403 policy) при балансе "
+            f"{balance_rub:.0f} ₽ — для member обычно нужно ≥{MEMBER_MIN_RUB:.0f} ₽. "
+            "Пополни в панели, затем gpu-rent up"
+        )
+    if balance_rub is not None:
+        return (
+            f"{what}: запрет create (403 policy) при балансе {balance_rub:.0f} ₽ — "
+            "это IAM/роль сервисного пользователя, не квота GPU. docs/setup.md"
+        )
+    return (
+        f"{what}: Selectel запретил create (403 policy). "
+        "Обычно кончились деньги, не квота GPU. "
+        "Пополни баланс в панели; SELECTEL_API_TOKEN покажет сумму. "
+        "Затем gpu-rent up"
+    )
+
+
+def _wrap(
+    exc: BaseException,
+    what: str,
+    *,
+    api_token: str | None = None,
+) -> CloudError:
+    status = _http_status(exc)
+    low = str(exc).lower()
+    brief = _exc_brief(exc)
+
+    if status == 402 or "payment required" in low:
+        return CloudError(
+            f"{what}: нет средств. Пополни баланс в панели Selectel, затем gpu-rent up"
+        )
+
+    if "quota" in low or "overlimit" in low or "over limit" in low:
+        return CloudError(
+            f"{what}: квота исчерпана. Панель IAM → проект → квоты; "
+            f"GPU=0 — тикет, docs/setup.md. {brief}"
+        )
+
+    create_deny = _is_create_policy_deny(exc) or (
+        status == 403 and ("create" in what.lower() or what == "сервер")
+    )
+    if create_deny:
+        from gpu_rent.balance import peek_balance_rub
+
+        return CloudError(_funds_message(what, peek_balance_rub(api_token)))
+
+    if status == 403 or "forbidden" in low:
+        return CloudError(
+            f"{what}: запрет (403). Часто нет денег на балансе, не квота GPU. {brief}"
+        )
+    return CloudError(f"{what}: {brief}")
 
 
 def _oid(obj: Any) -> str:
@@ -385,6 +466,7 @@ def create_gpu_server(
     spot: bool,
     public_key: str,
     log: Callable[[str], None],
+    api_token: str | None = None,
 ) -> Any:
     bdm = [
         {
@@ -415,13 +497,12 @@ def create_gpu_server(
         "security_groups": [{"name": sg_name}],
     }
     ud_b64 = _user_data_b64(public_key)
-    ud_plain = _ssh_user_data(public_key)
 
     # Selectel: OpenStack API expects Base64 user_data; config_drive often unsupported.
+    # Do not send plaintext user_data — Nova 400 dumps the whole cloud-config (SSH keys).
     attempts: list[tuple[str, dict[str, Any]]] = [
         ("tags+user_data_b64", {**base, "tags": tags, "user_data": ud_b64}),
         ("user_data_b64", {**base, "user_data": ud_b64}),
-        ("user_data_plain", {**base, "user_data": ud_plain}),
         ("key_name_only", {**base, "tags": tags}),
         ("minimal", dict(base)),
     ]
@@ -435,9 +516,17 @@ def create_gpu_server(
             break
         except Exception as exc:
             last_exc = exc
-            log(f"create {label}: {exc}")
+            status = _http_status(exc)
+            # 401/402/403 / policy deny — payload variants cannot help; stop.
+            if status in {401, 402, 403} or _is_create_policy_deny(exc):
+                break
+            log(f"create {label}: {_exc_brief(exc)}")
     if server is None:
-        raise _wrap(last_exc or RuntimeError("create failed"), "create server")
+        raise _wrap(
+            last_exc or RuntimeError("create failed"),
+            "сервер",
+            api_token=api_token,
+        )
 
     if "user_data" not in used:
         log(

src/gpu_rent/doctor.py

@@ -40,6 +40,47 @@ class Check:
     detail: str
 
 
+def _check_balance(cfg: Config, checks: list[Check]) -> None:
+    from gpu_rent.balance import MEMBER_MIN_RUB, fetch_balance_rub
+
+    token = (cfg.selectel_api_token or "").strip()
+    if not token:
+        checks.append(
+            Check(
+                "баланс ₽",
+                True,
+                False,
+                "нет SELECTEL_API_TOKEN — ₽ не проверяем (403 create часто = нет денег)",
+            )
+        )
+        return
+    try:
+        rub = fetch_balance_rub(token)
+    except Exception as exc:
+        checks.append(Check("баланс ₽", True, False, f"не прочитать: {exc}"))
+        return
+    if rub <= 0:
+        checks.append(
+            Check(
+                "баланс ₽",
+                False,
+                True,
+                f"{rub:.0f} ₽ — GPU не создать. Пополни в панели Selectel.",
+            )
+        )
+    elif rub < MEMBER_MIN_RUB:
+        checks.append(
+            Check(
+                "баланс ₽",
+                True,
+                False,
+                f"{rub:.0f} ₽ — мало (member обычно ≥{MEMBER_MIN_RUB:.0f} ₽)",
+            )
+        )
+    else:
+        checks.append(Check("баланс ₽", True, False, f"{rub:.0f} ₽"))
+
+
 def run_doctor() -> list[Check]:
     checks: list[Check] = []
     runtime_dir().mkdir(parents=True, exist_ok=True)
@@ -123,6 +164,8 @@ def run_doctor() -> list[Check]:
     except CloudError as exc:
         checks.append(Check("квота GPU", False, True, str(exc)))
 
+    _check_balance(cfg, checks)
+
     flavors = list(iter_flavors(conn))
     gpu_flavors = [f for f in flavors if looks_like_gpu(f)]
     ranked = rank_flavors(

src/gpu_rent/session.py

@@ -613,6 +613,7 @@ def cmd_up(
             spot=spot,
             public_key=public_key,
             log=log,
+            api_token=cfg.selectel_api_token,
         )
         state.server_id = server.id
         state.server_name = getattr(server, "name", None)

tests/test_balance.py

@@ -1,9 +1,11 @@
 from gpu_rent.balance import (
     BalanceWatchState,
     balance_rub_from_payload,
+    peek_balance_rub,
     plan_balance_notices,
     spend_steps,
 )
+from gpu_rent.errors import CloudError
 
 
 def test_balance_kopecks():
@@ -54,3 +56,17 @@ def test_plan_low_once():
     assert any("низкий" in m for m in msgs)
     state, msgs2 = plan_balance_notices(state, 100.0, step_rub=200.0, low_rub=200.0)
     assert not any("низкий" in m for m in msgs2)
+
+
+def test_peek_balance_rub_empty_token():
+    assert peek_balance_rub("") is None
+    assert peek_balance_rub(None) is None
+
+
+def test_peek_balance_rub_swallows_errors(monkeypatch):
+    def boom(*_a, **_k):
+        raise CloudError("сеть")
+
+    monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", boom)
+    assert peek_balance_rub("tok") is None
+

tests/test_create_errors.py

@@ -0,0 +1,111 @@
+from types import SimpleNamespace
+
+import pytest
+
+from gpu_rent.cloud import (
+    _exc_brief,
+    _http_status,
+    _is_create_policy_deny,
+    _wrap,
+    create_gpu_server,
+)
+from gpu_rent.errors import CloudError
+
+POLICY_403 = (
+    "ForbiddenException: 403: Client Error for url: "
+    "https://ru-6.cloud.api.selcloud.ru/compute/v2.1/servers, "
+    "Policy doesn't allow os_compute_api:servers:create to be performed."
+)
+
+
+class Forbidden(Exception):
+    def __init__(self, msg: str, http_status: int = 403):
+        super().__init__(msg)
+        self.http_status = http_status
+
+
+def test_http_status_from_attr_and_text():
+    assert _http_status(Forbidden(POLICY_403)) == 403
+    assert _http_status(RuntimeError("no status")) is None
+
+
+def test_is_create_policy_deny():
+    assert _is_create_policy_deny(Forbidden(POLICY_403))
+    assert not _is_create_policy_deny(RuntimeError("quota exceeded for cores"))
+
+
+def test_exc_brief_strips_url_and_user_data():
+    dump = (
+        "BadRequestException: 400: Invalid input for field/attribute user_data. "
+        "Value: #cloud-config\nssh_authorized_keys:\n"
+        "  - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFake gpu-rent\n"
+    )
+    brief = _exc_brief(RuntimeError(dump))
+    assert "ssh-ed25519" not in brief
+    assert "AAAAC3" not in brief
+    assert "http" not in brief.lower() or "https://" not in brief
+
+
+def test_wrap_403_create_is_funds_not_gpu_quota():
+    err = _wrap(Forbidden(POLICY_403), "create server")
+    msg = str(err)
+    assert isinstance(err, CloudError)
+    assert "GPU=0" not in msg
+    assert "тикет" not in msg
+    assert "ssh-ed25519" not in msg
+    assert "https://" not in msg
+    assert "Пополни" in msg or "нет средств" in msg or "кончились деньги" in msg
+
+
+def test_wrap_403_create_with_zero_balance(monkeypatch):
+    monkeypatch.setattr("gpu_rent.balance.peek_balance_rub", lambda *_a, **_k: 0.0)
+    msg = str(_wrap(Forbidden(POLICY_403), "create server", api_token="tok"))
+    assert "0 ₽" in msg
+    assert "нет средств" in msg
+    assert "Пополни" in msg
+    assert "GPU=0" not in msg
+
+
+def test_wrap_403_create_with_healthy_balance_is_iam(monkeypatch):
+    monkeypatch.setattr("gpu_rent.balance.peek_balance_rub", lambda *_a, **_k: 1500.0)
+    msg = str(_wrap(Forbidden(POLICY_403), "create server", api_token="tok"))
+    assert "1500 ₽" in msg
+    assert "IAM" in msg
+    assert "GPU=0" not in msg
+
+
+def test_wrap_quota_message_only_when_quota_in_error():
+    exc = RuntimeError("403 OverLimit: quota exceeded for instances")
+    msg = str(_wrap(exc, "create server"))
+    assert "квота" in msg.lower()
+
+
+def test_create_gpu_server_stops_on_first_403():
+    calls: list[int] = []
+
+    class Compute:
+        def create_server(self, **kw):
+            calls.append(1)
+            raise Forbidden(POLICY_403)
+
+    logs: list[str] = []
+    with pytest.raises(CloudError) as caught:
+        create_gpu_server(
+            SimpleNamespace(compute=Compute()),
+            flavor_id="1",
+            net_id="n",
+            sg_name="g",
+            boot_volume_id="b",
+            data_volume_id="d",
+            az="ru-6a",
+            spot=True,
+            public_key="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFake gpu-rent",
+            log=logs.append,
+        )
+    assert len(calls) == 1
+    assert logs == []
+    msg = str(caught.value)
+    assert "GPU=0" not in msg
+    assert "тикет" not in msg
+    assert "ssh-ed25519" not in msg
+    assert "Пополни" in msg or "нет средств" in msg or "кончились деньги" in msg

tests/test_doctor_balance.py

@@ -0,0 +1,51 @@
+from types import SimpleNamespace
+
+from gpu_rent.doctor import Check, _check_balance
+from gpu_rent.errors import CloudError
+
+
+def test_check_balance_skips_without_token():
+    checks: list[Check] = []
+    _check_balance(SimpleNamespace(selectel_api_token=""), checks)
+    assert checks[0].name == "баланс ₽"
+    assert checks[0].ok is True
+    assert checks[0].blocking is False
+
+
+def test_check_balance_blocks_when_empty(monkeypatch):
+    monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", lambda *_a, **_k: 0.0)
+    checks: list[Check] = []
+    _check_balance(SimpleNamespace(selectel_api_token="tok"), checks)
+    assert checks[0].ok is False
+    assert checks[0].blocking is True
+    assert "0 ₽" in checks[0].detail
+    assert "Пополни" in checks[0].detail
+
+
+def test_check_balance_warns_when_low(monkeypatch):
+    monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", lambda *_a, **_k: 40.0)
+    checks: list[Check] = []
+    _check_balance(SimpleNamespace(selectel_api_token="tok"), checks)
+    assert checks[0].ok is True
+    assert checks[0].blocking is False
+    assert "40 ₽" in checks[0].detail
+
+
+def test_check_balance_ok_when_healthy(monkeypatch):
+    monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", lambda *_a, **_k: 1200.0)
+    checks: list[Check] = []
+    _check_balance(SimpleNamespace(selectel_api_token="tok"), checks)
+    assert checks[0].ok is True
+    assert "1200 ₽" in checks[0].detail
+
+
+def test_check_balance_fetch_error_is_warning(monkeypatch):
+    def boom(*_a, **_k):
+        raise CloudError("сеть")
+
+    monkeypatch.setattr("gpu_rent.balance.fetch_balance_rub", boom)
+    checks: list[Check] = []
+    _check_balance(SimpleNamespace(selectel_api_token="tok"), checks)
+    assert checks[0].ok is True
+    assert checks[0].blocking is False
+    assert "не прочитать" in checks[0].detail

tests/test_up_stop_on_fail.py

@@ -3,7 +3,14 @@ from types import SimpleNamespace
 from gpu_rent.cli import _stop_after_failed_up
 
 
-def test_stop_after_failed_up_calls_cmd_stop(monkeypatch):
+def _quiet(monkeypatch):
+    monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None)
+    monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None)
+
+
+def test_stop_after_failed_up_skips_when_no_server(monkeypatch):
     calls: list[dict] = []
 
     def fake_stop(cfg, *, no_pull=False, log=None, destroy_disks=False):
@@ -11,13 +18,28 @@ def test_stop_after_failed_up_calls_cmd_stop(monkeypatch):
         return SimpleNamespace()
 
     monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop)
-    monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None)
+    _quiet(monkeypatch)
-    monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None)
-    monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None)
-    monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None)
     monkeypatch.setattr(
         "gpu_rent.state.load_state",
-        lambda: SimpleNamespace(floating_ip=None),
+        lambda: SimpleNamespace(server_id=None, floating_ip=None),
+    )
+
+    _stop_after_failed_up(SimpleNamespace(), RuntimeError("boom"))
+    assert calls == []
+
+
+def test_stop_after_failed_up_runs_when_server_exists(monkeypatch):
+    calls: list[dict] = []
+
+    def fake_stop(cfg, *, no_pull=False, log=None, destroy_disks=False):
+        calls.append({"no_pull": no_pull, "destroy_disks": destroy_disks})
+        return SimpleNamespace()
+
+    monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop)
+    _quiet(monkeypatch)
+    monkeypatch.setattr(
+        "gpu_rent.state.load_state",
+        lambda: SimpleNamespace(server_id="s1", floating_ip=None),
     )
 
     _stop_after_failed_up(SimpleNamespace(), RuntimeError("boom"))
@@ -33,20 +55,20 @@ def test_stop_after_failed_up_prints_digest_when_fip(monkeypatch):
         return SimpleNamespace()
 
     monkeypatch.setattr("gpu_rent.cli.cmd_stop", fake_stop)
-    monkeypatch.setattr("gpu_rent.cli.warn", lambda *_a, **_k: None)
+    _quiet(monkeypatch)
-    monkeypatch.setattr("gpu_rent.cli.ok", lambda *_a, **_k: None)
-    monkeypatch.setattr("gpu_rent.cli.err", lambda *_a, **_k: None)
-    monkeypatch.setattr("gpu_rent.cli.log", lambda *_a, **_k: None)
     monkeypatch.setattr(
         "gpu_rent.state.load_state",
-        lambda: SimpleNamespace(floating_ip="10.0.0.1"),
+        lambda: SimpleNamespace(server_id="s1", floating_ip="10.0.0.1"),
     )
     monkeypatch.setattr(
         "gpu_rent.vm_logs.print_log_digest",
         lambda cfg, host, **kw: digests.append(host),
     )
 
-    _stop_after_failed_up(SimpleNamespace(enable_swarmui=True, llm_runtime="none"), RuntimeError("boom"))
+    _stop_after_failed_up(
+        SimpleNamespace(enable_swarmui=True, llm_runtime="none"),
+        RuntimeError("boom"),
+    )
     assert digests == ["10.0.0.1"]
     assert calls == [{"no_pull": True}]