Enhance Selectel balance handling and error messaging in GPU rental system

- Updated Selectel documentation to clarify error responses related to balance issues.
- Introduced `peek_balance_rub` function for best-effort balance checks, handling API failures gracefully.
- Improved error messages for insufficient funds and quota issues, specifying Selectel's 403 policy response.
- Added balance checks in the doctor command to ensure users are informed about their balance status before attempting GPU creation.
- Refactored exception handling in cloud operations to provide clearer feedback on balance-related errors.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 17:54:11 +03:00
co-authored by Cursor
parent 322577cf9f
commit 0bad693b12
10 changed files with 389 additions and 28 deletions
+13
View File
@@ -18,6 +18,8 @@ Log = Callable[[str], None]
BALANCES_URL = "https://api.selectel.ru/v3/balances"
DEFAULT_STEP_RUB = 200.0
# Selectel member role often cannot create compute below this (docs/setup.md).
MEMBER_MIN_RUB = 100.0
@dataclass
@@ -124,6 +126,17 @@ def fetch_balance_rub(api_token: str, *, timeout: float = 20.0) -> float:
return balance_rub_from_payload(payload)
def peek_balance_rub(api_token: str | None, *, timeout: float = 8.0) -> float | None:
"""Best-effort balance; None if token missing or API fails."""
token = (api_token or "").strip()
if not token:
return None
try:
return fetch_balance_rub(token, timeout=timeout)
except Exception:
return None
def spend_steps(baseline_rub: float, current_rub: float, step_rub: float) -> int:
"""How many full step intervals have been spent since baseline."""
if step_rub <= 0:
+18 -3
View File
@@ -52,21 +52,36 @@ def _die(exc: BaseException) -> None:
"(Ctrl+C / Ctrl+D на туннеле гасят GPU, диски остаются)"
)
msg = str(exc)
if "gpu-rent status" not in msg and "Дальше:" not in msg:
skip = any(
s in msg
for s in (
"gpu-rent status",
"Дальше:",
"Пополни",
"нет средств",
"gpu-rent up",
)
)
if not skip:
err(hint)
raise typer.Exit(1)
def _stop_after_failed_up(cfg, cause: BaseException) -> None:
"""Delete compute after a failed up so billing does not continue unnoticed."""
from gpu_rent.state import load_state
state = load_state()
if not getattr(state, "server_id", None) and not getattr(state, "floating_ip", None):
# Create never succeeded — nothing to bill, don't print "гашу GPU".
return
warn(
"up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)"
)
try:
from gpu_rent.state import load_state
from gpu_rent.vm_logs import print_log_digest
fip = load_state().floating_ip
fip = getattr(state, "floating_ip", None)
if fip:
print_log_digest(cfg, fip, console=console, log=log)
except Exception as dig_exc:
+101 -12
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import re
import time
from typing import Any, Callable
@@ -27,16 +28,96 @@ from gpu_rent.os_client import (
)
def _wrap(exc: BaseException, what: str) -> CloudError:
def _http_status(exc: BaseException) -> int | None:
for attr in ("http_status", "status_code"):
val = getattr(exc, attr, None)
if isinstance(val, int) and val >= 400:
return val
match = re.search(r"\b([45]\d{2})\b", str(exc)[:200])
return int(match.group(1)) if match else None
def _exc_brief(exc: BaseException, *, limit: int = 160) -> str:
"""One-line exception for logs — no URLs, no user_data / SSH keys."""
text = str(exc)
low = text.lower()
if "402" in text or "payment" in low:
return CloudError(f"{what}: нет средств / 402. {text}")
if "403" in text or "forbidden" in low or "quota" in low:
return CloudError(
f"{what}: квота или запрет (403). Часто GPU=0 — тикет в поддержку, docs/setup.md. {text}"
text = re.split(r"\nValue:|\bValue:|\n#cloud-config|\nusers:", text, maxsplit=1)[0]
text = re.sub(r"https?://\S+", "", text)
text = re.sub(r"ssh-(?:ed25519|rsa|dss) \S+", "ssh-…", text, flags=re.I)
text = re.sub(r"\s+", " ", text).strip(" :.,")
if len(text) > limit:
text = text[: limit - 1] + ""
return text or exc.__class__.__name__
def _is_create_policy_deny(exc: BaseException) -> bool:
low = str(exc).lower()
return "os_compute_api:servers:create" in low or (
"policy" in low and "servers:create" in low
)
def _funds_message(what: str, balance_rub: float | None) -> str:
from gpu_rent.balance import MEMBER_MIN_RUB
if balance_rub is not None and balance_rub <= 0:
return (
f"{what}: нет средств на балансе ({balance_rub:.0f} ₽). "
"Selectel отвечает 403 policy, не 402 и не квота GPU. "
"Пополни в панели Selectel, затем gpu-rent up"
)
return CloudError(f"{what}: {text}")
if balance_rub is not None and balance_rub < MEMBER_MIN_RUB:
return (
f"{what}: Selectel запретил create (403 policy) при балансе "
f"{balance_rub:.0f} ₽ — для member обычно нужно ≥{MEMBER_MIN_RUB:.0f} ₽. "
"Пополни в панели, затем gpu-rent up"
)
if balance_rub is not None:
return (
f"{what}: запрет create (403 policy) при балансе {balance_rub:.0f} ₽ — "
"это IAM/роль сервисного пользователя, не квота GPU. docs/setup.md"
)
return (
f"{what}: Selectel запретил create (403 policy). "
"Обычно кончились деньги, не квота GPU. "
"Пополни баланс в панели; SELECTEL_API_TOKEN покажет сумму. "
"Затем gpu-rent up"
)
def _wrap(
exc: BaseException,
what: str,
*,
api_token: str | None = None,
) -> CloudError:
status = _http_status(exc)
low = str(exc).lower()
brief = _exc_brief(exc)
if status == 402 or "payment required" in low:
return CloudError(
f"{what}: нет средств. Пополни баланс в панели Selectel, затем gpu-rent up"
)
if "quota" in low or "overlimit" in low or "over limit" in low:
return CloudError(
f"{what}: квота исчерпана. Панель IAM → проект → квоты; "
f"GPU=0 — тикет, docs/setup.md. {brief}"
)
create_deny = _is_create_policy_deny(exc) or (
status == 403 and ("create" in what.lower() or what == "сервер")
)
if create_deny:
from gpu_rent.balance import peek_balance_rub
return CloudError(_funds_message(what, peek_balance_rub(api_token)))
if status == 403 or "forbidden" in low:
return CloudError(
f"{what}: запрет (403). Часто нет денег на балансе, не квота GPU. {brief}"
)
return CloudError(f"{what}: {brief}")
def _oid(obj: Any) -> str:
@@ -385,6 +466,7 @@ def create_gpu_server(
spot: bool,
public_key: str,
log: Callable[[str], None],
api_token: str | None = None,
) -> Any:
bdm = [
{
@@ -415,13 +497,12 @@ def create_gpu_server(
"security_groups": [{"name": sg_name}],
}
ud_b64 = _user_data_b64(public_key)
ud_plain = _ssh_user_data(public_key)
# Selectel: OpenStack API expects Base64 user_data; config_drive often unsupported.
# Do not send plaintext user_data — Nova 400 dumps the whole cloud-config (SSH keys).
attempts: list[tuple[str, dict[str, Any]]] = [
("tags+user_data_b64", {**base, "tags": tags, "user_data": ud_b64}),
("user_data_b64", {**base, "user_data": ud_b64}),
("user_data_plain", {**base, "user_data": ud_plain}),
("key_name_only", {**base, "tags": tags}),
("minimal", dict(base)),
]
@@ -435,9 +516,17 @@ def create_gpu_server(
break
except Exception as exc:
last_exc = exc
log(f"create {label}: {exc}")
status = _http_status(exc)
# 401/402/403 / policy deny — payload variants cannot help; stop.
if status in {401, 402, 403} or _is_create_policy_deny(exc):
break
log(f"create {label}: {_exc_brief(exc)}")
if server is None:
raise _wrap(last_exc or RuntimeError("create failed"), "create server")
raise _wrap(
last_exc or RuntimeError("create failed"),
"сервер",
api_token=api_token,
)
if "user_data" not in used:
log(
+43
View File
@@ -40,6 +40,47 @@ class Check:
detail: str
def _check_balance(cfg: Config, checks: list[Check]) -> None:
from gpu_rent.balance import MEMBER_MIN_RUB, fetch_balance_rub
token = (cfg.selectel_api_token or "").strip()
if not token:
checks.append(
Check(
"баланс ₽",
True,
False,
"нет SELECTEL_API_TOKEN — ₽ не проверяем (403 create часто = нет денег)",
)
)
return
try:
rub = fetch_balance_rub(token)
except Exception as exc:
checks.append(Check("баланс ₽", True, False, f"не прочитать: {exc}"))
return
if rub <= 0:
checks.append(
Check(
"баланс ₽",
False,
True,
f"{rub:.0f} ₽ — GPU не создать. Пополни в панели Selectel.",
)
)
elif rub < MEMBER_MIN_RUB:
checks.append(
Check(
"баланс ₽",
True,
False,
f"{rub:.0f} ₽ — мало (member обычно ≥{MEMBER_MIN_RUB:.0f} ₽)",
)
)
else:
checks.append(Check("баланс ₽", True, False, f"{rub:.0f}"))
def run_doctor() -> list[Check]:
checks: list[Check] = []
runtime_dir().mkdir(parents=True, exist_ok=True)
@@ -123,6 +164,8 @@ def run_doctor() -> list[Check]:
except CloudError as exc:
checks.append(Check("квота GPU", False, True, str(exc)))
_check_balance(cfg, checks)
flavors = list(iter_flavors(conn))
gpu_flavors = [f for f in flavors if looks_like_gpu(f)]
ranked = rank_flavors(
+1
View File
@@ -613,6 +613,7 @@ def cmd_up(
spot=spot,
public_key=public_key,
log=log,
api_token=cfg.selectel_api_token,
)
state.server_id = server.id
state.server_name = getattr(server, "name", None)