Enhance Selectel balance handling and error messaging in GPU rental system
- Updated Selectel documentation to clarify error responses related to balance issues. - Introduced `peek_balance_rub` function for best-effort balance checks, handling API failures gracefully. - Improved error messages for insufficient funds and quota issues, specifying Selectel's 403 policy response. - Added balance checks in the doctor command to ensure users are informed about their balance status before attempting GPU creation. - Refactored exception handling in cloud operations to provide clearer feedback on balance-related errors. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -18,6 +18,8 @@ Log = Callable[[str], None]
|
||||
|
||||
BALANCES_URL = "https://api.selectel.ru/v3/balances"
|
||||
DEFAULT_STEP_RUB = 200.0
|
||||
# Selectel member role often cannot create compute below this (docs/setup.md).
|
||||
MEMBER_MIN_RUB = 100.0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -124,6 +126,17 @@ def fetch_balance_rub(api_token: str, *, timeout: float = 20.0) -> float:
|
||||
return balance_rub_from_payload(payload)
|
||||
|
||||
|
||||
def peek_balance_rub(api_token: str | None, *, timeout: float = 8.0) -> float | None:
|
||||
"""Best-effort balance; None if token missing or API fails."""
|
||||
token = (api_token or "").strip()
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
return fetch_balance_rub(token, timeout=timeout)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def spend_steps(baseline_rub: float, current_rub: float, step_rub: float) -> int:
|
||||
"""How many full step intervals have been spent since baseline."""
|
||||
if step_rub <= 0:
|
||||
|
||||
+18
-3
@@ -52,21 +52,36 @@ def _die(exc: BaseException) -> None:
|
||||
"(Ctrl+C / Ctrl+D на туннеле гасят GPU, диски остаются)"
|
||||
)
|
||||
msg = str(exc)
|
||||
if "gpu-rent status" not in msg and "Дальше:" not in msg:
|
||||
skip = any(
|
||||
s in msg
|
||||
for s in (
|
||||
"gpu-rent status",
|
||||
"Дальше:",
|
||||
"Пополни",
|
||||
"нет средств",
|
||||
"gpu-rent up",
|
||||
)
|
||||
)
|
||||
if not skip:
|
||||
err(hint)
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
def _stop_after_failed_up(cfg, cause: BaseException) -> None:
|
||||
"""Delete compute after a failed up so billing does not continue unnoticed."""
|
||||
from gpu_rent.state import load_state
|
||||
|
||||
state = load_state()
|
||||
if not getattr(state, "server_id", None) and not getattr(state, "floating_ip", None):
|
||||
# Create never succeeded — nothing to bill, don't print "гашу GPU".
|
||||
return
|
||||
warn(
|
||||
"up упал — гашу GPU (UP_STOP_ON_FAIL; оставить: --keep-on-fail / UP_STOP_ON_FAIL=false)"
|
||||
)
|
||||
try:
|
||||
from gpu_rent.state import load_state
|
||||
from gpu_rent.vm_logs import print_log_digest
|
||||
|
||||
fip = load_state().floating_ip
|
||||
fip = getattr(state, "floating_ip", None)
|
||||
if fip:
|
||||
print_log_digest(cfg, fip, console=console, log=log)
|
||||
except Exception as dig_exc:
|
||||
|
||||
+101
-12
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
@@ -27,16 +28,96 @@ from gpu_rent.os_client import (
|
||||
)
|
||||
|
||||
|
||||
def _wrap(exc: BaseException, what: str) -> CloudError:
|
||||
def _http_status(exc: BaseException) -> int | None:
|
||||
for attr in ("http_status", "status_code"):
|
||||
val = getattr(exc, attr, None)
|
||||
if isinstance(val, int) and val >= 400:
|
||||
return val
|
||||
match = re.search(r"\b([45]\d{2})\b", str(exc)[:200])
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def _exc_brief(exc: BaseException, *, limit: int = 160) -> str:
|
||||
"""One-line exception for logs — no URLs, no user_data / SSH keys."""
|
||||
text = str(exc)
|
||||
low = text.lower()
|
||||
if "402" in text or "payment" in low:
|
||||
return CloudError(f"{what}: нет средств / 402. {text}")
|
||||
if "403" in text or "forbidden" in low or "quota" in low:
|
||||
return CloudError(
|
||||
f"{what}: квота или запрет (403). Часто GPU=0 — тикет в поддержку, docs/setup.md. {text}"
|
||||
text = re.split(r"\nValue:|\bValue:|\n#cloud-config|\nusers:", text, maxsplit=1)[0]
|
||||
text = re.sub(r"https?://\S+", "", text)
|
||||
text = re.sub(r"ssh-(?:ed25519|rsa|dss) \S+", "ssh-…", text, flags=re.I)
|
||||
text = re.sub(r"\s+", " ", text).strip(" :.,")
|
||||
if len(text) > limit:
|
||||
text = text[: limit - 1] + "…"
|
||||
return text or exc.__class__.__name__
|
||||
|
||||
|
||||
def _is_create_policy_deny(exc: BaseException) -> bool:
|
||||
low = str(exc).lower()
|
||||
return "os_compute_api:servers:create" in low or (
|
||||
"policy" in low and "servers:create" in low
|
||||
)
|
||||
|
||||
|
||||
def _funds_message(what: str, balance_rub: float | None) -> str:
|
||||
from gpu_rent.balance import MEMBER_MIN_RUB
|
||||
|
||||
if balance_rub is not None and balance_rub <= 0:
|
||||
return (
|
||||
f"{what}: нет средств на балансе ({balance_rub:.0f} ₽). "
|
||||
"Selectel отвечает 403 policy, не 402 и не квота GPU. "
|
||||
"Пополни в панели Selectel, затем gpu-rent up"
|
||||
)
|
||||
return CloudError(f"{what}: {text}")
|
||||
if balance_rub is not None and balance_rub < MEMBER_MIN_RUB:
|
||||
return (
|
||||
f"{what}: Selectel запретил create (403 policy) при балансе "
|
||||
f"{balance_rub:.0f} ₽ — для member обычно нужно ≥{MEMBER_MIN_RUB:.0f} ₽. "
|
||||
"Пополни в панели, затем gpu-rent up"
|
||||
)
|
||||
if balance_rub is not None:
|
||||
return (
|
||||
f"{what}: запрет create (403 policy) при балансе {balance_rub:.0f} ₽ — "
|
||||
"это IAM/роль сервисного пользователя, не квота GPU. docs/setup.md"
|
||||
)
|
||||
return (
|
||||
f"{what}: Selectel запретил create (403 policy). "
|
||||
"Обычно кончились деньги, не квота GPU. "
|
||||
"Пополни баланс в панели; SELECTEL_API_TOKEN покажет сумму. "
|
||||
"Затем gpu-rent up"
|
||||
)
|
||||
|
||||
|
||||
def _wrap(
|
||||
exc: BaseException,
|
||||
what: str,
|
||||
*,
|
||||
api_token: str | None = None,
|
||||
) -> CloudError:
|
||||
status = _http_status(exc)
|
||||
low = str(exc).lower()
|
||||
brief = _exc_brief(exc)
|
||||
|
||||
if status == 402 or "payment required" in low:
|
||||
return CloudError(
|
||||
f"{what}: нет средств. Пополни баланс в панели Selectel, затем gpu-rent up"
|
||||
)
|
||||
|
||||
if "quota" in low or "overlimit" in low or "over limit" in low:
|
||||
return CloudError(
|
||||
f"{what}: квота исчерпана. Панель IAM → проект → квоты; "
|
||||
f"GPU=0 — тикет, docs/setup.md. {brief}"
|
||||
)
|
||||
|
||||
create_deny = _is_create_policy_deny(exc) or (
|
||||
status == 403 and ("create" in what.lower() or what == "сервер")
|
||||
)
|
||||
if create_deny:
|
||||
from gpu_rent.balance import peek_balance_rub
|
||||
|
||||
return CloudError(_funds_message(what, peek_balance_rub(api_token)))
|
||||
|
||||
if status == 403 or "forbidden" in low:
|
||||
return CloudError(
|
||||
f"{what}: запрет (403). Часто нет денег на балансе, не квота GPU. {brief}"
|
||||
)
|
||||
return CloudError(f"{what}: {brief}")
|
||||
|
||||
|
||||
def _oid(obj: Any) -> str:
|
||||
@@ -385,6 +466,7 @@ def create_gpu_server(
|
||||
spot: bool,
|
||||
public_key: str,
|
||||
log: Callable[[str], None],
|
||||
api_token: str | None = None,
|
||||
) -> Any:
|
||||
bdm = [
|
||||
{
|
||||
@@ -415,13 +497,12 @@ def create_gpu_server(
|
||||
"security_groups": [{"name": sg_name}],
|
||||
}
|
||||
ud_b64 = _user_data_b64(public_key)
|
||||
ud_plain = _ssh_user_data(public_key)
|
||||
|
||||
# Selectel: OpenStack API expects Base64 user_data; config_drive often unsupported.
|
||||
# Do not send plaintext user_data — Nova 400 dumps the whole cloud-config (SSH keys).
|
||||
attempts: list[tuple[str, dict[str, Any]]] = [
|
||||
("tags+user_data_b64", {**base, "tags": tags, "user_data": ud_b64}),
|
||||
("user_data_b64", {**base, "user_data": ud_b64}),
|
||||
("user_data_plain", {**base, "user_data": ud_plain}),
|
||||
("key_name_only", {**base, "tags": tags}),
|
||||
("minimal", dict(base)),
|
||||
]
|
||||
@@ -435,9 +516,17 @@ def create_gpu_server(
|
||||
break
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
log(f"create {label}: {exc}")
|
||||
status = _http_status(exc)
|
||||
# 401/402/403 / policy deny — payload variants cannot help; stop.
|
||||
if status in {401, 402, 403} or _is_create_policy_deny(exc):
|
||||
break
|
||||
log(f"create {label}: {_exc_brief(exc)}")
|
||||
if server is None:
|
||||
raise _wrap(last_exc or RuntimeError("create failed"), "create server")
|
||||
raise _wrap(
|
||||
last_exc or RuntimeError("create failed"),
|
||||
"сервер",
|
||||
api_token=api_token,
|
||||
)
|
||||
|
||||
if "user_data" not in used:
|
||||
log(
|
||||
|
||||
@@ -40,6 +40,47 @@ class Check:
|
||||
detail: str
|
||||
|
||||
|
||||
def _check_balance(cfg: Config, checks: list[Check]) -> None:
|
||||
from gpu_rent.balance import MEMBER_MIN_RUB, fetch_balance_rub
|
||||
|
||||
token = (cfg.selectel_api_token or "").strip()
|
||||
if not token:
|
||||
checks.append(
|
||||
Check(
|
||||
"баланс ₽",
|
||||
True,
|
||||
False,
|
||||
"нет SELECTEL_API_TOKEN — ₽ не проверяем (403 create часто = нет денег)",
|
||||
)
|
||||
)
|
||||
return
|
||||
try:
|
||||
rub = fetch_balance_rub(token)
|
||||
except Exception as exc:
|
||||
checks.append(Check("баланс ₽", True, False, f"не прочитать: {exc}"))
|
||||
return
|
||||
if rub <= 0:
|
||||
checks.append(
|
||||
Check(
|
||||
"баланс ₽",
|
||||
False,
|
||||
True,
|
||||
f"{rub:.0f} ₽ — GPU не создать. Пополни в панели Selectel.",
|
||||
)
|
||||
)
|
||||
elif rub < MEMBER_MIN_RUB:
|
||||
checks.append(
|
||||
Check(
|
||||
"баланс ₽",
|
||||
True,
|
||||
False,
|
||||
f"{rub:.0f} ₽ — мало (member обычно ≥{MEMBER_MIN_RUB:.0f} ₽)",
|
||||
)
|
||||
)
|
||||
else:
|
||||
checks.append(Check("баланс ₽", True, False, f"{rub:.0f} ₽"))
|
||||
|
||||
|
||||
def run_doctor() -> list[Check]:
|
||||
checks: list[Check] = []
|
||||
runtime_dir().mkdir(parents=True, exist_ok=True)
|
||||
@@ -123,6 +164,8 @@ def run_doctor() -> list[Check]:
|
||||
except CloudError as exc:
|
||||
checks.append(Check("квота GPU", False, True, str(exc)))
|
||||
|
||||
_check_balance(cfg, checks)
|
||||
|
||||
flavors = list(iter_flavors(conn))
|
||||
gpu_flavors = [f for f in flavors if looks_like_gpu(f)]
|
||||
ranked = rank_flavors(
|
||||
|
||||
@@ -613,6 +613,7 @@ def cmd_up(
|
||||
spot=spot,
|
||||
public_key=public_key,
|
||||
log=log,
|
||||
api_token=cfg.selectel_api_token,
|
||||
)
|
||||
state.server_id = server.id
|
||||
state.server_name = getattr(server, "name", None)
|
||||
|
||||
Reference in New Issue
Block a user