Implement balance monitoring and notification for Selectel API integration

- Added support for balance tracking using `SELECTEL_API_TOKEN` in the configuration.
- Introduced new balance notification logic in the local watchdog, alerting users on balance changes based on defined thresholds.
- Updated documentation to include instructions for setting up balance notifications and the required environment variables.
- Enhanced the `ready` and `session` modules to initialize balance state and handle notifications during GPU operations.
- Refactored the CLI and related components to support the new balance monitoring features, ensuring a seamless user experience.
This commit is contained in:
Leonid Pershin
2026-08-21 06:52:51 +03:00
parent 7ed6a99df2
commit 09b7c36f3b
14 changed files with 759 additions and 12 deletions
+41 -1
View File
@@ -22,7 +22,7 @@ from gpu_rent.cloud import (
)
from gpu_rent.bootstrap import run_bootstrap
from gpu_rent.provision import provision_vm, tune_swarm_perf
from gpu_rent.ready import wait_backend_idle
from gpu_rent.ready import verify_stack_on_vm, wait_backend_idle
from gpu_rent.snapshot import ensure_boot_snapshot
from gpu_rent.notify import notify_ready
from gpu_rent.config import Config
@@ -127,6 +127,19 @@ def _bind_access(
log(f"perf tune: {exc}")
else:
log("ready: llm-only (без ожидания SwarmUI Idle)")
try:
checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0)
state.notes = dict(state.notes or {})
state.notes["stack_vm"] = [
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
]
except CloudError as exc:
state.notes = dict(state.notes or {})
state.notes["stack_vm_error"] = str(exc)[:500]
save_state(state)
raise
try:
ensure_boot_snapshot(
conn,
@@ -137,6 +150,27 @@ def _bind_access(
except CloudError as exc:
log(f"snapshot: {exc}")
notify_ready(cfg, log)
try:
from gpu_rent.balance import (
BalanceWatchState,
fetch_balance_rub,
save_balance_state,
)
if cfg.selectel_api_token:
rub = fetch_balance_rub(cfg.selectel_api_token)
save_balance_state(
BalanceWatchState(baseline_rub=rub, last_balance_rub=rub)
)
log(
f"balance: baseline {rub:.0f}"
f"(уведомление каждые {cfg.balance_notify_step_rub:.0f} ₽, "
"нужен local-watchdog)"
)
else:
log("balance: SELECTEL_API_TOKEN не задан — шаги по ₽ skip")
except Exception as exc:
log(f"balance baseline: {exc}")
state.bootstrapped = True
state.phase = "ready_cloud"
state.notes = dict(state.notes or {})
@@ -464,5 +498,11 @@ def cmd_stop(
clear_lease()
except Exception:
pass
try:
from gpu_rent.balance import clear_balance_state
clear_balance_state()
except Exception:
pass
log("фаза idle" + ("" if destroy_disks else " (диски на месте)"))
return state