Implement balance monitoring and notification for Selectel API integration
- Added support for balance tracking using `SELECTEL_API_TOKEN` in the configuration. - Introduced new balance notification logic in the local watchdog, alerting users on balance changes based on defined thresholds. - Updated documentation to include instructions for setting up balance notifications and the required environment variables. - Enhanced the `ready` and `session` modules to initialize balance state and handle notifications during GPU operations. - Refactored the CLI and related components to support the new balance monitoring features, ensuring a seamless user experience.
This commit is contained in:
+41
-1
@@ -22,7 +22,7 @@ from gpu_rent.cloud import (
|
||||
)
|
||||
from gpu_rent.bootstrap import run_bootstrap
|
||||
from gpu_rent.provision import provision_vm, tune_swarm_perf
|
||||
from gpu_rent.ready import wait_backend_idle
|
||||
from gpu_rent.ready import verify_stack_on_vm, wait_backend_idle
|
||||
from gpu_rent.snapshot import ensure_boot_snapshot
|
||||
from gpu_rent.notify import notify_ready
|
||||
from gpu_rent.config import Config
|
||||
@@ -127,6 +127,19 @@ def _bind_access(
|
||||
log(f"perf tune: {exc}")
|
||||
else:
|
||||
log("ready: llm-only (без ожидания SwarmUI Idle)")
|
||||
|
||||
try:
|
||||
checks = verify_stack_on_vm(cfg, ip, log, timeout=300.0)
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["stack_vm"] = [
|
||||
{"name": c.name, "ok": c.ok, "detail": c.detail} for c in checks
|
||||
]
|
||||
except CloudError as exc:
|
||||
state.notes = dict(state.notes or {})
|
||||
state.notes["stack_vm_error"] = str(exc)[:500]
|
||||
save_state(state)
|
||||
raise
|
||||
|
||||
try:
|
||||
ensure_boot_snapshot(
|
||||
conn,
|
||||
@@ -137,6 +150,27 @@ def _bind_access(
|
||||
except CloudError as exc:
|
||||
log(f"snapshot: {exc}")
|
||||
notify_ready(cfg, log)
|
||||
try:
|
||||
from gpu_rent.balance import (
|
||||
BalanceWatchState,
|
||||
fetch_balance_rub,
|
||||
save_balance_state,
|
||||
)
|
||||
|
||||
if cfg.selectel_api_token:
|
||||
rub = fetch_balance_rub(cfg.selectel_api_token)
|
||||
save_balance_state(
|
||||
BalanceWatchState(baseline_rub=rub, last_balance_rub=rub)
|
||||
)
|
||||
log(
|
||||
f"balance: baseline {rub:.0f} ₽ "
|
||||
f"(уведомление каждые {cfg.balance_notify_step_rub:.0f} ₽, "
|
||||
"нужен local-watchdog)"
|
||||
)
|
||||
else:
|
||||
log("balance: SELECTEL_API_TOKEN не задан — шаги по ₽ skip")
|
||||
except Exception as exc:
|
||||
log(f"balance baseline: {exc}")
|
||||
state.bootstrapped = True
|
||||
state.phase = "ready_cloud"
|
||||
state.notes = dict(state.notes or {})
|
||||
@@ -464,5 +498,11 @@ def cmd_stop(
|
||||
clear_lease()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from gpu_rent.balance import clear_balance_state
|
||||
|
||||
clear_balance_state()
|
||||
except Exception:
|
||||
pass
|
||||
log("фаза idle" + ("" if destroy_disks else " (диски на месте)"))
|
||||
return state
|
||||
|
||||
Reference in New Issue
Block a user