Enhance SSH handling in session management

- Introduced `probe_ssh` function for quick SSH checks, improving the handling of server bootstrap scenarios.
- Updated `cmd_up` to utilize `probe_ssh`, enhancing the logic for managing server states based on SSH availability.
- Improved logging for server state transitions and error handling during SSH connection attempts.
- Adjusted timeout settings in `wait_ssh` for better performance and reliability in SSH key acceptance checks.
This commit is contained in:
Leonid Pershin
2026-08-21 04:37:09 +03:00
parent 1f7237f7db
commit c7a56bdd56
2 changed files with 71 additions and 42 deletions
+26 -35
View File
@@ -46,7 +46,7 @@ from gpu_rent.os_client import (
iter_volume_types,
)
from gpu_rent.ssh_keys import ensure_ed25519
from gpu_rent.ssh_ops import wait_ssh
from gpu_rent.ssh_ops import probe_ssh, wait_ssh
from gpu_rent.state import SessionState, load_state, save_state, utc_now
Log = Callable[[str], None]
@@ -147,7 +147,8 @@ def cmd_up(
save_state(state)
_bind_access(conn, existing, state, cfg, log)
return state
# First boot / stuck without injected key: probe SSH briefly.
# Bootstrap не завершён: почти всегда VM без authorized_keys.
fip = state.floating_ip
if not fip:
try:
@@ -160,43 +161,33 @@ def cmd_up(
save_state(state)
except Exception as exc:
log(f"FIP: {exc}")
if fip:
try:
wait_ssh(cfg, fip, timeout=90)
log("сервер уже ACTIVE — второй GPU не создаём")
state.phase = "ready_cloud"
save_state(state)
_bind_access(conn, existing, state, cfg, log)
return state
except CloudError as exc:
msg = str(exc)
if "ключ отклонён" in msg or "Authentication" in msg:
log(
"SSH ключ не на VM (boot-from-volume) — "
"удаляем compute, диски оставляем, create с config_drive"
)
delete_server(conn, existing, log)
state.server_id = None
state.server_name = None
state.bootstrapped = False
state.phase = "idle"
save_state(state)
for vid in (state.boot_volume_id, state.data_volume_id):
if not vid:
continue
try:
wait_volume(conn, conn.block_storage.get_volume(vid), "available")
except Exception as vol_exc:
log(f"wait volume {vid}: {vol_exc}")
existing = None
else:
raise
if existing is not None:
log("сервер уже ACTIVE — второй GPU не создаём")
outcome = probe_ssh(cfg, fip, attempts=2) if fip else "down"
if outcome == "ok":
log("сервер ACTIVE, SSH ок — продолжаем bootstrap")
state.phase = "ready_cloud"
save_state(state)
_bind_access(conn, existing, state, cfg, log)
return state
log(
f"ACTIVE без bootstrap, SSH={outcome}"
"удаляем compute (диски оставляем), create с user_data Base64"
)
delete_server(conn, existing, log)
state.server_id = None
state.server_name = None
state.bootstrapped = False
state.phase = "idle"
save_state(state)
for vid in (state.boot_volume_id, state.data_volume_id):
if not vid:
continue
try:
wait_volume(conn, conn.block_storage.get_volume(vid), "available")
except Exception as vol_exc:
log(f"wait volume {vid}: {vol_exc}")
existing = None
if existing is not None and status in {"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"}:
existing = unshelve(conn, existing, log)
state.server_id = existing.id