Enhance SSH handling in session management
- Introduced `probe_ssh` function for quick SSH checks, improving the handling of server bootstrap scenarios. - Updated `cmd_up` to utilize `probe_ssh`, enhancing the logic for managing server states based on SSH availability. - Improved logging for server state transitions and error handling during SSH connection attempts. - Adjusted timeout settings in `wait_ssh` for better performance and reliability in SSH key acceptance checks.
This commit is contained in:
+26
-35
@@ -46,7 +46,7 @@ from gpu_rent.os_client import (
|
||||
iter_volume_types,
|
||||
)
|
||||
from gpu_rent.ssh_keys import ensure_ed25519
|
||||
from gpu_rent.ssh_ops import wait_ssh
|
||||
from gpu_rent.ssh_ops import probe_ssh, wait_ssh
|
||||
from gpu_rent.state import SessionState, load_state, save_state, utc_now
|
||||
|
||||
Log = Callable[[str], None]
|
||||
@@ -147,7 +147,8 @@ def cmd_up(
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
return state
|
||||
# First boot / stuck without injected key: probe SSH briefly.
|
||||
|
||||
# Bootstrap не завершён: почти всегда VM без authorized_keys.
|
||||
fip = state.floating_ip
|
||||
if not fip:
|
||||
try:
|
||||
@@ -160,43 +161,33 @@ def cmd_up(
|
||||
save_state(state)
|
||||
except Exception as exc:
|
||||
log(f"FIP: {exc}")
|
||||
if fip:
|
||||
try:
|
||||
wait_ssh(cfg, fip, timeout=90)
|
||||
log("сервер уже ACTIVE — второй GPU не создаём")
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
return state
|
||||
except CloudError as exc:
|
||||
msg = str(exc)
|
||||
if "ключ отклонён" in msg or "Authentication" in msg:
|
||||
log(
|
||||
"SSH ключ не на VM (boot-from-volume) — "
|
||||
"удаляем compute, диски оставляем, create с config_drive"
|
||||
)
|
||||
delete_server(conn, existing, log)
|
||||
state.server_id = None
|
||||
state.server_name = None
|
||||
state.bootstrapped = False
|
||||
state.phase = "idle"
|
||||
save_state(state)
|
||||
for vid in (state.boot_volume_id, state.data_volume_id):
|
||||
if not vid:
|
||||
continue
|
||||
try:
|
||||
wait_volume(conn, conn.block_storage.get_volume(vid), "available")
|
||||
except Exception as vol_exc:
|
||||
log(f"wait volume {vid}: {vol_exc}")
|
||||
existing = None
|
||||
else:
|
||||
raise
|
||||
if existing is not None:
|
||||
log("сервер уже ACTIVE — второй GPU не создаём")
|
||||
|
||||
outcome = probe_ssh(cfg, fip, attempts=2) if fip else "down"
|
||||
if outcome == "ok":
|
||||
log("сервер ACTIVE, SSH ок — продолжаем bootstrap")
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
return state
|
||||
|
||||
log(
|
||||
f"ACTIVE без bootstrap, SSH={outcome} — "
|
||||
"удаляем compute (диски оставляем), create с user_data Base64"
|
||||
)
|
||||
delete_server(conn, existing, log)
|
||||
state.server_id = None
|
||||
state.server_name = None
|
||||
state.bootstrapped = False
|
||||
state.phase = "idle"
|
||||
save_state(state)
|
||||
for vid in (state.boot_volume_id, state.data_volume_id):
|
||||
if not vid:
|
||||
continue
|
||||
try:
|
||||
wait_volume(conn, conn.block_storage.get_volume(vid), "available")
|
||||
except Exception as vol_exc:
|
||||
log(f"wait volume {vid}: {vol_exc}")
|
||||
existing = None
|
||||
if existing is not None and status in {"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"}:
|
||||
existing = unshelve(conn, existing, log)
|
||||
state.server_id = existing.id
|
||||
|
||||
Reference in New Issue
Block a user