Enhance SSH handling in session management
- Introduced `probe_ssh` function for quick SSH checks, improving the handling of server bootstrap scenarios. - Updated `cmd_up` to utilize `probe_ssh`, enhancing the logic for managing server states based on SSH availability. - Improved logging for server state transitions and error handling during SSH connection attempts. - Adjusted timeout settings in `wait_ssh` for better performance and reliability in SSH key acceptance checks.
This commit is contained in:
+45
-7
@@ -28,6 +28,48 @@ def wait_tcp(host: str, port: int, timeout: float = 300.0) -> None:
|
||||
raise CloudError(f"TCP {host}:{port} не открылся за {int(timeout)} с ({last})")
|
||||
|
||||
|
||||
def probe_ssh(cfg: Config, host: str, attempts: int = 3) -> str:
|
||||
"""Quick SSH check. Returns 'ok' | 'auth' | 'down'."""
|
||||
import logging
|
||||
|
||||
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("paramiko.transport").setLevel(logging.CRITICAL)
|
||||
try:
|
||||
with socket.create_connection((host, 22), timeout=8):
|
||||
pass
|
||||
except OSError:
|
||||
return "down"
|
||||
|
||||
key = str(cfg.ssh_private_key_path)
|
||||
auth_seen = False
|
||||
for _ in range(max(1, attempts)):
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
client.connect(
|
||||
hostname=host,
|
||||
username=cfg.ssh_user,
|
||||
key_filename=key,
|
||||
timeout=12,
|
||||
banner_timeout=20,
|
||||
auth_timeout=15,
|
||||
allow_agent=False,
|
||||
look_for_keys=False,
|
||||
)
|
||||
client.close()
|
||||
return "ok"
|
||||
except Exception as exc:
|
||||
name = type(exc).__name__
|
||||
if "Authentication" in name or "Authentication" in str(exc):
|
||||
auth_seen = True
|
||||
try:
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
return "auth" if auth_seen else "down"
|
||||
|
||||
|
||||
def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
|
||||
"""Wait until sshd accepts our key. Paramiko banner noise is muted."""
|
||||
import logging
|
||||
@@ -70,17 +112,13 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
|
||||
if attempt == 1 or attempt % 6 == 0:
|
||||
print(f"жду SSH {cfg.ssh_user}@{host}… ({name})", flush=True)
|
||||
# Key never injected (boot-from-volume): don't burn full timeout.
|
||||
if auth_fails >= 8:
|
||||
if auth_fails >= 3:
|
||||
raise CloudError(
|
||||
f"SSH {cfg.ssh_user}@{host}: ключ отклонён (AuthenticationException). "
|
||||
"Nova keypair не попал в authorized_keys при boot-from-volume. "
|
||||
"Останови up (Ctrl+C), затем снова: gpu-rent up --yes "
|
||||
"(create теперь с config_drive + user_data). "
|
||||
"Или в консоли панели (root): "
|
||||
f"добавь содержимое {cfg.ssh_private_key_path}.pub в "
|
||||
f"/home/{cfg.ssh_user}/.ssh/authorized_keys"
|
||||
"up пересоздаст compute с user_data (Base64), диски оставит."
|
||||
) from exc
|
||||
time.sleep(8)
|
||||
time.sleep(5)
|
||||
raise CloudError(
|
||||
f"SSH {cfg.ssh_user}@{host} не принял ключ за {int(timeout)} с ({last}). "
|
||||
"Часто cloud-init ещё поднимает sshd на GPU-образе. "
|
||||
|
||||
Reference in New Issue
Block a user