Enhance SSH handling in session management

- Introduced `probe_ssh` function for quick SSH checks, improving the handling of server bootstrap scenarios.
- Updated `cmd_up` to utilize `probe_ssh`, enhancing the logic for managing server states based on SSH availability.
- Improved logging for server state transitions and error handling during SSH connection attempts.
- Adjusted timeout settings in `wait_ssh` for better performance and reliability in SSH key acceptance checks.
This commit is contained in:
Leonid Pershin
2026-08-21 04:37:09 +03:00
parent 1f7237f7db
commit c7a56bdd56
2 changed files with 71 additions and 42 deletions
+45 -7
View File
@@ -28,6 +28,48 @@ def wait_tcp(host: str, port: int, timeout: float = 300.0) -> None:
raise CloudError(f"TCP {host}:{port} не открылся за {int(timeout)} с ({last})")
def probe_ssh(cfg: Config, host: str, attempts: int = 3) -> str:
"""Quick SSH check. Returns 'ok' | 'auth' | 'down'."""
import logging
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
logging.getLogger("paramiko.transport").setLevel(logging.CRITICAL)
try:
with socket.create_connection((host, 22), timeout=8):
pass
except OSError:
return "down"
key = str(cfg.ssh_private_key_path)
auth_seen = False
for _ in range(max(1, attempts)):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
client.connect(
hostname=host,
username=cfg.ssh_user,
key_filename=key,
timeout=12,
banner_timeout=20,
auth_timeout=15,
allow_agent=False,
look_for_keys=False,
)
client.close()
return "ok"
except Exception as exc:
name = type(exc).__name__
if "Authentication" in name or "Authentication" in str(exc):
auth_seen = True
try:
client.close()
except Exception:
pass
time.sleep(2)
return "auth" if auth_seen else "down"
def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
"""Wait until sshd accepts our key. Paramiko banner noise is muted."""
import logging
@@ -70,17 +112,13 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
if attempt == 1 or attempt % 6 == 0:
print(f"жду SSH {cfg.ssh_user}@{host}… ({name})", flush=True)
# Key never injected (boot-from-volume): don't burn full timeout.
if auth_fails >= 8:
if auth_fails >= 3:
raise CloudError(
f"SSH {cfg.ssh_user}@{host}: ключ отклонён (AuthenticationException). "
"Nova keypair не попал в authorized_keys при boot-from-volume. "
"Останови up (Ctrl+C), затем снова: gpu-rent up --yes "
"(create теперь с config_drive + user_data). "
"Или в консоли панели (root): "
f"добавь содержимое {cfg.ssh_private_key_path}.pub в "
f"/home/{cfg.ssh_user}/.ssh/authorized_keys"
"up пересоздаст compute с user_data (Base64), диски оставит."
) from exc
time.sleep(8)
time.sleep(5)
raise CloudError(
f"SSH {cfg.ssh_user}@{host} не принял ключ за {int(timeout)} с ({last}). "
"Часто cloud-init ещё поднимает sshd на GPU-образе. "