Enhance SSH configuration and security group management
- Added optional SSH security-group CIDR configuration in `env.example` to allow broader access if needed. - Improved `guess_operator_cidr` function to support multiple IP retrieval services and allow overriding CIDR via environment variable. - Refactored security group management to ensure idempotency and proper logging of security group rules. - Introduced helper function `_get_volume_or_none` for better volume management and error handling. - Updated `wait_ssh` function to enhance SSH connection handling with improved timeout settings and logging. - Added state reset logic in `cmd_up` to handle scenarios where the server is not found in the cloud.
This commit is contained in:
+24
-7
@@ -28,12 +28,20 @@ def wait_tcp(host: str, port: int, timeout: float = 300.0) -> None:
|
||||
raise CloudError(f"TCP {host}:{port} не открылся за {int(timeout)} с ({last})")
|
||||
|
||||
|
||||
def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
|
||||
wait_tcp(host, 22, timeout=min(timeout, 240))
|
||||
def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
|
||||
"""Wait until sshd accepts our key. Paramiko banner noise is muted."""
|
||||
import logging
|
||||
|
||||
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
|
||||
logging.getLogger("paramiko.transport").setLevel(logging.CRITICAL)
|
||||
|
||||
wait_tcp(host, 22, timeout=min(timeout, 300))
|
||||
deadline = time.time() + timeout
|
||||
key = str(cfg.ssh_private_key_path)
|
||||
last = None
|
||||
attempt = 0
|
||||
while time.time() < deadline:
|
||||
attempt += 1
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
try:
|
||||
@@ -41,9 +49,11 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
|
||||
hostname=host,
|
||||
username=cfg.ssh_user,
|
||||
key_filename=key,
|
||||
timeout=12,
|
||||
banner_timeout=12,
|
||||
auth_timeout=12,
|
||||
timeout=20,
|
||||
banner_timeout=60,
|
||||
auth_timeout=30,
|
||||
allow_agent=False,
|
||||
look_for_keys=False,
|
||||
)
|
||||
client.close()
|
||||
return
|
||||
@@ -53,8 +63,15 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(5)
|
||||
raise CloudError(f"SSH {cfg.ssh_user}@{host} не принял ключ ({last})")
|
||||
if attempt == 1 or attempt % 6 == 0:
|
||||
# Progress without paramiko traceback spam.
|
||||
print(f"жду SSH {cfg.ssh_user}@{host}… ({type(exc).__name__})", flush=True)
|
||||
time.sleep(8)
|
||||
raise CloudError(
|
||||
f"SSH {cfg.ssh_user}@{host} не принял ключ за {int(timeout)} с ({last}). "
|
||||
"Часто cloud-init ещё поднимает sshd на GPU-образе. "
|
||||
"Если SG узкий (VPN/WARP): в .env GPU_RENT_SSH_CIDR=0.0.0.0/0 и снова up."
|
||||
)
|
||||
|
||||
|
||||
def ssh_argv(cfg: Config, host: str, remote: list[str] | None = None) -> list[str]:
|
||||
|
||||
Reference in New Issue
Block a user