Enhance SSH configuration and security group management

- Added optional SSH security-group CIDR configuration in `env.example` to allow broader access if needed.
- Improved `guess_operator_cidr` function to support multiple IP retrieval services and allow overriding CIDR via environment variable.
- Refactored security group management to ensure idempotency and proper logging of security group rules.
- Introduced helper function `_get_volume_or_none` for better volume management and error handling.
- Updated `wait_ssh` function to enhance SSH connection handling with improved timeout settings and logging.
- Added state reset logic in `cmd_up` to handle scenarios where the server is not found in the cloud.
This commit is contained in:
Leonid Pershin
2026-08-21 04:12:38 +03:00
parent 28019d1d09
commit d6d247875b
4 changed files with 123 additions and 39 deletions
+24 -7
View File
@@ -28,12 +28,20 @@ def wait_tcp(host: str, port: int, timeout: float = 300.0) -> None:
raise CloudError(f"TCP {host}:{port} не открылся за {int(timeout)} с ({last})")
def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
wait_tcp(host, 22, timeout=min(timeout, 240))
def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
"""Wait until sshd accepts our key. Paramiko banner noise is muted."""
import logging
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
logging.getLogger("paramiko.transport").setLevel(logging.CRITICAL)
wait_tcp(host, 22, timeout=min(timeout, 300))
deadline = time.time() + timeout
key = str(cfg.ssh_private_key_path)
last = None
attempt = 0
while time.time() < deadline:
attempt += 1
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
@@ -41,9 +49,11 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
hostname=host,
username=cfg.ssh_user,
key_filename=key,
timeout=12,
banner_timeout=12,
auth_timeout=12,
timeout=20,
banner_timeout=60,
auth_timeout=30,
allow_agent=False,
look_for_keys=False,
)
client.close()
return
@@ -53,8 +63,15 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
client.close()
except Exception:
pass
time.sleep(5)
raise CloudError(f"SSH {cfg.ssh_user}@{host} не принял ключ ({last})")
if attempt == 1 or attempt % 6 == 0:
# Progress without paramiko traceback spam.
print(f"жду SSH {cfg.ssh_user}@{host}… ({type(exc).__name__})", flush=True)
time.sleep(8)
raise CloudError(
f"SSH {cfg.ssh_user}@{host} не принял ключ за {int(timeout)} с ({last}). "
"Часто cloud-init ещё поднимает sshd на GPU-образе. "
"Если SG узкий (VPN/WARP): в .env GPU_RENT_SSH_CIDR=0.0.0.0/0 и снова up."
)
def ssh_argv(cfg: Config, host: str, remote: list[str] | None = None) -> list[str]: