Enhance SSH configuration and security group management

- Added optional SSH security-group CIDR configuration in `env.example` to allow broader access if needed.
- Improved `guess_operator_cidr` function to support multiple IP retrieval services and allow overriding CIDR via environment variable.
- Refactored security group management to ensure idempotency and proper logging of security group rules.
- Introduced helper function `_get_volume_or_none` for better volume management and error handling.
- Updated `wait_ssh` function to enhance SSH connection handling with improved timeout settings and logging.
- Added state reset logic in `cmd_up` to handle scenarios where the server is not found in the cloud.
This commit is contained in:
Leonid Pershin
2026-08-21 04:12:38 +03:00
parent 28019d1d09
commit d6d247875b
4 changed files with 123 additions and 39 deletions
+3
View File
@@ -12,6 +12,9 @@ GPU_RENT_AZ=ru-7a
SSH_PRIVATE_KEY_PATH= SSH_PRIVATE_KEY_PATH=
SSH_USER=ubuntu SSH_USER=ubuntu
# Optional: force SSH security-group CIDR (default = your public IP /32).
# If WARP/VPN and SSH hangs: GPU_RENT_SSH_CIDR=0.0.0.0/0
# GPU_RENT_SSH_CIDR=0.0.0.0/0
BOOT_VOLUME_ID= BOOT_VOLUME_ID=
DATA_VOLUME_ID= DATA_VOLUME_ID=
+86 -32
View File
@@ -44,13 +44,24 @@ def _oid(obj: Any) -> str:
def guess_operator_cidr() -> str: def guess_operator_cidr() -> str:
try: """Public egress /32 for SSH SG. Override: GPU_RENT_SSH_CIDR=0.0.0.0/0."""
response = httpx.get("https://ifconfig.me/ip", timeout=5.0) import os
ip = response.text.strip()
if ip.count(".") == 3 and all(p.isdigit() for p in ip.split(".")): override = (os.environ.get("GPU_RENT_SSH_CIDR") or "").strip()
return f"{ip}/32" if override:
except httpx.HTTPError: return override
pass for url in (
"https://ifconfig.me/ip",
"https://api.ipify.org",
"https://icanhazip.com",
):
try:
response = httpx.get(url, timeout=5.0, follow_redirects=True)
ip = response.text.strip().split()[0]
if ip.count(".") == 3 and all(p.isdigit() for p in ip.split(".")):
return f"{ip}/32"
except httpx.HTTPError:
continue
return "0.0.0.0/0" return "0.0.0.0/0"
@@ -142,27 +153,54 @@ def ensure_network(conn, log: Callable[[str], None]) -> tuple[Any, Any]:
def ensure_security_group(conn, cidr: str, log: Callable[[str], None]) -> Any: def ensure_security_group(conn, cidr: str, log: Callable[[str], None]) -> Any:
sg = conn.network.find_security_group(SG_NAME) sg = conn.network.find_security_group(SG_NAME)
if sg: created = False
log(f"security group {SG_NAME} уже есть") if not sg:
return sg try:
sg = conn.network.create_security_group(
name=SG_NAME,
description="gpu-rent SSH only",
)
created = True
except Exception as exc:
raise _wrap(exc, "security group") from exc
# Ensure TCP/22 from operator CIDR (idempotent; old /32 from WARP may be stale).
have_cidr = False
try: try:
sg = conn.network.create_security_group( for rule in conn.network.security_group_rules(security_group_id=sg.id):
name=SG_NAME, if (
description="gpu-rent SSH only", getattr(rule, "direction", None) == "ingress"
) and getattr(rule, "protocol", None) == "tcp"
conn.network.create_security_group_rule( and int(getattr(rule, "port_range_min", 0) or 0) == 22
security_group_id=sg.id, and int(getattr(rule, "port_range_max", 0) or 0) == 22
direction="ingress", and (getattr(rule, "remote_ip_prefix", None) or "") == cidr
ethertype="IPv4", ):
protocol="tcp", have_cidr = True
port_range_min=22, break
port_range_max=22, except Exception:
remote_ip_prefix=cidr, have_cidr = False
) if not have_cidr:
try:
conn.network.create_security_group_rule(
security_group_id=sg.id,
direction="ingress",
ethertype="IPv4",
protocol="tcp",
port_range_min=22,
port_range_max=22,
remote_ip_prefix=cidr,
)
log(f"SG {SG_NAME}: + TCP/22 с {cidr}")
except Exception as exc:
text = str(exc).lower()
if "already" not in text and "conflict" not in text and "409" not in text:
raise _wrap(exc, "security group rule") from exc
log(f"SG {SG_NAME}: TCP/22 с {cidr} уже есть")
elif created:
log(f"SG {SG_NAME}: TCP/22 с {cidr}") log(f"SG {SG_NAME}: TCP/22 с {cidr}")
return sg else:
except Exception as exc: log(f"security group {SG_NAME} уже есть (SSH {cidr})")
raise _wrap(exc, "security group") from exc return sg
def ensure_keypair(conn, public_key: str, log: Callable[[str], None]) -> Any: def ensure_keypair(conn, public_key: str, log: Callable[[str], None]) -> Any:
@@ -186,6 +224,18 @@ def ensure_keypair(conn, public_key: str, log: Callable[[str], None]) -> Any:
raise _wrap(exc, "keypair") from exc raise _wrap(exc, "keypair") from exc
def _get_volume_or_none(conn, volume_id: str) -> Any | None:
if not volume_id:
return None
try:
return conn.block_storage.get_volume(volume_id)
except Exception as exc:
text = str(exc).lower()
if "404" in text or "not found" in text or "could not be found" in text:
return None
raise
def ensure_boot_volume( def ensure_boot_volume(
conn, conn,
*, *,
@@ -197,9 +247,11 @@ def ensure_boot_volume(
log: Callable[[str], None], log: Callable[[str], None],
) -> Any: ) -> Any:
if existing_id: if existing_id:
vol = conn.block_storage.get_volume(existing_id) vol = _get_volume_or_none(conn, existing_id)
log(f"boot volume из state {existing_id}") if vol is not None:
return vol log(f"boot volume из state {existing_id}")
return vol
log(f"boot volume {existing_id} нет в облаке — создаём заново")
found = find_volumes_by_name(conn, BOOT_VOLUME_NAME) found = find_volumes_by_name(conn, BOOT_VOLUME_NAME)
if found: if found:
log(f"boot volume {BOOT_VOLUME_NAME} уже в проекте") log(f"boot volume {BOOT_VOLUME_NAME} уже в проекте")
@@ -245,9 +297,11 @@ def ensure_data_volume(
log: Callable[[str], None], log: Callable[[str], None],
) -> Any: ) -> Any:
if existing_id: if existing_id:
vol = conn.block_storage.get_volume(existing_id) vol = _get_volume_or_none(conn, existing_id)
log(f"data volume из state {existing_id}") if vol is not None:
return vol log(f"data volume из state {existing_id}")
return vol
log(f"data volume {existing_id} нет в облаке — создаём заново")
found = find_volumes_by_name(conn, DATA_VOLUME_NAME) found = find_volumes_by_name(conn, DATA_VOLUME_NAME)
if found: if found:
log(f"data volume {DATA_VOLUME_NAME} уже в проекте") log(f"data volume {DATA_VOLUME_NAME} уже в проекте")
+10
View File
@@ -155,6 +155,16 @@ def cmd_up(
return state return state
raise CloudError(f"сервер gpu-rent в статусе {status} — разбери в панели") raise CloudError(f"сервер gpu-rent в статусе {status} — разбери в панели")
if state.server_id:
log(f"в state был server {state.server_id}, в облаке нет — создаём заново")
state.server_id = None
state.server_name = None
state.floating_ip = None
state.floating_ip_id = None
state.bootstrapped = False
state.phase = "idle"
save_state(state)
flavors = list(iter_flavors(conn)) flavors = list(iter_flavors(conn))
try: try:
picked = resolve_flavor( picked = resolve_flavor(
+24 -7
View File
@@ -28,12 +28,20 @@ def wait_tcp(host: str, port: int, timeout: float = 300.0) -> None:
raise CloudError(f"TCP {host}:{port} не открылся за {int(timeout)} с ({last})") raise CloudError(f"TCP {host}:{port} не открылся за {int(timeout)} с ({last})")
def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None: def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None:
wait_tcp(host, 22, timeout=min(timeout, 240)) """Wait until sshd accepts our key. Paramiko banner noise is muted."""
import logging
logging.getLogger("paramiko").setLevel(logging.CRITICAL)
logging.getLogger("paramiko.transport").setLevel(logging.CRITICAL)
wait_tcp(host, 22, timeout=min(timeout, 300))
deadline = time.time() + timeout deadline = time.time() + timeout
key = str(cfg.ssh_private_key_path) key = str(cfg.ssh_private_key_path)
last = None last = None
attempt = 0
while time.time() < deadline: while time.time() < deadline:
attempt += 1
client = paramiko.SSHClient() client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try: try:
@@ -41,9 +49,11 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
hostname=host, hostname=host,
username=cfg.ssh_user, username=cfg.ssh_user,
key_filename=key, key_filename=key,
timeout=12, timeout=20,
banner_timeout=12, banner_timeout=60,
auth_timeout=12, auth_timeout=30,
allow_agent=False,
look_for_keys=False,
) )
client.close() client.close()
return return
@@ -53,8 +63,15 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None:
client.close() client.close()
except Exception: except Exception:
pass pass
time.sleep(5) if attempt == 1 or attempt % 6 == 0:
raise CloudError(f"SSH {cfg.ssh_user}@{host} не принял ключ ({last})") # Progress without paramiko traceback spam.
print(f"жду SSH {cfg.ssh_user}@{host}… ({type(exc).__name__})", flush=True)
time.sleep(8)
raise CloudError(
f"SSH {cfg.ssh_user}@{host} не принял ключ за {int(timeout)} с ({last}). "
"Часто cloud-init ещё поднимает sshd на GPU-образе. "
"Если SG узкий (VPN/WARP): в .env GPU_RENT_SSH_CIDR=0.0.0.0/0 и снова up."
)
def ssh_argv(cfg: Config, host: str, remote: list[str] | None = None) -> list[str]: def ssh_argv(cfg: Config, host: str, remote: list[str] | None = None) -> list[str]: