Refactor GPU rental network handling and enhance IP management

- Renamed "FIP" to "IP" in the status output for clarity.
- Updated `create_gpu_server` to accept `port_id` as an optional parameter, allowing for more flexible network configurations.
- Introduced `delete_access_ip` function to manage direct public ports or floating IPs more effectively.
- Enhanced session management to prioritize direct public IPs over floating IPs, improving connection reliability.
- Added checks for direct network availability in the doctor command to ensure proper network functionality.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 18:19:34 +03:00
co-authored by Cursor
parent 0bad693b12
commit 8074426c43
7 changed files with 419 additions and 24 deletions
+1 -1
View File
@@ -297,7 +297,7 @@ def status() -> None:
table.add_row("flavor", state.flavor_name or state.flavor_id or "") table.add_row("flavor", state.flavor_name or state.flavor_id or "")
table.add_row("boot volume", state.boot_volume_id or "") table.add_row("boot volume", state.boot_volume_id or "")
table.add_row("data volume", state.data_volume_id or "") table.add_row("data volume", state.data_volume_id or "")
table.add_row("FIP", state.floating_ip or "") table.add_row("IP", state.floating_ip or "")
end = preempt_window_end(state) end = preempt_window_end(state)
if end: if end:
left = end - datetime.now(timezone.utc) left = end - datetime.now(timezone.utc)
+38 -2
View File
@@ -458,7 +458,8 @@ def create_gpu_server(
conn, conn,
*, *,
flavor_id: str, flavor_id: str,
net_id: str, net_id: str | None = None,
port_id: str | None = None,
sg_name: str, sg_name: str,
boot_volume_id: str, boot_volume_id: str,
data_volume_id: str, data_volume_id: str,
@@ -487,10 +488,16 @@ def create_gpu_server(
tags = [RESOURCE_TAG] tags = [RESOURCE_TAG]
if spot: if spot:
tags.append(PREEMPTIBLE_TAG) tags.append(PREEMPTIBLE_TAG)
if port_id:
nics: list[dict[str, str]] = [{"port": port_id}]
elif net_id:
nics = [{"uuid": net_id}]
else:
raise CloudError("create_gpu_server: нужен port_id (прямой IP) или net_id")
base: dict[str, Any] = { base: dict[str, Any] = {
"name": SERVER_NAME, "name": SERVER_NAME,
"flavor_id": flavor_id, "flavor_id": flavor_id,
"networks": [{"uuid": net_id}], "networks": nics,
"key_name": KEYPAIR_NAME, "key_name": KEYPAIR_NAME,
"availability_zone": az, "availability_zone": az,
"block_device_mapping_v2": bdm, "block_device_mapping_v2": bdm,
@@ -607,6 +614,28 @@ def delete_floating_ip(conn, fip_id: str | None, address: str | None, log: Calla
return return
def delete_access_ip(
conn,
region: str,
port_or_fip_id: str | None,
address: str | None,
log: Callable[[str], None],
) -> None:
"""Drop direct public port (preferred) or leftover Neutron floating IP."""
if port_or_fip_id and region:
try:
from gpu_rent.public_net import delete_public_port, get_public_port
if get_public_port(conn, region, port_or_fip_id) is not None:
delete_public_port(conn, region, port_or_fip_id, log)
return
except CloudError as exc:
log(f"прямой IP: {exc} — пробуем FIP")
except Exception as exc:
log(f"прямой IP не удалился: {exc} — пробуем FIP")
delete_floating_ip(conn, port_or_fip_id, address, log)
def delete_server(conn, server, log: Callable[[str], None]) -> None: def delete_server(conn, server, log: Callable[[str], None]) -> None:
sid = _oid(server) sid = _oid(server)
log(f"удаляем compute {sid} (диски оставляем)") log(f"удаляем compute {sid} (диски оставляем)")
@@ -648,6 +677,13 @@ def server_floating_ip(server: Any) -> str | None:
return None return None
def server_access_ip(server: Any) -> str | None:
"""SSH target: floating IP (legacy) or fixed public IPv4 (direct port)."""
from gpu_rent.public_net import server_public_ip
return server_public_ip(server)
def pick_existing_server(conn) -> Any | None: def pick_existing_server(conn) -> Any | None:
found = find_tagged_servers(conn) found = find_tagged_servers(conn)
if len(found) > 1: if len(found) > 1:
+37
View File
@@ -81,6 +81,42 @@ def _check_balance(cfg: Config, checks: list[Check]) -> None:
checks.append(Check("баланс ₽", True, False, f"{rub:.0f}")) checks.append(Check("баланс ₽", True, False, f"{rub:.0f}"))
def _check_direct_net(cfg: Config, conn, checks: list[Check]) -> None:
from gpu_rent.os_client import ROUTER_NAME
from gpu_rent.public_net import public_net_request
try:
response = public_net_request(conn, cfg.os_region_name, "GET", "/v1/public_ports")
if response.status_code >= 400:
checks.append(
Check(
"прямой IP API",
True,
False,
f"public-net HTTP {response.status_code} в {cfg.os_region_name} — up покажет ошибку",
)
)
else:
checks.append(
Check("прямой IP API", True, False, f"public-net ok ({cfg.os_region_name})")
)
except Exception as exc:
checks.append(Check("прямой IP API", True, False, f"не достучались: {exc}"))
try:
router = conn.network.find_router(ROUTER_NAME)
except Exception:
router = None
if router:
checks.append(
Check(
"роутер gpu-rent",
True,
False,
"ещё жив — 200 ₽/мес; up/stop удалит (SSH идёт на прямой публичный IP)",
)
)
def run_doctor() -> list[Check]: def run_doctor() -> list[Check]:
checks: list[Check] = [] checks: list[Check] = []
runtime_dir().mkdir(parents=True, exist_ok=True) runtime_dir().mkdir(parents=True, exist_ok=True)
@@ -129,6 +165,7 @@ def run_doctor() -> list[Check]:
try: try:
conn = connect(cfg) conn = connect(cfg)
checks.append(Check("Keystone", True, True, "IAM-токен выдан (TTL ~24 ч, sdk обновит)")) checks.append(Check("Keystone", True, True, "IAM-токен выдан (TTL ~24 ч, sdk обновит)"))
_check_direct_net(cfg, conn, checks)
except CloudError as exc: except CloudError as exc:
checks.append(Check("Keystone", False, True, str(exc))) checks.append(Check("Keystone", False, True, str(exc)))
_local_manifests(cfg, checks) _local_manifests(cfg, checks)
+290
View File
@@ -0,0 +1,290 @@
"""Selectel Public Net API: direct public IP (no router, no floating IP).
Endpoint (Keystone project token):
https://<pool>.cloud.api.selcloud.ru/public-net/v1/public_ports
Docs: https://docs.selectel.ru/api/cloud-public-network/
Go SDK example URL: https://ru-3.cloud.api.selcloud.ru/public-net/
"""
from __future__ import annotations
import ipaddress
import time
from dataclasses import dataclass
from typing import Any, Callable
import httpx
from gpu_rent.errors import CloudError
from gpu_rent.os_client import NET_NAME, ROUTER_NAME, SUBNET_NAME
Log = Callable[[str], None]
PORT_DESCRIPTION = "gpu-rent"
def public_net_base(region: str) -> str:
pool = (region or "").strip()
if not pool:
raise CloudError("OS_REGION_NAME пуст — нет пула для public-net API")
return f"https://{pool}.cloud.api.selcloud.ru/public-net"
def keystone_token(conn) -> str:
sess = getattr(conn, "session", None)
if sess is None:
raise CloudError("нет Keystone session — не вызвать public-net API")
token = sess.get_token()
if not token:
authorize = getattr(conn, "authorize", None)
if callable(authorize):
authorize()
token = sess.get_token()
if not token:
raise CloudError("Keystone не дал токен для public-net API")
return str(token)
def _project_id(conn) -> str | None:
pid = getattr(conn, "current_project_id", None)
if pid:
return str(pid)
sess = getattr(conn, "session", None)
get = getattr(sess, "get_project_id", None) if sess is not None else None
if callable(get):
got = get()
return str(got) if got else None
return None
@dataclass
class PublicPort:
id: str
ip_address: str
network_id: str = ""
gateway: str | None = None
def _parse_port(raw: dict[str, Any]) -> PublicPort:
pid = str(raw.get("id") or "")
ip = str(raw.get("ip_address") or raw.get("ipAddress") or "")
if not pid or not ip:
raise CloudError(f"public-net: неполный port в ответе: {sorted(raw)}")
return PublicPort(
id=pid,
ip_address=ip,
network_id=str(raw.get("network_id") or raw.get("networkId") or ""),
gateway=str(raw.get("gateway") or "") or None,
)
def _raise_http(what: str, response: httpx.Response) -> None:
body = (response.text or "")[:300].replace("\n", " ")
raise CloudError(f"{what}: HTTP {response.status_code} {body}".strip())
def public_net_request(
conn,
region: str,
method: str,
path: str,
*,
json: dict[str, Any] | None = None,
timeout: float = 30.0,
) -> httpx.Response:
url = public_net_base(region).rstrip("/") + path
headers = {
"X-Auth-Token": keystone_token(conn),
"Accept": "application/json",
}
return httpx.request(method, url, headers=headers, json=json, timeout=timeout)
def get_public_port(conn, region: str, port_id: str) -> PublicPort | None:
response = public_net_request(conn, region, "GET", f"/v1/public_ports/{port_id}")
if response.status_code == 404:
return None
if response.status_code >= 400:
_raise_http("public-net GET port", response)
data = response.json()
raw = data.get("port") if isinstance(data, dict) else None
if not isinstance(raw, dict):
raise CloudError("public-net GET: нет объекта port")
return _parse_port(raw)
def create_public_port(
conn,
region: str,
*,
security_group_ids: list[str] | None = None,
description: str = PORT_DESCRIPTION,
) -> PublicPort:
body: dict[str, Any] = {
"description": description,
"admin_state_up": True,
}
if security_group_ids:
body["security_group_ids"] = security_group_ids
pid = _project_id(conn)
if pid:
body["project_id"] = pid
response = public_net_request(conn, region, "POST", "/v1/public_ports", json=body)
if response.status_code not in {200, 201}:
_raise_http("public-net создать прямой IP", response)
data = response.json()
raw = data.get("port") if isinstance(data, dict) else None
if not isinstance(raw, dict):
raise CloudError("public-net create: нет объекта port")
return _parse_port(raw)
def delete_public_port(conn, region: str, port_id: str, log: Log) -> None:
"""Detach-safe: retry 409 until Nova dropped the NIC (or 404)."""
path = f"/v1/public_ports/{port_id}"
deadline = time.time() + 90
last = 0
while time.time() < deadline:
response = public_net_request(conn, region, "DELETE", path)
last = response.status_code
if last in {204, 200, 404}:
if last != 404:
log(f"удалён прямой публичный IP {port_id}")
return
if last == 409:
time.sleep(2)
continue
_raise_http("public-net удалить прямой IP", response)
raise CloudError(f"public-net DELETE {port_id} всё ещё {last} (порт занят?)")
def is_rfc1918(addr: str) -> bool:
try:
ip = ipaddress.ip_address(addr)
except ValueError:
return False
return bool(ip.version == 4 and ip.is_private)
def server_public_ip(server: Any) -> str | None:
"""Floating IP or fixed public IPv4 (direct public port)."""
addrs = getattr(server, "addresses", None) or {}
if not isinstance(addrs, dict):
return None
floating: str | None = None
fixed_public: str | None = None
for nets in addrs.values():
if not isinstance(nets, list):
continue
for item in nets:
if not isinstance(item, dict):
continue
addr = str(item.get("addr") or "")
if not addr:
continue
if item.get("OS-EXT-IPS:type") == "floating":
floating = addr
elif not is_rfc1918(addr) and ":" not in addr:
fixed_public = addr
return floating or fixed_public
def neutron_port_attached(conn, port_id: str) -> bool:
try:
port = conn.network.get_port(port_id)
except Exception:
return False
device = getattr(port, "device_id", None) or ""
return bool(str(device).strip())
def allocate_public_port(
conn,
region: str,
*,
sg_id: str | None,
existing_id: str | None,
log: Log,
reuse: bool,
) -> PublicPort:
"""Create or reuse a direct public IP port. Not a Neutron floating IP."""
if reuse and existing_id:
try:
port = get_public_port(conn, region, existing_id)
except CloudError as exc:
log(f"старый прямой IP {existing_id}: {exc}")
port = None
if port and not neutron_port_attached(conn, port.id):
log(f"прямой публичный IP {port.ip_address} (reuse)")
return port
if port and neutron_port_attached(conn, port.id):
log(f"прямой IP {port.ip_address} уже на NIC — используем")
return port
sgs = [sg_id] if sg_id else None
port = create_public_port(conn, region, security_group_ids=sgs)
log(f"прямой публичный IP {port.ip_address}")
return port
def wait_server_public_ip(conn, server, log: Log, *, timeout: float = 180.0) -> str:
ip = server_public_ip(server)
if ip:
return ip
deadline = time.time() + timeout
sid = getattr(server, "id", server)
while time.time() < deadline:
try:
fresh = conn.compute.get_server(sid)
except Exception:
fresh = server
ip = server_public_ip(fresh)
if ip:
return ip
time.sleep(4)
raise CloudError("сервер ACTIVE, но публичный IPv4 не появился (прямой IP / FIP)")
def drop_legacy_private_network(conn, log: Log) -> None:
"""Remove leftover gpu-rent router+net from the FIP era (200 ₽/мес)."""
router = None
try:
router = conn.network.find_router(ROUTER_NAME)
except Exception:
router = None
if router:
try:
for port in conn.network.ports(device_id=router.id):
try:
conn.network.remove_interface_from_router(router, port_id=port.id)
except Exception as exc:
log(f"router interface {getattr(port, 'id', '?')}: {exc}")
conn.network.delete_router(router, ignore_missing=True)
log("удалён legacy-роутер gpu-rent (больше не нужен, 200 ₽/мес)")
except Exception as exc:
log(f"legacy-роутер gpu-rent не удалился: {exc}")
net = None
try:
net = conn.network.find_network(NET_NAME)
except Exception:
net = None
if not net:
return
try:
leftover = []
for port in conn.network.ports(network_id=net.id):
owner = getattr(port, "device_owner", "") or ""
if owner.startswith("network:"):
continue
leftover.append(port)
if leftover:
log(f"сеть {NET_NAME} ещё с {len(leftover)} портом(ами) — не трогаем")
return
for subnet in conn.network.subnets(network_id=net.id):
try:
conn.network.delete_subnet(subnet, ignore_missing=True)
except Exception:
pass
conn.network.delete_network(net, ignore_missing=True)
log(f"удалена legacy-сеть {NET_NAME} / {SUBNET_NAME}")
except Exception as exc:
log(f"legacy-сеть {NET_NAME} не удалилась: {exc}")
+40 -13
View File
@@ -6,13 +6,12 @@ from collections.abc import Callable
from gpu_rent.cloud import ( from gpu_rent.cloud import (
create_gpu_server, create_gpu_server,
delete_floating_ip, delete_access_ip,
delete_server, delete_server,
ensure_boot_volume, ensure_boot_volume,
ensure_data_volume, ensure_data_volume,
ensure_floating_ip, ensure_floating_ip,
ensure_keypair, ensure_keypair,
ensure_network,
ensure_security_group, ensure_security_group,
guess_operator_cidr, guess_operator_cidr,
pick_existing_server, pick_existing_server,
@@ -20,6 +19,11 @@ from gpu_rent.cloud import (
unshelve, unshelve,
wait_volume, wait_volume,
) )
from gpu_rent.public_net import (
allocate_public_port,
drop_legacy_private_network,
wait_server_public_ip,
)
from gpu_rent.bootstrap import run_bootstrap from gpu_rent.bootstrap import run_bootstrap
from gpu_rent.provision import ( from gpu_rent.provision import (
ensure_data_binds, ensure_data_binds,
@@ -94,13 +98,19 @@ def _bind_access(
phases: PhaseTimes | None = None, phases: PhaseTimes | None = None,
) -> SessionState: ) -> SessionState:
clock = phases or PhaseTimes() clock = phases or PhaseTimes()
ip, fip_id = ensure_floating_ip( try:
conn, server, state.floating_ip_id, state.floating_ip, log ip = wait_server_public_ip(conn, server, log)
) state.floating_ip = ip
state.floating_ip = ip save_state(state)
if fip_id: except CloudError:
state.floating_ip_id = fip_id ip, fip_id = ensure_floating_ip(
save_state(state) conn, server, state.floating_ip_id, state.floating_ip, log
)
state.floating_ip = ip
if fip_id:
state.floating_ip_id = fip_id
save_state(state)
log("legacy FIP — следующий stop снимет, up поднимет прямой IP")
wait_ssh(cfg, ip, log=log) wait_ssh(cfg, ip, log=log)
clock.mark("SSH", log) clock.mark("SSH", log)
log(f"SSH {cfg.ssh_user}@{ip}") log(f"SSH {cfg.ssh_user}@{ip}")
@@ -560,7 +570,6 @@ def cmd_up(
public_key = pub.read_text(encoding="utf-8") public_key = pub.read_text(encoding="utf-8")
ensure_keypair(conn, public_key, log) ensure_keypair(conn, public_key, log)
net, _subnet = ensure_network(conn, log)
cidr = guess_operator_cidr() cidr = guess_operator_cidr()
if cidr == "0.0.0.0/0": if cidr == "0.0.0.0/0":
import os import os
@@ -570,6 +579,18 @@ def cmd_up(
else: else:
log("не удалось узнать твой IP — SG откроет SSH с 0.0.0.0/0") log("не удалось узнать твой IP — SG откроет SSH с 0.0.0.0/0")
sg = ensure_security_group(conn, cidr, log) sg = ensure_security_group(conn, cidr, log)
drop_legacy_private_network(conn, log)
pub = allocate_public_port(
conn,
cfg.os_region_name,
sg_id=sg.id,
existing_id=state.floating_ip_id if cfg.keep_floating_ip else None,
log=log,
reuse=cfg.keep_floating_ip,
)
state.floating_ip = pub.ip_address
state.floating_ip_id = pub.id
state.network_id = pub.network_id or None
boot = ensure_boot_volume( boot = ensure_boot_volume(
conn, conn,
@@ -595,7 +616,6 @@ def cmd_up(
state.boot_volume_id = boot.id state.boot_volume_id = boot.id
state.data_volume_id = data.id state.data_volume_id = data.id
state.image_id = image.id state.image_id = image.id
state.network_id = net.id
state.security_group_id = sg.id state.security_group_id = sg.id
state.availability_zone = cfg.gpu_rent_az state.availability_zone = cfg.gpu_rent_az
state.spot = spot state.spot = spot
@@ -605,7 +625,7 @@ def cmd_up(
server = create_gpu_server( server = create_gpu_server(
conn, conn,
flavor_id=picked.id, flavor_id=picked.id,
net_id=net.id, port_id=pub.id,
sg_name=SG_NAME, sg_name=SG_NAME,
boot_volume_id=boot.id, boot_volume_id=boot.id,
data_volume_id=data.id, data_volume_id=data.id,
@@ -663,9 +683,16 @@ def cmd_stop(
log(f"revoke idle-killer app cred: {exc}") log(f"revoke idle-killer app cred: {exc}")
if not cfg.keep_floating_ip: if not cfg.keep_floating_ip:
delete_floating_ip(conn, state.floating_ip_id, state.floating_ip, log) delete_access_ip(
conn,
cfg.os_region_name,
state.floating_ip_id,
state.floating_ip,
log,
)
state.floating_ip = None state.floating_ip = None
state.floating_ip_id = None state.floating_ip_id = None
drop_legacy_private_network(conn, log)
if destroy_disks: if destroy_disks:
for vid in (state.data_volume_id, state.boot_volume_id): for vid in (state.data_volume_id, state.boot_volume_id):
+11 -6
View File
@@ -18,6 +18,7 @@ from gpu_rent.cloud import (
server_status, server_status,
unshelve, unshelve,
) )
from gpu_rent.public_net import wait_server_public_ip
from gpu_rent.config import Config from gpu_rent.config import Config
from gpu_rent.errors import CloudError, GpuRentError from gpu_rent.errors import CloudError, GpuRentError
from gpu_rent.llm_runtime import normalize_runtime from gpu_rent.llm_runtime import normalize_runtime
@@ -207,12 +208,16 @@ def _recover_unshelve(cfg: Config, log: Log) -> str:
raise CloudError(f"после preempt статус {status} — не unshelve") raise CloudError(f"после preempt статус {status} — не unshelve")
state = load_state() state = load_state()
ip, fip_id = ensure_floating_ip( try:
conn, server, state.floating_ip_id, state.floating_ip, log ip = wait_server_public_ip(conn, server, log)
) state.floating_ip = ip
state.floating_ip = ip except CloudError:
if fip_id: ip, fip_id = ensure_floating_ip(
state.floating_ip_id = fip_id conn, server, state.floating_ip_id, state.floating_ip, log
)
state.floating_ip = ip
if fip_id:
state.floating_ip_id = fip_id
state.server_id = server.id state.server_id = server.id
state.unshelved_at = utc_now() state.unshelved_at = utc_now()
state.phase = "ready_tunneled" state.phase = "ready_tunneled"
+2 -2
View File
@@ -144,8 +144,8 @@ def test_cmd_stop_deletes_compute_keeps_disks(monkeypatch):
lambda conn, server, log: deleted.append(server.id), lambda conn, server, log: deleted.append(server.id),
) )
monkeypatch.setattr( monkeypatch.setattr(
"gpu_rent.session.delete_floating_ip", "gpu_rent.session.delete_access_ip",
lambda conn, fip_id, address, log: deleted.append("fip"), lambda conn, region, fip_id, address, log: deleted.append("fip"),
) )
save_state( save_state(
SessionState( SessionState(