Add package data for GPU rent and update CLI documentation
- Added package data configuration for the 'gpu_rent' package in pyproject.toml. - Updated README.md to include usage instructions for Windows and Unix launchers. - Enhanced CLI documentation in cli.md to reflect new commands and their functionalities. - Revised setup.md to clarify installation steps and environment setup. - Improved error handling and command descriptions in the CLI implementation. - Added new functions for model version handling and flavor resolution in the codebase. - Updated state management to include additional properties for better tracking.
This commit is contained in:
Leonid Pershin
@@ -0,0 +1,450 @@
|
||||
"""Find-or-create Selectel OpenStack resources for one gpu-rent session."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any, Callable
|
||||
|
||||
import httpx
|
||||
|
||||
from gpu_rent.errors import CloudError
|
||||
from gpu_rent.os_client import (
|
||||
BOOT_VOLUME_NAME,
|
||||
BOOT_VOLUME_SIZE_GB,
|
||||
DATA_VOLUME_NAME,
|
||||
KEYPAIR_NAME,
|
||||
NET_NAME,
|
||||
PREEMPTIBLE_TAG,
|
||||
RESOURCE_TAG,
|
||||
ROUTER_NAME,
|
||||
SERVER_NAME,
|
||||
SG_NAME,
|
||||
SUBNET_CIDR,
|
||||
SUBNET_NAME,
|
||||
find_snapshot_by_name,
|
||||
find_tagged_servers,
|
||||
find_volumes_by_name,
|
||||
)
|
||||
|
||||
|
||||
def _wrap(exc: BaseException, what: str) -> CloudError:
|
||||
text = str(exc)
|
||||
low = text.lower()
|
||||
if "402" in text or "payment" in low:
|
||||
return CloudError(f"{what}: нет средств / 402. {text}")
|
||||
if "403" in text or "forbidden" in low or "quota" in low:
|
||||
return CloudError(
|
||||
f"{what}: квота или запрет (403). Часто GPU=0 — тикет в поддержку, docs/setup.md. {text}"
|
||||
)
|
||||
return CloudError(f"{what}: {text}")
|
||||
|
||||
|
||||
def _oid(obj: Any) -> str:
|
||||
return str(getattr(obj, "id", "") or "")
|
||||
|
||||
|
||||
def guess_operator_cidr() -> str:
|
||||
try:
|
||||
response = httpx.get("https://ifconfig.me/ip", timeout=5.0)
|
||||
ip = response.text.strip()
|
||||
if ip.count(".") == 3 and all(p.isdigit() for p in ip.split(".")):
|
||||
return f"{ip}/32"
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
return "0.0.0.0/0"
|
||||
|
||||
|
||||
def wait_volume(conn, volume, status: str = "available", timeout: int = 900) -> Any:
|
||||
deadline = time.time() + timeout
|
||||
vid = _oid(volume)
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
current = conn.block_storage.get_volume(vid)
|
||||
last = getattr(current, "status", None)
|
||||
if last == status:
|
||||
return current
|
||||
if last in {"error", "error_restoring"}:
|
||||
raise CloudError(f"том {vid} статус {last}")
|
||||
time.sleep(5)
|
||||
raise CloudError(f"том {vid} не стал {status} (последний {last})")
|
||||
|
||||
|
||||
def wait_server(conn, server, status: str = "ACTIVE", timeout: int = 900) -> Any:
|
||||
deadline = time.time() + timeout
|
||||
sid = _oid(server)
|
||||
last = None
|
||||
while time.time() < deadline:
|
||||
current = conn.compute.get_server(sid)
|
||||
last = (getattr(current, "status", None) or "").upper()
|
||||
if last == status.upper():
|
||||
return current
|
||||
if last in {"ERROR"}:
|
||||
fault = getattr(current, "fault", None)
|
||||
raise CloudError(f"сервер {sid} ERROR{f' {fault}' if fault else ''}")
|
||||
time.sleep(5)
|
||||
raise CloudError(f"сервер {sid} не стал {status} (последний {last})")
|
||||
|
||||
|
||||
def wait_gone(fetch: Callable[[], Any], timeout: int = 300) -> None:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
obj = fetch()
|
||||
except Exception:
|
||||
return
|
||||
if obj is None:
|
||||
return
|
||||
time.sleep(4)
|
||||
raise CloudError("ресурс не исчез вовремя")
|
||||
|
||||
|
||||
def find_external_network(conn) -> Any:
|
||||
for net in conn.network.networks():
|
||||
if getattr(net, "is_router_external", False):
|
||||
return net
|
||||
for net in conn.network.networks():
|
||||
name = (getattr(net, "name", "") or "").lower()
|
||||
if name in {"external-network", "wan", "public"}:
|
||||
return net
|
||||
raise CloudError("нет внешней сети для floating IP / router gateway")
|
||||
|
||||
|
||||
def ensure_network(conn, log: Callable[[str], None]) -> tuple[Any, Any]:
|
||||
net = conn.network.find_network(NET_NAME)
|
||||
if net:
|
||||
subnet = next(iter(conn.network.subnets(network_id=net.id)), None)
|
||||
if not subnet:
|
||||
raise CloudError(f"сеть {NET_NAME} есть, subnet нет")
|
||||
log(f"сеть {NET_NAME} уже есть")
|
||||
return net, subnet
|
||||
try:
|
||||
net = conn.network.create_network(name=NET_NAME)
|
||||
subnet = conn.network.create_subnet(
|
||||
name=SUBNET_NAME,
|
||||
network_id=net.id,
|
||||
ip_version=4,
|
||||
cidr=SUBNET_CIDR,
|
||||
dns_nameservers=["1.1.1.1", "8.8.8.8"],
|
||||
)
|
||||
ext = find_external_network(conn)
|
||||
router = conn.network.find_router(ROUTER_NAME)
|
||||
if not router:
|
||||
router = conn.network.create_router(
|
||||
name=ROUTER_NAME,
|
||||
external_gateway_info={"network_id": ext.id},
|
||||
)
|
||||
conn.network.add_interface_to_router(router, subnet_id=subnet.id)
|
||||
log(f"создана сеть {NET_NAME} + router")
|
||||
return net, subnet
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "сеть") from exc
|
||||
|
||||
|
||||
def ensure_security_group(conn, cidr: str, log: Callable[[str], None]) -> Any:
|
||||
sg = conn.network.find_security_group(SG_NAME)
|
||||
if sg:
|
||||
log(f"security group {SG_NAME} уже есть")
|
||||
return sg
|
||||
try:
|
||||
sg = conn.network.create_security_group(
|
||||
name=SG_NAME,
|
||||
description="gpu-rent SSH only",
|
||||
)
|
||||
conn.network.create_security_group_rule(
|
||||
security_group_id=sg.id,
|
||||
direction="ingress",
|
||||
ethertype="IPv4",
|
||||
protocol="tcp",
|
||||
port_range_min=22,
|
||||
port_range_max=22,
|
||||
remote_ip_prefix=cidr,
|
||||
)
|
||||
log(f"SG {SG_NAME}: TCP/22 с {cidr}")
|
||||
return sg
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "security group") from exc
|
||||
|
||||
|
||||
def ensure_keypair(conn, public_key: str, log: Callable[[str], None]) -> Any:
|
||||
existing = conn.compute.find_keypair(KEYPAIR_NAME)
|
||||
pub = public_key.strip()
|
||||
if existing:
|
||||
have = (getattr(existing, "public_key", "") or "").strip()
|
||||
if have == pub:
|
||||
log(f"keypair {KEYPAIR_NAME} совпадает")
|
||||
return existing
|
||||
log(f"keypair {KEYPAIR_NAME} другой — пересоздаём")
|
||||
try:
|
||||
conn.compute.delete_keypair(existing)
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "удалить keypair") from exc
|
||||
try:
|
||||
key = conn.compute.create_keypair(name=KEYPAIR_NAME, public_key=pub)
|
||||
log(f"keypair {KEYPAIR_NAME} создан")
|
||||
return key
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "keypair") from exc
|
||||
|
||||
|
||||
def ensure_boot_volume(
|
||||
conn,
|
||||
*,
|
||||
az: str,
|
||||
volume_type: str | None,
|
||||
image_id: str | None,
|
||||
snapshot_name: str,
|
||||
existing_id: str,
|
||||
log: Callable[[str], None],
|
||||
) -> Any:
|
||||
if existing_id:
|
||||
vol = conn.block_storage.get_volume(existing_id)
|
||||
log(f"boot volume из state {existing_id}")
|
||||
return vol
|
||||
found = find_volumes_by_name(conn, BOOT_VOLUME_NAME)
|
||||
if found:
|
||||
log(f"boot volume {BOOT_VOLUME_NAME} уже в проекте")
|
||||
return found[0]
|
||||
snap = find_snapshot_by_name(conn, snapshot_name)
|
||||
kwargs: dict[str, Any] = {
|
||||
"name": BOOT_VOLUME_NAME,
|
||||
"size": BOOT_VOLUME_SIZE_GB,
|
||||
"availability_zone": az,
|
||||
}
|
||||
if volume_type:
|
||||
kwargs["volume_type"] = volume_type
|
||||
if snap:
|
||||
kwargs["snapshot_id"] = snap.id
|
||||
log(f"boot volume из snapshot {snapshot_name}")
|
||||
elif image_id:
|
||||
kwargs["image_id"] = image_id
|
||||
log("boot volume из GPU-образа")
|
||||
else:
|
||||
raise CloudError("нет ни snapshot, ни image_id для boot volume")
|
||||
try:
|
||||
vol = conn.block_storage.create_volume(**kwargs)
|
||||
return wait_volume(conn, vol)
|
||||
except TypeError:
|
||||
if "image_id" in kwargs:
|
||||
kwargs["imageRef"] = kwargs.pop("image_id")
|
||||
try:
|
||||
vol = conn.block_storage.create_volume(**kwargs)
|
||||
return wait_volume(conn, vol)
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "boot volume") from exc
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "boot volume") from exc
|
||||
|
||||
|
||||
def ensure_data_volume(
|
||||
conn,
|
||||
*,
|
||||
az: str,
|
||||
volume_type: str | None,
|
||||
size_gb: int,
|
||||
existing_id: str,
|
||||
log: Callable[[str], None],
|
||||
) -> Any:
|
||||
if existing_id:
|
||||
vol = conn.block_storage.get_volume(existing_id)
|
||||
log(f"data volume из state {existing_id}")
|
||||
return vol
|
||||
found = find_volumes_by_name(conn, DATA_VOLUME_NAME)
|
||||
if found:
|
||||
log(f"data volume {DATA_VOLUME_NAME} уже в проекте")
|
||||
return found[0]
|
||||
kwargs: dict[str, Any] = {
|
||||
"name": DATA_VOLUME_NAME,
|
||||
"size": size_gb,
|
||||
"availability_zone": az,
|
||||
}
|
||||
if volume_type:
|
||||
kwargs["volume_type"] = volume_type
|
||||
try:
|
||||
vol = conn.block_storage.create_volume(**kwargs)
|
||||
log(f"data volume {size_gb} GB")
|
||||
return wait_volume(conn, vol)
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "data volume") from exc
|
||||
|
||||
|
||||
def _tag_server(conn, server, spot: bool) -> None:
|
||||
tags = [RESOURCE_TAG]
|
||||
if spot:
|
||||
tags.append(PREEMPTIBLE_TAG)
|
||||
try:
|
||||
conn.compute.set_server_tags(server, tags)
|
||||
except Exception:
|
||||
try:
|
||||
for tag in tags:
|
||||
conn.compute.add_tag_to_server(server, tag)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def create_gpu_server(
|
||||
conn,
|
||||
*,
|
||||
flavor_id: str,
|
||||
net_id: str,
|
||||
sg_name: str,
|
||||
boot_volume_id: str,
|
||||
data_volume_id: str,
|
||||
az: str,
|
||||
spot: bool,
|
||||
log: Callable[[str], None],
|
||||
) -> Any:
|
||||
bdm = [
|
||||
{
|
||||
"boot_index": 0,
|
||||
"uuid": boot_volume_id,
|
||||
"source_type": "volume",
|
||||
"destination_type": "volume",
|
||||
"delete_on_termination": False,
|
||||
},
|
||||
{
|
||||
"boot_index": 1,
|
||||
"uuid": data_volume_id,
|
||||
"source_type": "volume",
|
||||
"destination_type": "volume",
|
||||
"delete_on_termination": False,
|
||||
},
|
||||
]
|
||||
tags = [RESOURCE_TAG]
|
||||
if spot:
|
||||
tags.append(PREEMPTIBLE_TAG)
|
||||
kwargs: dict[str, Any] = {
|
||||
"name": SERVER_NAME,
|
||||
"flavor_id": flavor_id,
|
||||
"networks": [{"uuid": net_id}],
|
||||
"key_name": KEYPAIR_NAME,
|
||||
"availability_zone": az,
|
||||
"block_device_mapping_v2": bdm,
|
||||
"security_groups": [{"name": sg_name}],
|
||||
"tags": tags,
|
||||
}
|
||||
try:
|
||||
server = conn.compute.create_server(**kwargs)
|
||||
except Exception as exc:
|
||||
kwargs.pop("tags", None)
|
||||
try:
|
||||
server = conn.compute.create_server(**kwargs)
|
||||
except Exception as exc2:
|
||||
raise _wrap(exc2, "create server") from exc
|
||||
log("ждём Nova ACTIVE (GPU create может занять несколько минут)")
|
||||
server = wait_server(conn, server, "ACTIVE")
|
||||
_tag_server(conn, server, spot)
|
||||
return server
|
||||
|
||||
|
||||
def unshelve(conn, server, log: Callable[[str], None]) -> Any:
|
||||
log("unshelve (EXPIRED / shelved)")
|
||||
try:
|
||||
conn.compute.unshelve_server(server)
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "unshelve") from exc
|
||||
return wait_server(conn, server, "ACTIVE", timeout=900)
|
||||
|
||||
|
||||
def associate_floating_ip(conn, server, log: Callable[[str], None]) -> tuple[str, str]:
|
||||
ports = list(conn.network.ports(device_id=server.id))
|
||||
if not ports:
|
||||
raise CloudError("у сервера нет neutron-порта — не к чему привязать FIP")
|
||||
ext = find_external_network(conn)
|
||||
try:
|
||||
fip = conn.network.create_ip(floating_network_id=ext.id)
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "floating IP allocate") from exc
|
||||
try:
|
||||
fip = conn.network.update_ip(fip, port_id=ports[0].id)
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "floating IP associate") from exc
|
||||
addr = getattr(fip, "floating_ip_address", None) or getattr(fip, "name", None)
|
||||
log(f"floating IP {addr}")
|
||||
return str(addr), _oid(fip)
|
||||
|
||||
|
||||
def ensure_floating_ip(
|
||||
conn,
|
||||
server,
|
||||
existing_id: str | None,
|
||||
existing_addr: str | None,
|
||||
log: Callable[[str], None],
|
||||
) -> tuple[str, str]:
|
||||
have = server_floating_ip(server)
|
||||
if have:
|
||||
return have, existing_id or ""
|
||||
ports = list(conn.network.ports(device_id=server.id))
|
||||
if existing_id and ports:
|
||||
try:
|
||||
fip = conn.network.update_ip(existing_id, port_id=ports[0].id)
|
||||
addr = getattr(fip, "floating_ip_address", None) or existing_addr
|
||||
log(f"вернули FIP {addr}")
|
||||
return str(addr), existing_id
|
||||
except Exception:
|
||||
log("старый FIP не привязался — выделяем новый")
|
||||
return associate_floating_ip(conn, server, log)
|
||||
|
||||
|
||||
def delete_floating_ip(conn, fip_id: str | None, address: str | None, log: Callable[[str], None]) -> None:
|
||||
if fip_id:
|
||||
try:
|
||||
conn.network.delete_ip(fip_id, ignore_missing=True)
|
||||
log(f"удалён FIP {fip_id}")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
if address:
|
||||
for ip in conn.network.ips():
|
||||
if getattr(ip, "floating_ip_address", None) == address:
|
||||
conn.network.delete_ip(ip, ignore_missing=True)
|
||||
log(f"удалён FIP {address}")
|
||||
return
|
||||
|
||||
|
||||
def delete_server(conn, server, log: Callable[[str], None]) -> None:
|
||||
sid = _oid(server)
|
||||
log(f"удаляем compute {sid} (диски оставляем)")
|
||||
try:
|
||||
conn.compute.delete_server(server, ignore_missing=True)
|
||||
except Exception as exc:
|
||||
raise _wrap(exc, "delete server") from exc
|
||||
|
||||
def _get():
|
||||
try:
|
||||
current = conn.compute.get_server(sid)
|
||||
except Exception:
|
||||
return None
|
||||
if current is None:
|
||||
return None
|
||||
if server_status(current) in {"DELETED", "SOFT_DELETED"}:
|
||||
return None
|
||||
return current
|
||||
|
||||
wait_gone(_get, timeout=420)
|
||||
|
||||
|
||||
def server_status(server: Any) -> str:
|
||||
return (getattr(server, "status", None) or "").upper()
|
||||
|
||||
|
||||
def server_floating_ip(server: Any) -> str | None:
|
||||
addrs = getattr(server, "addresses", None) or {}
|
||||
if not isinstance(addrs, dict):
|
||||
return None
|
||||
for nets in addrs.values():
|
||||
if not isinstance(nets, list):
|
||||
continue
|
||||
for item in nets:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if item.get("OS-EXT-IPS:type") == "floating":
|
||||
return str(item.get("addr") or "") or None
|
||||
return None
|
||||
|
||||
|
||||
def pick_existing_server(conn) -> Any | None:
|
||||
found = find_tagged_servers(conn)
|
||||
if len(found) > 1:
|
||||
ids = ", ".join(_oid(s) for s in found)
|
||||
raise CloudError(f"несколько серверов gpu-rent: {ids}. Разбери вручную или --adopt один.")
|
||||
return found[0] if found else None
|
||||
Reference in New Issue
Block a user