Add package data for GPU rent and update CLI documentation
- Added package data configuration for the 'gpu_rent' package in pyproject.toml. - Updated README.md to include usage instructions for Windows and Unix launchers. - Enhanced CLI documentation in cli.md to reflect new commands and their functionalities. - Revised setup.md to clarify installation steps and environment setup. - Improved error handling and command descriptions in the CLI implementation. - Added new functions for model version handling and flavor resolution in the codebase. - Updated state management to include additional properties for better tracking.
This commit is contained in:
Leonid Pershin
@@ -0,0 +1,282 @@
|
||||
"""up / stop / destroy: GPU lifetime; SwarmUI bootstrap after SSH."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
from gpu_rent.cloud import (
|
||||
create_gpu_server,
|
||||
delete_floating_ip,
|
||||
delete_server,
|
||||
ensure_boot_volume,
|
||||
ensure_data_volume,
|
||||
ensure_floating_ip,
|
||||
ensure_keypair,
|
||||
ensure_network,
|
||||
ensure_security_group,
|
||||
guess_operator_cidr,
|
||||
pick_existing_server,
|
||||
server_status,
|
||||
unshelve,
|
||||
)
|
||||
from gpu_rent.bootstrap import run_bootstrap
|
||||
from gpu_rent.provision import provision_vm
|
||||
from gpu_rent.config import Config
|
||||
from gpu_rent.errors import CloudError, GpuRentError
|
||||
from gpu_rent.inventory import (
|
||||
gpu_quota_from_compute,
|
||||
pick_boot_image,
|
||||
pick_volume_type,
|
||||
resolve_flavor,
|
||||
)
|
||||
from gpu_rent.lock import SessionLock
|
||||
from gpu_rent.os_client import (
|
||||
KEYPAIR_NAME,
|
||||
SG_NAME,
|
||||
compute_quotas,
|
||||
connect,
|
||||
iter_flavors,
|
||||
iter_images,
|
||||
iter_volume_types,
|
||||
)
|
||||
from gpu_rent.ssh_keys import ensure_ed25519
|
||||
from gpu_rent.ssh_ops import wait_ssh
|
||||
from gpu_rent.state import SessionState, load_state, save_state, utc_now
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
|
||||
def _log_default(msg: str) -> None:
|
||||
print(msg)
|
||||
|
||||
|
||||
def _require_gpu_quota(conn) -> None:
|
||||
quota = compute_quotas(conn)
|
||||
limit = gpu_quota_from_compute(quota)
|
||||
if limit is not None and limit <= 0:
|
||||
raise CloudError(
|
||||
"квота GPU = 0. Напиши в поддержку Selectel (текст в docs/setup.md). "
|
||||
"CLI не создаст сервер."
|
||||
)
|
||||
|
||||
|
||||
def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> SessionState:
|
||||
ip, fip_id = ensure_floating_ip(
|
||||
conn, server, state.floating_ip_id, state.floating_ip, log
|
||||
)
|
||||
state.floating_ip = ip
|
||||
if fip_id:
|
||||
state.floating_ip_id = fip_id
|
||||
save_state(state)
|
||||
wait_ssh(cfg, ip)
|
||||
log(f"SSH {cfg.ssh_user}@{ip}")
|
||||
run_bootstrap(cfg, ip, log)
|
||||
provision_vm(cfg, ip, log)
|
||||
state.bootstrapped = True
|
||||
save_state(state)
|
||||
log("gpu-rent tunnel — UI на localhost:17801")
|
||||
log("gpu-rent stop — удалить compute, диски оставить")
|
||||
return state
|
||||
|
||||
|
||||
def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState:
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
raise CloudError("нечего adopt: нет сервера с тегом/именем gpu-rent")
|
||||
state = load_state()
|
||||
state.server_id = server.id
|
||||
state.server_name = getattr(server, "name", None)
|
||||
state.flavor_id = getattr(server, "flavor_id", None) or (
|
||||
(server.flavor or {}).get("id") if isinstance(getattr(server, "flavor", None), dict) else None
|
||||
)
|
||||
state.phase = "ready_cloud"
|
||||
state.availability_zone = cfg.gpu_rent_az
|
||||
save_state(state)
|
||||
log(f"подхватили {server.id} статус {server_status(server)}")
|
||||
if server_status(server) == "ACTIVE":
|
||||
_bind_access(conn, server, state, cfg, log)
|
||||
return state
|
||||
|
||||
|
||||
def cmd_up(
|
||||
cfg: Config,
|
||||
*,
|
||||
no_spot: bool = False,
|
||||
flavor: str | None = None,
|
||||
yes: bool = False,
|
||||
adopt: bool = False,
|
||||
confirm: Callable[[str], bool] | None = None,
|
||||
log: Log = _log_default,
|
||||
) -> SessionState:
|
||||
with SessionLock():
|
||||
if adopt:
|
||||
return adopt_server(cfg, log=log)
|
||||
conn = connect(cfg)
|
||||
_require_gpu_quota(conn)
|
||||
state = load_state()
|
||||
existing = pick_existing_server(conn)
|
||||
if existing:
|
||||
status = server_status(existing)
|
||||
state.server_id = existing.id
|
||||
state.server_name = getattr(existing, "name", None)
|
||||
if status == "ACTIVE":
|
||||
log("сервер уже ACTIVE — второй GPU не создаём")
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
return state
|
||||
if status in {"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"}:
|
||||
existing = unshelve(conn, existing, log)
|
||||
state.server_id = existing.id
|
||||
state.phase = "ready_cloud"
|
||||
state.unshelved_at = utc_now()
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
return state
|
||||
raise CloudError(f"сервер gpu-rent в статусе {status} — разбери в панели")
|
||||
|
||||
flavors = list(iter_flavors(conn))
|
||||
try:
|
||||
picked = resolve_flavor(
|
||||
flavors,
|
||||
cfg.flavor_preference,
|
||||
explicit=flavor,
|
||||
default_id=cfg.default_flavor_id or None,
|
||||
fallback=cfg.flavor_fallback,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise CloudError(str(exc)) from exc
|
||||
|
||||
image = pick_boot_image(list(iter_images(conn)))
|
||||
if not image:
|
||||
raise CloudError("нет GPU-образа Ubuntu Driver 580/535 в Glance")
|
||||
vtype = pick_volume_type(list(iter_volume_types(conn)), cfg.gpu_rent_az)
|
||||
spot = cfg.default_spot and not no_spot
|
||||
|
||||
prompt = (
|
||||
f"Создать {'preemptible ' if spot else ''}GPU {picked.name} "
|
||||
f"в {cfg.gpu_rent_az}, образ {getattr(image, 'name', image.id)}, "
|
||||
f"data {cfg.data_volume_size_gb} GB. Диск тарифицируется всегда. Продолжить?"
|
||||
)
|
||||
if not yes:
|
||||
ok = confirm(prompt) if confirm else False
|
||||
if not ok:
|
||||
raise GpuRentError("отменено")
|
||||
|
||||
private_path, pub = ensure_ed25519(cfg.ssh_private_key_path)
|
||||
del private_path
|
||||
public_key = pub.read_text(encoding="utf-8")
|
||||
ensure_keypair(conn, public_key, log)
|
||||
|
||||
net, _subnet = ensure_network(conn, log)
|
||||
cidr = guess_operator_cidr()
|
||||
if cidr == "0.0.0.0/0":
|
||||
log("не удалось узнать твой IP — SG откроет SSH с 0.0.0.0/0")
|
||||
sg = ensure_security_group(conn, cidr, log)
|
||||
|
||||
boot = ensure_boot_volume(
|
||||
conn,
|
||||
az=cfg.gpu_rent_az,
|
||||
volume_type=vtype,
|
||||
image_id=image.id,
|
||||
snapshot_name=cfg.boot_snapshot_name,
|
||||
existing_id=state.boot_volume_id or cfg.boot_volume_id,
|
||||
log=log,
|
||||
)
|
||||
data = ensure_data_volume(
|
||||
conn,
|
||||
az=cfg.gpu_rent_az,
|
||||
volume_type=vtype,
|
||||
size_gb=cfg.data_volume_size_gb,
|
||||
existing_id=state.data_volume_id or cfg.data_volume_id,
|
||||
log=log,
|
||||
)
|
||||
|
||||
state.phase = "provisioning"
|
||||
state.flavor_id = picked.id
|
||||
state.flavor_name = picked.name
|
||||
state.boot_volume_id = boot.id
|
||||
state.data_volume_id = data.id
|
||||
state.image_id = image.id
|
||||
state.network_id = net.id
|
||||
state.security_group_id = sg.id
|
||||
state.availability_zone = cfg.gpu_rent_az
|
||||
state.spot = spot
|
||||
state.keypair_name = KEYPAIR_NAME
|
||||
save_state(state)
|
||||
|
||||
server = create_gpu_server(
|
||||
conn,
|
||||
flavor_id=picked.id,
|
||||
net_id=net.id,
|
||||
sg_name=SG_NAME,
|
||||
boot_volume_id=boot.id,
|
||||
data_volume_id=data.id,
|
||||
az=cfg.gpu_rent_az,
|
||||
spot=spot,
|
||||
log=log,
|
||||
)
|
||||
state.server_id = server.id
|
||||
state.server_name = getattr(server, "name", None)
|
||||
state.created_at = utc_now()
|
||||
state.unshelved_at = None
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
_bind_access(conn, server, state, cfg, log)
|
||||
return state
|
||||
|
||||
|
||||
def cmd_stop(
|
||||
cfg: Config,
|
||||
*,
|
||||
destroy_disks: bool = False,
|
||||
no_pull: bool = False,
|
||||
log: Log = _log_default,
|
||||
) -> SessionState:
|
||||
with SessionLock():
|
||||
conn = connect(cfg)
|
||||
state = load_state()
|
||||
if cfg.pull_output and not no_pull and state.floating_ip:
|
||||
try:
|
||||
from gpu_rent.sync_files import pull_tree
|
||||
|
||||
pull_tree(cfg, state.floating_ip, "/mnt/swarm_data/Output", cfg.local_output_dir, log)
|
||||
except Exception as exc:
|
||||
log(f"pull Output не удался: {exc}")
|
||||
server = None
|
||||
if state.server_id:
|
||||
try:
|
||||
server = conn.compute.get_server(state.server_id)
|
||||
except Exception:
|
||||
server = None
|
||||
if server is None:
|
||||
server = pick_existing_server(conn)
|
||||
if server:
|
||||
delete_server(conn, server, log)
|
||||
else:
|
||||
log("compute уже нет")
|
||||
|
||||
if not cfg.keep_floating_ip:
|
||||
delete_floating_ip(conn, state.floating_ip_id, state.floating_ip, log)
|
||||
state.floating_ip = None
|
||||
state.floating_ip_id = None
|
||||
|
||||
if destroy_disks:
|
||||
for vid in (state.data_volume_id, state.boot_volume_id):
|
||||
if not vid:
|
||||
continue
|
||||
try:
|
||||
conn.block_storage.delete_volume(vid, ignore_missing=True)
|
||||
log(f"удалён том {vid}")
|
||||
except Exception as exc:
|
||||
log(f"том {vid} не удалился: {exc}")
|
||||
state.boot_volume_id = None
|
||||
state.data_volume_id = None
|
||||
|
||||
state.server_id = None
|
||||
state.server_name = None
|
||||
state.phase = "idle"
|
||||
save_state(state)
|
||||
log("фаза idle" + ("" if destroy_disks else " (диски на месте)"))
|
||||
return state
|
||||
Reference in New Issue
Block a user