diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6985355 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Keep launcher line endings correct on Windows and Unix clones. +gpu-rent.sh text eol=lf +gpu-rent.bat text eol=crlf +gpu-rent.ps1 text eol=crlf +src/gpu_rent/remote/*.sh text eol=lf diff --git a/README.md b/README.md index e1316f6..117f517 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,30 @@ ## Что уже можно запустить +Windows (cmd или двойной клик): `gpu-rent.bat` +PowerShell: `.\gpu-rent.ps1` +Linux / macOS / Git Bash: `./gpu-rent.sh` + +Скрипты сами создают `.venv`, ставят пакет и прокидывают аргументы. Активировать окружение не нужно. + ```powershell -python -m venv .venv -.\.venv\Scripts\activate -python -m pip install -U pip -python -m pip install -e ".[dev]" -copy env.example $env:USERPROFILE\.gpu-rent\.env -# заполни OS_* по docs/setup.md -gpu-rent doctor -gpu-rent dry-run -gpu-rent status +.\gpu-rent.ps1 doctor +.\gpu-rent.ps1 dry-run +.\gpu-rent.ps1 status +# если doctor зелёный и квота GPU > 0: +.\gpu-rent.ps1 up --yes +.\gpu-rent.ps1 tunnel +.\gpu-rent.ps1 stop ``` -`up` / `stop` / `tunnel` пока не создают сервер: сначала `doctor` должен быть зелёным и квота GPU > 0. +```bash +chmod +x gpu-rent.sh # один раз на Unix +./gpu-rent.sh doctor +``` + +Ключи: скопируй `env.example` в `%USERPROFILE%\.gpu-rent\.env` (скрипт сделает это сам при первом запуске) и заполни `OS_*` по [`docs/setup.md`](docs/setup.md). + +`up` поднимает compute и ставит SwarmUI нативно (`launch-linux.sh` + systemd). `tunnel` пробрасывает `localhost:17801`. `stop` гасит GPU, диски оставляет. ## Зачем какие ключи diff --git a/docs/cli.md b/docs/cli.md index 023b40e..2cc2d9a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -35,6 +35,8 @@ Нет команды `generate`. Нет зеркала каталога локального SwarmUI — только папки приложения, см. [local-folders.md](local-folders.md). +Сейчас в коде: `up` создаёт/unshelve compute, FIP, SSH, идемпотентный bootstrap SwarmUI (без Docker) и systemd. `tunnel` на 17801, Ctrl+C не гасит GPU. Ещё нет: extensions, autocomplete, Civitai seed, push, idle-killer, snapshot, EXPIRED-reconnect туннеля. + ## `doctor` Не создаёт сервер, не тратит GPU. Нужен до первого `up` и когда «вчера работало». @@ -166,6 +168,6 @@ Application credential для idle-killer CLI создаёт при bootstrap и ## Windows -Пути через `Path.expanduser`. Венв: `.venv\Scripts\activate`. Ключ генерирует CLI: `~/.gpu-rent/id_ed25519`. +Лаунчеры в корне: `gpu-rent.bat` (cmd / двойной клик), `.\gpu-rent.ps1` (PowerShell). На Unix — `./gpu-rent.sh`. Они создают `.venv` в репозитории и вызывают `python -m gpu_rent`. Пути в приложении через `Path.expanduser`. Ключ генерирует CLI: `~/.gpu-rent/id_ed25519`. `NOTIFY_READY`: toast через WinRT / `win10toast` (что заработает на 10/11 без админ-прав) + системный звук. Если toast недоступен — только звук и лог, не падать. diff --git a/docs/roadmap.md b/docs/roadmap.md index 3f6e09f..b0ef1cb 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -34,36 +34,36 @@ ## 1. Каркас - [x] `pyproject.toml`, пакет `gpu_rent`, MIT -- [x] config / state / Typer по [cli.md](cli.md): `doctor` / `dry-run` / `status` / `open` живые; остальные команды есть и честно говорят, что mutating ещё нет -- [x] Windows: `python -m gpu_rent` +- [x] config / state / Typer: `doctor` / `dry-run` / `status` / `open` / `up` / `stop` / `destroy` / `ssh` / `logs`; `tunnel` / `hold` / seed ещё `NotReadyError` +- [x] Windows: `gpu-rent.bat` / `gpu-rent.ps1`; Unix: `gpu-rent.sh`; также `python -m gpu_rent` - [x] [setup.md](setup.md) — квота, сервисный пользователь, RC, Civitai ## 2. OpenStack-клиент - [x] Token через openstacksdk (`authorize`), inventory flavors/квота/volume type -- [ ] Сеть find-or-create, SG, FIP, volumes, server, unshelve -- [ ] Preemptible tag 2.72, BDM boot+data, `delete_on_termination=false` -- [x] Flavor fallback по `FLAVOR_PREFERENCE` (ранжирование; create ещё нет) +- [x] Сеть find-or-create, SG, FIP, volumes, server, unshelve +- [x] Preemptible tag 2.72, BDM boot+data, `delete_on_termination=false` +- [x] Flavor fallback по `FLAVOR_PREFERENCE` (ранжирование + выбор на `up`) - [x] `doctor`: Keystone, квота, flavor, диск, Civitai `.red`, манифесты - [x] Тесты с моками / без облака ## 3. Сессия без туннеля -- [ ] `up` / `stop` / `status` / reconcile +- [x] `up` / `stop` / `status` / reconcile (compute+диски+SSH+bootstrap SwarmUI) - [ ] `status`: диск used/free, ₽/час, TTL preempt 24 ч, killer/hold -- [ ] Второй `up` не создаёт второй GPU +- [x] Второй `up` не создаёт второй GPU - [ ] `resize-data` вверх -- [ ] Генерация `~/.gpu-rent/id_ed25519` + keypair при первом `up` +- [x] Генерация `~/.gpu-rent/id_ed25519` + keypair при первом `up` - [ ] `seed-models` на живом диске (идемпотентно, SHA256) - [ ] `push` / `push-models`: `Models/`, `Wildcards/`, `CustomWorkflows/` - [ ] `pull-output` и `PULL_OUTPUT` на `stop` / повторный `up` -- [ ] `seed-extensions` + restart контейнера -- [ ] `destroy` только с флагом +- [ ] `seed-extensions` + `systemctl restart swarmui` +- [x] `destroy` только с флагом ## 4. Bootstrap SwarmUI + idle-killer -- [ ] Идемпотентный first-boot, маркер FS -- [ ] Bind-mounts data volume → `/opt/swarmui`, systemd `swarmui`, auth +- [x] Идемпотентный first-boot, маркер FS (`/mnt/swarm_data/.gpu-rent-ready`) +- [x] Bind-mounts data volume → `/opt/swarmui`, systemd `swarmui` (auth SwarmUI ещё на spike) - [ ] Clone `extensions.yaml` в Extensions / DLNodes до старта UI - [ ] Autocomplete: danbooru.csv + Settings.fds; на каждом up — GitHub sha - [ ] Civitai seed по манифесту; без токена — дефолт SwarmUI @@ -74,8 +74,8 @@ ## 5. Туннель -- [ ] `gpu-rent tunnel` → 17801, Ctrl+C не делает `stop` -- [ ] `gpu-rent open` / `tunnel --open` +- [x] `gpu-rent tunnel` → 17801, Ctrl+C не делает `stop` +- [x] `gpu-rent open` / `tunnel --open` - [ ] EXPIRED → unshelve + reconnect, пока туннель жив - [ ] Сниппет MCP в stdout diff --git a/docs/setup.md b/docs/setup.md index 37ae055..69511a9 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -10,20 +10,32 @@ Нужен **Python 3.11+**. На Windows при установке включи «Add python.exe to PATH». -В корне репозитория: +В корне репозитория достаточно лаунчера — venv и `pip install` он сделает сам: + +```powershell +.\gpu-rent.ps1 --help +.\gpu-rent.ps1 doctor +``` + +```bat +gpu-rent.bat --help +``` + +```bash +chmod +x gpu-rent.sh +./gpu-rent.sh --help +``` + +Первый запуск копирует `env.example` в `%USERPROFILE%\.gpu-rent\.env` / `~/.gpu-rent/.env`, если файла ещё нет. Заполни `OS_*` (шаги ниже). После `git pull`, если изменился `pyproject.toml`, лаунчер переустановит пакет. + +Ручной венв по желанию (для разработки / pytest): ```powershell python -m venv .venv .\.venv\Scripts\activate python -m pip install -U pip -python -m pip install -e . -``` - -Проверка: - -```powershell +python -m pip install -e ".[dev]" python -m gpu_rent --help -gpu-rent --help ``` --- diff --git a/gpu-rent.bat b/gpu-rent.bat new file mode 100644 index 0000000..d8ae7e7 --- /dev/null +++ b/gpu-rent.bat @@ -0,0 +1,15 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0" +chcp 65001 >nul + +rem cmd.exe / Explorer: bypass ExecutionPolicy. Logic lives in gpu-rent.ps1. +if "%~1"=="" ( + powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0gpu-rent.ps1" + echo. + pause + exit /b %ERRORLEVEL% +) + +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0gpu-rent.ps1" %* +exit /b %ERRORLEVEL% diff --git a/gpu-rent.ps1 b/gpu-rent.ps1 new file mode 100644 index 0000000..080cb6d --- /dev/null +++ b/gpu-rent.ps1 @@ -0,0 +1,89 @@ +#Requires -Version 5.1 +# Windows launcher: venv + pip + python -m gpu_rent. Call via gpu-rent.bat if ExecutionPolicy blocks this file. +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$Root = Split-Path -Parent $MyInvocation.MyCommand.Path +Set-Location -LiteralPath $Root +try { + $utf8 = New-Object System.Text.UTF8Encoding $false + [Console]::OutputEncoding = $utf8 + $script:OutputEncoding = $utf8 +} catch { +} + +function Test-Python311 { + param( + [Parameter(Mandatory = $true)][string]$Exe, + [string[]]$Prefix = @() + ) + $cmd = Get-Command $Exe -ErrorAction SilentlyContinue + if (-not $cmd) { + return $false + } + try { + & $Exe @Prefix -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)" 2>$null | Out-Null + return ($LASTEXITCODE -eq 0) + } catch { + return $false + } +} + +$PyExe = $null +$PyPrefix = @() +$found = $false +foreach ($row in @( + @{ Exe = "py"; Prefix = @("-3.12") }, + @{ Exe = "py"; Prefix = @("-3.11") }, + @{ Exe = "py"; Prefix = @("-3") }, + @{ Exe = "python3"; Prefix = @() }, + @{ Exe = "python"; Prefix = @() } + )) { + if (Test-Python311 -Exe $row.Exe -Prefix $row.Prefix) { + $PyExe = $row.Exe + $PyPrefix = $row.Prefix + $found = $true + break + } +} + +if (-not $found) { + Write-Host "gpu-rent: нужен Python 3.11+. Поставь с python.org и включи Add python.exe to PATH." + exit 1 +} + +$VenvPy = Join-Path $Root ".venv\Scripts\python.exe" +if (-not (Test-Path -LiteralPath $VenvPy)) { + Write-Host "gpu-rent: создаю .venv" + & $PyExe @PyPrefix -m venv (Join-Path $Root ".venv") +} + +$Stamp = Join-Path $Root ".venv\.gpu-rent-installed" +$Pyproject = Join-Path $Root "pyproject.toml" +$needInstall = -not (Test-Path -LiteralPath $Stamp) +if (-not $needInstall -and (Test-Path -LiteralPath $Pyproject)) { + $needInstall = (Get-Item -LiteralPath $Pyproject).LastWriteTime -gt (Get-Item -LiteralPath $Stamp).LastWriteTime +} +if ($needInstall) { + Write-Host "gpu-rent: ставлю пакет в .venv" + & $VenvPy -m pip install -U pip + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + & $VenvPy -m pip install -e $Root + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Get-Date -Format o | Set-Content -LiteralPath $Stamp -Encoding ascii +} + +$GpuHome = Join-Path $env:USERPROFILE ".gpu-rent" +$EnvFile = Join-Path $GpuHome ".env" +$Example = Join-Path $Root "env.example" +if (-not (Test-Path -LiteralPath $EnvFile)) { + New-Item -ItemType Directory -Force -Path $GpuHome | Out-Null + if (Test-Path -LiteralPath $Example) { + Copy-Item -LiteralPath $Example -Destination $EnvFile + Write-Host "gpu-rent: created $EnvFile - fill OS_* (docs/setup.md)" + } +} + +$ErrorActionPreference = "Continue" +& $VenvPy -m gpu_rent @args +exit $LASTEXITCODE diff --git a/gpu-rent.sh b/gpu-rent.sh new file mode 100644 index 0000000..5d899b0 --- /dev/null +++ b/gpu-rent.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Linux / macOS / Git Bash: venv + pip + python -m gpu_rent +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$ROOT" + +ok_py() { + local exe="$1" + shift || true + command -v "$exe" >/dev/null 2>&1 || return 1 + "$exe" "$@" -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)' >/dev/null 2>&1 +} + +PY=() +if ok_py python3.12; then + PY=(python3.12) +elif ok_py python3.11; then + PY=(python3.11) +elif ok_py python3; then + PY=(python3) +elif ok_py python; then + PY=(python) +elif ok_py py -3.12; then + PY=(py -3.12) +elif ok_py py -3.11; then + PY=(py -3.11) +else + echo "gpu-rent: нужен Python 3.11+. Ubuntu: sudo apt install python3 python3-venv python3-pip" >&2 + exit 1 +fi + +if [[ -x "$ROOT/.venv/Scripts/python.exe" ]]; then + VENV_PY="$ROOT/.venv/Scripts/python.exe" +elif [[ -x "$ROOT/.venv/bin/python" ]]; then + VENV_PY="$ROOT/.venv/bin/python" +else + echo "gpu-rent: создаю .venv" + "${PY[@]}" -m venv "$ROOT/.venv" + if [[ -x "$ROOT/.venv/Scripts/python.exe" ]]; then + VENV_PY="$ROOT/.venv/Scripts/python.exe" + else + VENV_PY="$ROOT/.venv/bin/python" + fi +fi + +STAMP="$ROOT/.venv/.gpu-rent-installed" +if [[ ! -f "$STAMP" || "$ROOT/pyproject.toml" -nt "$STAMP" ]]; then + echo "gpu-rent: ставлю пакет в .venv" + "$VENV_PY" -m pip install -U pip + "$VENV_PY" -m pip install -e "$ROOT" + date -u +"%Y-%m-%dT%H:%M:%SZ" >"$STAMP" +fi + +GPU_HOME="${HOME}/.gpu-rent" +if [[ ! -f "$GPU_HOME/.env" && -f "$ROOT/env.example" ]]; then + mkdir -p "$GPU_HOME" + cp "$ROOT/env.example" "$GPU_HOME/.env" + echo "gpu-rent: created $GPU_HOME/.env - fill OS_* (see docs/setup.md)" +fi + +exec "$VENV_PY" -m gpu_rent "$@" diff --git a/pyproject.toml b/pyproject.toml index e114b4c..351cc7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,9 @@ package-dir = { "" = "src" } [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +gpu_rent = ["remote/*"] + [tool.pytest.ini_options] testpaths = ["tests"] pythonpath = ["src"] diff --git a/src/gpu_rent/bootstrap.py b/src/gpu_rent/bootstrap.py new file mode 100644 index 0000000..d1619db --- /dev/null +++ b/src/gpu_rent/bootstrap.py @@ -0,0 +1,33 @@ +"""Upload and run the idempotent SwarmUI bootstrap on the VM.""" + +from __future__ import annotations + +from collections.abc import Callable +from importlib.resources import files + +from gpu_rent.config import Config +from gpu_rent.errors import CloudError +from gpu_rent.ssh_ops import run_script_sudo + +Log = Callable[[str], None] + + +def bootstrap_script() -> str: + return files("gpu_rent.remote").joinpath("bootstrap.sh").read_text(encoding="utf-8") + + +def run_bootstrap(cfg: Config, host: str, log: Log) -> None: + log("bootstrap SwarmUI на VM (идемпотентно, без Docker)") + script = bootstrap_script() + out = run_script_sudo( + cfg, + host, + script, + remote_path="/tmp/gpu-rent-bootstrap.sh", + timeout=1800, + env={"SWARM_USER": cfg.ssh_user}, + log=log, + ) + if "bootstrap ok" not in out: + raise CloudError(f"bootstrap не подтвердил успех:\n{out[-800:]}") + log("диски и systemd unit готовы; дальше seed, потом start swarmui") diff --git a/src/gpu_rent/civitai.py b/src/gpu_rent/civitai.py index 8a52419..f0e0dae 100644 --- a/src/gpu_rent/civitai.py +++ b/src/gpu_rent/civitai.py @@ -6,6 +6,8 @@ from dataclasses import dataclass import httpx +from gpu_rent.errors import CloudError + ALLOWED_HOSTS = ("civitai.com", "civitai.red", "civitai.green") @@ -56,3 +58,51 @@ def probe_me(token: str, host: str, timeout: float = 15.0) -> CivitaiProbe: status=response.status_code, detail=response.text[:200] or response.reason_phrase, ) + + +def pick_primary_file(version: dict) -> dict | None: + files = version.get("files") or [] + if not isinstance(files, list): + return None + for item in files: + if isinstance(item, dict) and item.get("primary"): + return item + for item in files: + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").lower() + if "safetensor" in name or name.endswith(".safetensors"): + return item + for item in files: + if isinstance(item, dict): + return item + return None + + +def fetch_model_version(token: str, host: str, version_id: int, timeout: float = 30.0) -> tuple[str, dict]: + """GET /api/v1/model-versions/{id}; one retry on the other Civitai host.""" + first = _normalize_host(host) + order = [first, other_host(first)] + last_error = "нет ответа" + seen: set[str] = set() + for candidate in order: + if candidate in seen or candidate not in ALLOWED_HOSTS: + continue + seen.add(candidate) + url = f"https://{candidate}/api/v1/model-versions/{version_id}" + try: + with httpx.Client(timeout=timeout, follow_redirects=True) as client: + response = client.get(url, headers={"Authorization": f"Bearer {token}"}) + except httpx.HTTPError as exc: + last_error = str(exc) + continue + if response.status_code == 200: + data = response.json() + if isinstance(data, dict) and pick_primary_file(data): + return candidate, data + last_error = "пустой files[]" + continue + last_error = f"HTTP {response.status_code}" + if response.status_code not in {404, 400}: + break + raise CloudError(f"Civitai version {version_id}: {last_error} (хосты {', '.join(seen)})") diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 28b9de9..d1a096a 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -18,7 +18,12 @@ from gpu_rent.config import load_config from gpu_rent.doctor import blocking_failed, dry_run_plan, run_doctor from gpu_rent.errors import GpuRentError, NotReadyError from gpu_rent.os_client import connect, find_snapshot_by_name, find_tagged_servers +from gpu_rent.session import cmd_stop, cmd_up +from gpu_rent.ssh_ops import interactive_ssh, run_ssh from gpu_rent.state import load_state, preempt_window_end +from gpu_rent.provision import ensure_swarmui_running, seed_civitai, seed_extensions +from gpu_rent.sync_files import pull_tree, push_tree +from gpu_rent.tunnel import run_tunnel if sys.platform == "win32": for _stream in (sys.stdout, sys.stderr): @@ -55,10 +60,8 @@ def _die(exc: BaseException) -> None: def _nyi(name: str) -> None: raise NotReadyError( - f"`{name}` ещё не создаёт/не гасит GPU. Сейчас работают: doctor, dry-run, status, open.\n" - "1) Заполни ~/.gpu-rent/.env по docs/setup.md\n" - "2) gpu-rent doctor\n" - "3) Когда doctor зелёный и квота GPU > 0 — можно писать up." + f"`{name}` ещё не готов. Уже работают: doctor, dry-run, status, open, up, stop, destroy, ssh, logs, tunnel, seed-*, push, pull-output.\n" + "Ключи: docs/setup.md" ) @@ -77,10 +80,18 @@ def _print_checks(checks) -> int: if failed: console.print("\n[red]Сессию начинать нельзя.[/red] См. docs/setup.md") return 1 - console.print("\n[green]Можно идти дальше.[/green] mutating up пока не подключён.") + console.print("\n[green]Можно идти дальше.[/green] Дальше: gpu-rent up --yes") return 0 +def _live(): + cfg = load_config(require_auth=True) + state = load_state() + if not state.floating_ip: + raise GpuRentError("нет floating IP — сначала gpu-rent up") + return cfg, state.floating_ip + + @app.command() def version() -> None: """Версия пакета.""" @@ -185,14 +196,86 @@ def up( yes: bool = typer.Option(False, "--yes", help="Без вопросов"), adopt: bool = typer.Option(False, "--adopt", help="Подхватить тег gpu-rent без state"), ) -> None: - """Create/unshelve GPU. Пока: doctor, затем стоп — mutating ещё не подключён.""" - del no_spot, flavor, yes, adopt + """Create/unshelve GPU и bootstrap SwarmUI (без Docker).""" try: checks = run_doctor() code = _print_checks(checks) if code != 0: raise typer.Exit(1) - _nyi("up") + cfg = load_config(require_auth=True) + + def confirm(msg: str) -> bool: + return typer.confirm(msg) + + cmd_up( + cfg, + no_spot=no_spot, + flavor=flavor, + yes=yes, + adopt=adopt, + confirm=confirm, + log=lambda m: console.print(m), + ) + except GpuRentError as exc: + _die(exc) + + +@app.command() +def stop( + no_pull: bool = typer.Option(False, "--no-pull"), +) -> None: + """Удалить compute и FIP, диски оставить.""" + try: + cfg = load_config(require_auth=True) + cmd_stop(cfg, no_pull=no_pull, log=lambda m: console.print(m)) + except GpuRentError as exc: + _die(exc) + + +@app.command() +def destroy( + i_understand_data_loss: bool = typer.Option(False, "--i-understand-data-loss"), +) -> None: + """stop + диски.""" + if not i_understand_data_loss: + console.print("Нужен флаг --i-understand-data-loss") + raise typer.Exit(1) + try: + cfg = load_config(require_auth=True) + cmd_stop(cfg, destroy_disks=True, log=lambda m: console.print(m)) + except GpuRentError as exc: + _die(exc) + + +@app.command() +def ssh() -> None: + """Оболочка на VM (нужен живой compute и FIP).""" + try: + cfg = load_config(require_auth=True) + state = load_state() + if not state.floating_ip: + raise GpuRentError("нет floating IP в state — сначала gpu-rent up") + raise typer.Exit(interactive_ssh(cfg, state.floating_ip)) + except GpuRentError as exc: + _die(exc) + + +@app.command() +def logs() -> None: + """cloud-init / journalctl на VM.""" + try: + cfg = load_config(require_auth=True) + state = load_state() + if not state.floating_ip: + raise GpuRentError("нет IP — VM не поднята") + out = run_ssh( + cfg, + state.floating_ip, + "sudo -n tail -n 80 /var/log/cloud-init-output.log 2>/dev/null; " + "systemctl is-active swarmui 2>/dev/null || true", + check=False, + ) + console.print(out) except GpuRentError as exc: _die(exc) @@ -202,9 +285,17 @@ def tunnel( open_browser: bool = typer.Option(False, "--open", help="Открыть браузер на 17801"), ) -> None: """SSH localhost:17801 -> VM :7801. Ctrl+C закрывает туннель, GPU оставляет.""" - del open_browser try: - _nyi("tunnel") + cfg = load_config(require_auth=True) + state = load_state() + if not state.floating_ip: + raise GpuRentError("нет floating IP — сначала gpu-rent up") + run_tunnel( + cfg, + state.floating_ip, + open_browser=open_browser, + log=lambda m: console.print(m), + ) except GpuRentError as exc: _die(exc) @@ -223,84 +314,76 @@ def hold( _die(exc) -@app.command() -def stop( - no_pull: bool = typer.Option(False, "--no-pull"), -) -> None: - """Удалить compute и FIP, диски оставить.""" - del no_pull - try: - _nyi("stop") - except GpuRentError as exc: - _die(exc) - - -@app.command() -def destroy( - i_understand_data_loss: bool = typer.Option(False, "--i-understand-data-loss"), -) -> None: - """stop + диски.""" - if not i_understand_data_loss: - console.print("Нужен флаг --i-understand-data-loss") - raise typer.Exit(1) - try: - _nyi("destroy") - except GpuRentError as exc: - _die(exc) - - -@app.command() -def logs() -> None: - try: - _nyi("logs") - except GpuRentError as exc: - _die(exc) - - -@app.command() -def ssh() -> None: - try: - _nyi("ssh") - except GpuRentError as exc: - _die(exc) - - @app.command("seed-models") def seed_models() -> None: + """Докачать Civitai-манифест на живой диск.""" try: - _nyi("seed-models") + cfg, host = _live() + seed_civitai(cfg, host, log=lambda m: console.print(m)) except GpuRentError as exc: _die(exc) @app.command("push") def push_all() -> None: + """SFTP Models + Wildcards + CustomWorkflows.""" try: - _nyi("push") + cfg, host = _live() + + def log(msg: str) -> None: + console.print(msg) + + push_tree(cfg, host, cfg.local_models_dir, "/mnt/swarm_data/Models", log, models=True) + push_tree(cfg, host, cfg.local_wildcards_dir, "/mnt/swarm_data/Data/Wildcards", log, models=False) + push_tree(cfg, host, cfg.local_workflows_dir, "/mnt/swarm_data/CustomWorkflows", log, models=False) except GpuRentError as exc: _die(exc) @app.command("push-models") def push_models() -> None: + """SFTP только ./Models.""" try: - _nyi("push-models") + cfg, host = _live() + push_tree( + cfg, + host, + cfg.local_models_dir, + "/mnt/swarm_data/Models", + lambda m: console.print(m), + models=True, + ) except GpuRentError as exc: _die(exc) @app.command("pull-output") -def pull_output() -> None: +def pull_output_cmd() -> None: + """Забрать новые файлы Output/ с VM.""" try: - _nyi("pull-output") + cfg, host = _live() + pull_tree( + cfg, + host, + "/mnt/swarm_data/Output", + cfg.local_output_dir, + lambda m: console.print(m), + ) except GpuRentError as exc: _die(exc) @app.command("seed-extensions") -def seed_extensions() -> None: +def seed_extensions_cmd() -> None: + """Clone/fetch extensions.yaml, затем restart swarmui.""" try: - _nyi("seed-extensions") + cfg, host = _live() + + def log(msg: str) -> None: + console.print(msg) + + seed_extensions(cfg, host, log) + ensure_swarmui_running(cfg, host, log, restart=True) except GpuRentError as exc: _die(exc) diff --git a/src/gpu_rent/cloud.py b/src/gpu_rent/cloud.py new file mode 100644 index 0000000..23cbf33 --- /dev/null +++ b/src/gpu_rent/cloud.py @@ -0,0 +1,450 @@ +"""Find-or-create Selectel OpenStack resources for one gpu-rent session.""" + +from __future__ import annotations + +import time +from typing import Any, Callable + +import httpx + +from gpu_rent.errors import CloudError +from gpu_rent.os_client import ( + BOOT_VOLUME_NAME, + BOOT_VOLUME_SIZE_GB, + DATA_VOLUME_NAME, + KEYPAIR_NAME, + NET_NAME, + PREEMPTIBLE_TAG, + RESOURCE_TAG, + ROUTER_NAME, + SERVER_NAME, + SG_NAME, + SUBNET_CIDR, + SUBNET_NAME, + find_snapshot_by_name, + find_tagged_servers, + find_volumes_by_name, +) + + +def _wrap(exc: BaseException, what: str) -> CloudError: + text = str(exc) + low = text.lower() + if "402" in text or "payment" in low: + return CloudError(f"{what}: нет средств / 402. {text}") + if "403" in text or "forbidden" in low or "quota" in low: + return CloudError( + f"{what}: квота или запрет (403). Часто GPU=0 — тикет в поддержку, docs/setup.md. {text}" + ) + return CloudError(f"{what}: {text}") + + +def _oid(obj: Any) -> str: + return str(getattr(obj, "id", "") or "") + + +def guess_operator_cidr() -> str: + try: + response = httpx.get("https://ifconfig.me/ip", timeout=5.0) + ip = response.text.strip() + if ip.count(".") == 3 and all(p.isdigit() for p in ip.split(".")): + return f"{ip}/32" + except httpx.HTTPError: + pass + return "0.0.0.0/0" + + +def wait_volume(conn, volume, status: str = "available", timeout: int = 900) -> Any: + deadline = time.time() + timeout + vid = _oid(volume) + last = None + while time.time() < deadline: + current = conn.block_storage.get_volume(vid) + last = getattr(current, "status", None) + if last == status: + return current + if last in {"error", "error_restoring"}: + raise CloudError(f"том {vid} статус {last}") + time.sleep(5) + raise CloudError(f"том {vid} не стал {status} (последний {last})") + + +def wait_server(conn, server, status: str = "ACTIVE", timeout: int = 900) -> Any: + deadline = time.time() + timeout + sid = _oid(server) + last = None + while time.time() < deadline: + current = conn.compute.get_server(sid) + last = (getattr(current, "status", None) or "").upper() + if last == status.upper(): + return current + if last in {"ERROR"}: + fault = getattr(current, "fault", None) + raise CloudError(f"сервер {sid} ERROR{f' {fault}' if fault else ''}") + time.sleep(5) + raise CloudError(f"сервер {sid} не стал {status} (последний {last})") + + +def wait_gone(fetch: Callable[[], Any], timeout: int = 300) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + try: + obj = fetch() + except Exception: + return + if obj is None: + return + time.sleep(4) + raise CloudError("ресурс не исчез вовремя") + + +def find_external_network(conn) -> Any: + for net in conn.network.networks(): + if getattr(net, "is_router_external", False): + return net + for net in conn.network.networks(): + name = (getattr(net, "name", "") or "").lower() + if name in {"external-network", "wan", "public"}: + return net + raise CloudError("нет внешней сети для floating IP / router gateway") + + +def ensure_network(conn, log: Callable[[str], None]) -> tuple[Any, Any]: + net = conn.network.find_network(NET_NAME) + if net: + subnet = next(iter(conn.network.subnets(network_id=net.id)), None) + if not subnet: + raise CloudError(f"сеть {NET_NAME} есть, subnet нет") + log(f"сеть {NET_NAME} уже есть") + return net, subnet + try: + net = conn.network.create_network(name=NET_NAME) + subnet = conn.network.create_subnet( + name=SUBNET_NAME, + network_id=net.id, + ip_version=4, + cidr=SUBNET_CIDR, + dns_nameservers=["1.1.1.1", "8.8.8.8"], + ) + ext = find_external_network(conn) + router = conn.network.find_router(ROUTER_NAME) + if not router: + router = conn.network.create_router( + name=ROUTER_NAME, + external_gateway_info={"network_id": ext.id}, + ) + conn.network.add_interface_to_router(router, subnet_id=subnet.id) + log(f"создана сеть {NET_NAME} + router") + return net, subnet + except Exception as exc: + raise _wrap(exc, "сеть") from exc + + +def ensure_security_group(conn, cidr: str, log: Callable[[str], None]) -> Any: + sg = conn.network.find_security_group(SG_NAME) + if sg: + log(f"security group {SG_NAME} уже есть") + return sg + try: + sg = conn.network.create_security_group( + name=SG_NAME, + description="gpu-rent SSH only", + ) + conn.network.create_security_group_rule( + security_group_id=sg.id, + direction="ingress", + ethertype="IPv4", + protocol="tcp", + port_range_min=22, + port_range_max=22, + remote_ip_prefix=cidr, + ) + log(f"SG {SG_NAME}: TCP/22 с {cidr}") + return sg + except Exception as exc: + raise _wrap(exc, "security group") from exc + + +def ensure_keypair(conn, public_key: str, log: Callable[[str], None]) -> Any: + existing = conn.compute.find_keypair(KEYPAIR_NAME) + pub = public_key.strip() + if existing: + have = (getattr(existing, "public_key", "") or "").strip() + if have == pub: + log(f"keypair {KEYPAIR_NAME} совпадает") + return existing + log(f"keypair {KEYPAIR_NAME} другой — пересоздаём") + try: + conn.compute.delete_keypair(existing) + except Exception as exc: + raise _wrap(exc, "удалить keypair") from exc + try: + key = conn.compute.create_keypair(name=KEYPAIR_NAME, public_key=pub) + log(f"keypair {KEYPAIR_NAME} создан") + return key + except Exception as exc: + raise _wrap(exc, "keypair") from exc + + +def ensure_boot_volume( + conn, + *, + az: str, + volume_type: str | None, + image_id: str | None, + snapshot_name: str, + existing_id: str, + log: Callable[[str], None], +) -> Any: + if existing_id: + vol = conn.block_storage.get_volume(existing_id) + log(f"boot volume из state {existing_id}") + return vol + found = find_volumes_by_name(conn, BOOT_VOLUME_NAME) + if found: + log(f"boot volume {BOOT_VOLUME_NAME} уже в проекте") + return found[0] + snap = find_snapshot_by_name(conn, snapshot_name) + kwargs: dict[str, Any] = { + "name": BOOT_VOLUME_NAME, + "size": BOOT_VOLUME_SIZE_GB, + "availability_zone": az, + } + if volume_type: + kwargs["volume_type"] = volume_type + if snap: + kwargs["snapshot_id"] = snap.id + log(f"boot volume из snapshot {snapshot_name}") + elif image_id: + kwargs["image_id"] = image_id + log("boot volume из GPU-образа") + else: + raise CloudError("нет ни snapshot, ни image_id для boot volume") + try: + vol = conn.block_storage.create_volume(**kwargs) + return wait_volume(conn, vol) + except TypeError: + if "image_id" in kwargs: + kwargs["imageRef"] = kwargs.pop("image_id") + try: + vol = conn.block_storage.create_volume(**kwargs) + return wait_volume(conn, vol) + except Exception as exc: + raise _wrap(exc, "boot volume") from exc + except Exception as exc: + raise _wrap(exc, "boot volume") from exc + + +def ensure_data_volume( + conn, + *, + az: str, + volume_type: str | None, + size_gb: int, + existing_id: str, + log: Callable[[str], None], +) -> Any: + if existing_id: + vol = conn.block_storage.get_volume(existing_id) + log(f"data volume из state {existing_id}") + return vol + found = find_volumes_by_name(conn, DATA_VOLUME_NAME) + if found: + log(f"data volume {DATA_VOLUME_NAME} уже в проекте") + return found[0] + kwargs: dict[str, Any] = { + "name": DATA_VOLUME_NAME, + "size": size_gb, + "availability_zone": az, + } + if volume_type: + kwargs["volume_type"] = volume_type + try: + vol = conn.block_storage.create_volume(**kwargs) + log(f"data volume {size_gb} GB") + return wait_volume(conn, vol) + except Exception as exc: + raise _wrap(exc, "data volume") from exc + + +def _tag_server(conn, server, spot: bool) -> None: + tags = [RESOURCE_TAG] + if spot: + tags.append(PREEMPTIBLE_TAG) + try: + conn.compute.set_server_tags(server, tags) + except Exception: + try: + for tag in tags: + conn.compute.add_tag_to_server(server, tag) + except Exception: + pass + + +def create_gpu_server( + conn, + *, + flavor_id: str, + net_id: str, + sg_name: str, + boot_volume_id: str, + data_volume_id: str, + az: str, + spot: bool, + log: Callable[[str], None], +) -> Any: + bdm = [ + { + "boot_index": 0, + "uuid": boot_volume_id, + "source_type": "volume", + "destination_type": "volume", + "delete_on_termination": False, + }, + { + "boot_index": 1, + "uuid": data_volume_id, + "source_type": "volume", + "destination_type": "volume", + "delete_on_termination": False, + }, + ] + tags = [RESOURCE_TAG] + if spot: + tags.append(PREEMPTIBLE_TAG) + kwargs: dict[str, Any] = { + "name": SERVER_NAME, + "flavor_id": flavor_id, + "networks": [{"uuid": net_id}], + "key_name": KEYPAIR_NAME, + "availability_zone": az, + "block_device_mapping_v2": bdm, + "security_groups": [{"name": sg_name}], + "tags": tags, + } + try: + server = conn.compute.create_server(**kwargs) + except Exception as exc: + kwargs.pop("tags", None) + try: + server = conn.compute.create_server(**kwargs) + except Exception as exc2: + raise _wrap(exc2, "create server") from exc + log("ждём Nova ACTIVE (GPU create может занять несколько минут)") + server = wait_server(conn, server, "ACTIVE") + _tag_server(conn, server, spot) + return server + + +def unshelve(conn, server, log: Callable[[str], None]) -> Any: + log("unshelve (EXPIRED / shelved)") + try: + conn.compute.unshelve_server(server) + except Exception as exc: + raise _wrap(exc, "unshelve") from exc + return wait_server(conn, server, "ACTIVE", timeout=900) + + +def associate_floating_ip(conn, server, log: Callable[[str], None]) -> tuple[str, str]: + ports = list(conn.network.ports(device_id=server.id)) + if not ports: + raise CloudError("у сервера нет neutron-порта — не к чему привязать FIP") + ext = find_external_network(conn) + try: + fip = conn.network.create_ip(floating_network_id=ext.id) + except Exception as exc: + raise _wrap(exc, "floating IP allocate") from exc + try: + fip = conn.network.update_ip(fip, port_id=ports[0].id) + except Exception as exc: + raise _wrap(exc, "floating IP associate") from exc + addr = getattr(fip, "floating_ip_address", None) or getattr(fip, "name", None) + log(f"floating IP {addr}") + return str(addr), _oid(fip) + + +def ensure_floating_ip( + conn, + server, + existing_id: str | None, + existing_addr: str | None, + log: Callable[[str], None], +) -> tuple[str, str]: + have = server_floating_ip(server) + if have: + return have, existing_id or "" + ports = list(conn.network.ports(device_id=server.id)) + if existing_id and ports: + try: + fip = conn.network.update_ip(existing_id, port_id=ports[0].id) + addr = getattr(fip, "floating_ip_address", None) or existing_addr + log(f"вернули FIP {addr}") + return str(addr), existing_id + except Exception: + log("старый FIP не привязался — выделяем новый") + return associate_floating_ip(conn, server, log) + + +def delete_floating_ip(conn, fip_id: str | None, address: str | None, log: Callable[[str], None]) -> None: + if fip_id: + try: + conn.network.delete_ip(fip_id, ignore_missing=True) + log(f"удалён FIP {fip_id}") + return + except Exception: + pass + if address: + for ip in conn.network.ips(): + if getattr(ip, "floating_ip_address", None) == address: + conn.network.delete_ip(ip, ignore_missing=True) + log(f"удалён FIP {address}") + return + + +def delete_server(conn, server, log: Callable[[str], None]) -> None: + sid = _oid(server) + log(f"удаляем compute {sid} (диски оставляем)") + try: + conn.compute.delete_server(server, ignore_missing=True) + except Exception as exc: + raise _wrap(exc, "delete server") from exc + + def _get(): + try: + current = conn.compute.get_server(sid) + except Exception: + return None + if current is None: + return None + if server_status(current) in {"DELETED", "SOFT_DELETED"}: + return None + return current + + wait_gone(_get, timeout=420) + + +def server_status(server: Any) -> str: + return (getattr(server, "status", None) or "").upper() + + +def server_floating_ip(server: Any) -> str | None: + addrs = getattr(server, "addresses", None) or {} + if not isinstance(addrs, dict): + return None + for nets in addrs.values(): + if not isinstance(nets, list): + continue + for item in nets: + if not isinstance(item, dict): + continue + if item.get("OS-EXT-IPS:type") == "floating": + return str(item.get("addr") or "") or None + return None + + +def pick_existing_server(conn) -> Any | None: + found = find_tagged_servers(conn) + if len(found) > 1: + ids = ", ".join(_oid(s) for s in found) + raise CloudError(f"несколько серверов gpu-rent: {ids}. Разбери вручную или --adopt один.") + return found[0] if found else None diff --git a/src/gpu_rent/doctor.py b/src/gpu_rent/doctor.py index 7d1ec0b..84668cf 100644 --- a/src/gpu_rent/doctor.py +++ b/src/gpu_rent/doctor.py @@ -26,6 +26,7 @@ from gpu_rent.os_client import ( iter_volume_types, volume_quotas, ) +from gpu_rent.payload import folder_bytes as _folder_bytes, has_payload as _has_payload from gpu_rent.paths import env_path, home_dir from gpu_rent.ssh_keys import key_ready from gpu_rent.state import load_state @@ -39,28 +40,6 @@ class Check: detail: str -SKIP = {".gitkeep", "README.md", ".gitignore"} - - -def _folder_bytes(root: Path) -> int: - if not root.is_dir(): - return 0 - total = 0 - for path in root.rglob("*"): - if path.is_file() and path.name not in SKIP: - total += path.stat().st_size - return total - - -def _has_payload(root: Path) -> bool: - if not root.is_dir(): - return False - for path in root.rglob("*"): - if path.is_file() and path.name not in SKIP: - return True - return False - - def run_doctor() -> list[Check]: checks: list[Check] = [] home_dir().mkdir(parents=True, exist_ok=True) @@ -383,7 +362,8 @@ def dry_run_plan(checks: list[Check]) -> list[str]: f"preemptible: {cfg.default_spot} (обычный сервер: gpu-rent up --no-spot)", f"idle-killer: {cfg.idle_minutes} мин пустой очереди, льгота {cfg.idle_grace_minutes} мин", f"туннель: localhost:{cfg.swarmui_local_port} -> VM :7801", - "сейчас mutating up/stop ещё не подключены — только doctor / dry-run / status / open", + "gpu-rent up --yes создаст сеть/диски/compute и поставит SwarmUI (если doctor зелёный и квота GPU > 0)", + "после up: gpu-rent tunnel (Ctrl+C не гасит GPU)", ] flavor = next((c.detail for c in checks if c.name == "flavor" and c.ok), None) if flavor: diff --git a/src/gpu_rent/inventory.py b/src/gpu_rent/inventory.py index 62aa0c7..9c82f41 100644 --- a/src/gpu_rent/inventory.py +++ b/src/gpu_rent/inventory.py @@ -167,3 +167,31 @@ def pick_boot_image(images: list[Any]) -> Any | None: return None ranked.sort(key=lambda item: item[0], reverse=True) return ranked[0][1] + + +def resolve_flavor( + flavors: list[Any], + preference: tuple[str, ...], + *, + explicit: str | None = None, + default_id: str | None = None, + fallback: bool = True, +) -> FlavorInfo: + if explicit: + for flavor in flavors: + if _id(flavor) == explicit or _name(flavor) == explicit: + if is_disabled(flavor): + raise ValueError(f"flavor {explicit} disabled") + return flavor_info(flavor, label="explicit") + raise ValueError(f"flavor {explicit} не найден в регионе") + if not fallback: + if not default_id: + raise ValueError("FLAVOR_FALLBACK=false требует DEFAULT_FLAVOR_ID или --flavor") + return resolve_flavor(flavors, preference, explicit=default_id, fallback=True) + gpu = [f for f in flavors if looks_like_gpu(f)] + ranked = rank_flavors(gpu or flavors, preference) + if ranked: + return ranked[0] + if default_id: + return resolve_flavor(flavors, preference, explicit=default_id, fallback=True) + raise ValueError("нет доступного GPU flavor из FLAVOR_PREFERENCE") diff --git a/src/gpu_rent/lock.py b/src/gpu_rent/lock.py new file mode 100644 index 0000000..7d895e9 --- /dev/null +++ b/src/gpu_rent/lock.py @@ -0,0 +1,59 @@ +"""Single-instance lock for up/stop.""" + +from __future__ import annotations + +import os +import sys +from types import TracebackType + +from gpu_rent.errors import GpuRentError +from gpu_rent.paths import lock_path + + +def _pid_alive(pid: int) -> bool: + if pid <= 0: + return False + if sys.platform == "win32": + import ctypes + + handle = ctypes.windll.kernel32.OpenProcess(0x100000, False, pid) + if handle: + ctypes.windll.kernel32.CloseHandle(handle) + return True + return False + try: + os.kill(pid, 0) + return True + except OSError: + return False + + +class SessionLock: + def __enter__(self) -> SessionLock: + path = lock_path() + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_file(): + try: + old = int(path.read_text(encoding="utf-8").strip() or "0") + except ValueError: + old = 0 + if _pid_alive(old) and old != os.getpid(): + raise GpuRentError( + f"gpu-rent уже работает (pid {old}, {path}). Дождись окончания up/stop." + ) + path.write_text(str(os.getpid()), encoding="utf-8") + self._path = path + self._pid = os.getpid() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + try: + if self._path.is_file() and self._path.read_text(encoding="utf-8").strip() == str(self._pid): + self._path.unlink() + except OSError: + pass diff --git a/src/gpu_rent/manifests.py b/src/gpu_rent/manifests.py index 420669d..73e2ce6 100644 --- a/src/gpu_rent/manifests.py +++ b/src/gpu_rent/manifests.py @@ -2,9 +2,11 @@ from __future__ import annotations +import re from dataclasses import dataclass from pathlib import Path from typing import Any +from urllib.parse import parse_qs, urlparse import yaml @@ -17,8 +19,24 @@ MODEL_TYPES = ( "embedding", "controlnet", "upscaler", + "clip", ) +MODEL_DIRS = { + "checkpoint": "Stable-Diffusion", + "lora": "Lora", + "vae": "VAE", + "embedding": "Embeddings", + "controlnet": "controlnet", + "upscaler": "upscale_models", + "clip": "clip", +} + +_VERSION_QS = re.compile(r"modelVersionId=(\d+)", re.I) +_VERSION_PATH = re.compile(r"/model-versions/(\d+)", re.I) +_DOWNLOAD_PATH = re.compile(r"/api/download/models/(\d+)", re.I) +_SHA_REF = re.compile(r"^[0-9a-fA-F]{40}$") + @dataclass class ModelEntry: @@ -71,7 +89,10 @@ def parse_models(path: Path) -> list[ModelEntry]: if vid in (0, "0", None) and not url: continue version_id = int(vid) if vid not in (None, "", 0, "0") else None - entries.append(ModelEntry(kind=kind, version_id=version_id, url=str(url) if url else None)) + url_s = str(url) if url else None + if version_id is None and url_s: + version_id = extract_version_id(url_s) + entries.append(ModelEntry(kind=kind, version_id=version_id, url=url_s)) return entries @@ -94,9 +115,43 @@ def parse_extensions(path: Path) -> list[GitRepo]: repos.append( GitRepo( kind=kind, - url=str(item["url"]), + url=str(item["url"]).strip(), ref=str(item.get("ref") or "main"), directory=str(item["dir"]) if item.get("dir") else None, ) ) return repos + + +def extract_version_id(url: str) -> int | None: + text = url.strip() + for rx in (_VERSION_QS, _VERSION_PATH, _DOWNLOAD_PATH): + match = rx.search(text) + if match: + return int(match.group(1)) + parsed = urlparse(text) + ids = parse_qs(parsed.query).get("modelVersionId") or parse_qs(parsed.query).get("modelversionid") + if ids: + try: + return int(ids[0]) + except ValueError: + return None + return None + + +def repo_dirname(repo: GitRepo) -> str: + if repo.directory: + return repo.directory + name = repo.url.rstrip("/").rsplit("/", 1)[-1] + if name.endswith(".git"): + name = name[:-4] + return name or "extension" + + +def is_commit_sha(ref: str) -> bool: + return bool(_SHA_REF.match(ref.strip())) + + +def remote_root_for(repo: GitRepo) -> str: + base = "/mnt/swarm_data/Extensions" if repo.kind == "swarmui" else "/mnt/swarm_data/DLNodes" + return f"{base}/{repo_dirname(repo)}" diff --git a/src/gpu_rent/os_client.py b/src/gpu_rent/os_client.py index 7374fb9..769974b 100644 --- a/src/gpu_rent/os_client.py +++ b/src/gpu_rent/os_client.py @@ -9,8 +9,20 @@ from gpu_rent.config import Config from gpu_rent.errors import CloudError RESOURCE_TAG = "gpu-rent" +PREEMPTIBLE_TAG = "preemptible" COMPUTE_MICROVERSION = "2.72" +SERVER_NAME = "gpu-rent" +KEYPAIR_NAME = "gpu-rent" +NET_NAME = "gpu-rent" +SUBNET_NAME = "gpu-rent-subnet" +ROUTER_NAME = "gpu-rent" +SG_NAME = "gpu-rent" +BOOT_VOLUME_NAME = "gpu-rent-boot" +DATA_VOLUME_NAME = "gpu-rent-data" +SUBNET_CIDR = "192.168.77.0/24" +BOOT_VOLUME_SIZE_GB = 40 + def connect(cfg: Config): try: @@ -34,6 +46,10 @@ def connect(cfg: Config): app_version="0.1.0", ) conn.authorize() + try: + conn.compute.default_microversion = COMPUTE_MICROVERSION + except Exception: + pass except Exception as exc: raise CloudError( "Keystone не выдал токен. Проверь OS_USERNAME / OS_PASSWORD / " diff --git a/src/gpu_rent/payload.py b/src/gpu_rent/payload.py new file mode 100644 index 0000000..4a93868 --- /dev/null +++ b/src/gpu_rent/payload.py @@ -0,0 +1,93 @@ +"""Local app trees: Models / Wildcards / CustomWorkflows / Output.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +SKIP_NAMES = {".gitkeep", "README.md", "README.txt", ".gitignore"} +WEIGHT_SUFFIXES = {".safetensors", ".ckpt", ".pt", ".pth", ".bin", ".gguf", ".sft", ".onnx"} +META_SUFFIXES = {".json", ".civitai.json", ".swarm.json", ".preview.png", ".png", ".webp", ".jpg"} + + +def is_skipped(path: Path) -> bool: + name = path.name + if name in SKIP_NAMES or name.startswith("."): + return True + return False + + +def has_payload(root: Path) -> bool: + if not root.is_dir(): + return False + for path in root.rglob("*"): + if path.is_file() and not is_skipped(path): + return True + return False + + +def folder_bytes(root: Path) -> int: + if not root.is_dir(): + return 0 + total = 0 + for path in root.rglob("*"): + if path.is_file() and not is_skipped(path): + total += path.stat().st_size + return total + + +def iter_payload_files(root: Path) -> list[Path]: + if not root.is_dir(): + return [] + found = [] + for path in sorted(root.rglob("*")): + if path.is_file() and not is_skipped(path): + found.append(path) + return found + + +def is_weight(path: Path) -> bool: + return path.suffix.lower() in WEIGHT_SUFFIXES + + +def sha256_file(path: Path, chunk: int = 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + while True: + block = fh.read(chunk) + if not block: + break + digest.update(block) + return digest.hexdigest() + + +def sidecar_stem(name: str) -> str: + lower = name.lower() + for suffix in ( + ".civitai.json", + ".swarm.json", + ".preview.png", + ".preview.webp", + ".preview.jpg", + ".json", + ".png", + ".webp", + ".jpg", + ): + if lower.endswith(suffix): + return name[: -len(suffix)] + return Path(name).stem + + +def model_push_set(root: Path) -> list[Path]: + """Weights plus same-stem sidecars. Sidecar without weights is skipped.""" + files = iter_payload_files(root) + weights = [p for p in files if is_weight(p)] + wanted: set[Path] = set(weights) + stems = {p.stem for p in weights} + for path in files: + if is_weight(path): + continue + if sidecar_stem(path.name) in stems: + wanted.add(path) + return sorted(wanted) diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py new file mode 100644 index 0000000..65490b1 --- /dev/null +++ b/src/gpu_rent/provision.py @@ -0,0 +1,269 @@ +"""After OS bootstrap: extensions, autocomplete, Civitai, then start SwarmUI.""" + +from __future__ import annotations + +import json +import shlex +from collections.abc import Callable +from datetime import datetime, timezone +from importlib.resources import files +from pathlib import Path + +import httpx + +from gpu_rent.civitai import fetch_model_version, pick_primary_file +from gpu_rent.config import Config +from gpu_rent.errors import CloudError, GpuRentError +from gpu_rent.manifests import ( + MODEL_DIRS, + extract_version_id, + parse_extensions, + parse_models, + remote_root_for, +) +from gpu_rent.ssh_ops import put_text, remote_exists, run_python, run_ssh +from gpu_rent.sync_files import pull_tree, push_tree + +Log = Callable[[str], None] +DATA = "/mnt/swarm_data" + + +def _pkg_text(name: str) -> str: + return files("gpu_rent.remote").joinpath(name).read_text(encoding="utf-8") + + +def seed_extensions(cfg: Config, host: str, log: Log) -> bool: + repos = parse_extensions(cfg.extensions_manifest) + if not repos: + log("extensions.yaml пуст — стоковый SwarmUI") + return False + jobs = [] + for repo in repos: + jobs.append( + { + "kind": repo.kind, + "url": repo.url, + "ref": repo.ref, + "dest": remote_root_for(repo), + } + ) + put_text(cfg, host, "/tmp/gpu-rent-ext.json", json.dumps(jobs, indent=2)) + if cfg.git_token: + put_text(cfg, host, "/tmp/gpu-rent-git.token", cfg.git_token + "\n", mode=0o600) + log(f"clone {len(jobs)} git-реп на data volume") + out = run_python( + cfg, + host, + _pkg_text("clone_ext.py"), + remote_path="/tmp/gpu-rent-clone_ext.py", + timeout=1800, + log=log, + ) + return "cloned " in out or "updated " in out + + +def _github_blob(cfg: Config) -> dict | None: + url = ( + f"https://api.github.com/repos/{cfg.autocomplete_github_repo}/contents/" + f"{cfg.autocomplete_github_path}?ref={cfg.autocomplete_github_ref}" + ) + headers = {"Accept": "application/vnd.github+json", "User-Agent": "gpu-rent"} + if cfg.git_token: + headers["Authorization"] = f"Bearer {cfg.git_token}" + try: + with httpx.Client(timeout=20.0, follow_redirects=True) as client: + response = client.get(url, headers=headers) + except httpx.HTTPError as exc: + log_skip = str(exc) + return {"error": log_skip} + if response.status_code == 403: + return {"error": "GitHub rate limit — autocomplete не обновляю"} + if response.status_code != 200: + return {"error": f"GitHub HTTP {response.status_code}"} + data = response.json() + if not isinstance(data, dict): + return {"error": "неожиданный JSON GitHub"} + return data + + +def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool: + if not cfg.autocomplete_enabled: + log("autocomplete выключен") + return False + dest_dir = f"{DATA}/Data/Autocompletions" + dest = f"{dest_dir}/{cfg.autocomplete_filename}" + meta_path = f"{dest}.gpu-rent-meta.json" + blob = _github_blob(cfg) + if blob is None: + return False + if blob.get("error"): + log(str(blob["error"])) + return False + sha = str(blob.get("sha") or "") + download_url = str(blob.get("download_url") or "") + old_sha = "" + if remote_exists(cfg, host, meta_path): + raw = run_ssh(cfg, host, f"cat {meta_path}", check=False) + try: + old_sha = str(json.loads(raw).get("github_blob_sha") or "") + except json.JSONDecodeError: + old_sha = "" + changed = sha != old_sha or not remote_exists(cfg, host, dest) + if changed: + if not download_url: + log("GitHub не дал download_url") + return False + log(f"качаю {cfg.autocomplete_filename}") + run_ssh( + cfg, + host, + "mkdir -p {dir} && curl -fsSL -o {part} {url} && mv {part} {dest}".format( + dir=shlex.quote(dest_dir), + part=shlex.quote(dest + ".partial"), + url=shlex.quote(download_url), + dest=shlex.quote(dest), + ), + timeout=180, + ) + meta = { + "repo": cfg.autocomplete_github_repo, + "path": cfg.autocomplete_github_path, + "ref": cfg.autocomplete_github_ref, + "github_blob_sha": sha, + "filename": cfg.autocomplete_filename, + "fetched_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(), + "settings_applied": True, + } + put_text(cfg, host, meta_path, json.dumps(meta, indent=2) + "\n") + settings = f"{DATA}/Data/Settings.fds" + applied = False + if remote_exists(cfg, host, meta_path): + raw = run_ssh(cfg, host, f"cat {meta_path}", check=False) + try: + applied = bool(json.loads(raw).get("settings_applied")) + except json.JSONDecodeError: + applied = False + if not remote_exists(cfg, host, settings) or not applied: + fds = ( + "DefaultUser:\n" + " AutoComplete:\n" + f" Source: {cfg.autocomplete_filename}\n" + " EscapeParens: true\n" + ) + put_text(cfg, host, settings, fds) + log(f"Settings.fds AutoComplete.Source = {cfg.autocomplete_filename}") + return changed + + +def _download_url(host: str, version_id: int, file_info: dict) -> str: + raw = str(file_info.get("downloadUrl") or "") + if "civitai." in raw and "/api/download/" in raw: + # NSFW downloadUrl often points at .com — use the host that had files. + return f"https://{host}/api/download/models/{version_id}" + if raw.startswith("https://"): + return raw + return f"https://{host}/api/download/models/{version_id}" + + +def seed_civitai(cfg: Config, host: str, log: Log) -> None: + entries = parse_models(cfg.models_manifest) + if not cfg.civitai_api_token: + log("Civitai-seed пропущен: нет CIVITAI_API_TOKEN — дефолт SwarmUI") + return + if not entries: + log("Civitai-seed пропущен: манифест пуст — дефолт SwarmUI") + return + jobs = [] + for entry in entries: + vid = entry.version_id or (extract_version_id(entry.url) if entry.url else None) + if not vid: + log(f"пропуск {entry.kind}: нет version_id") + continue + try: + api_host, version = fetch_model_version(cfg.civitai_api_token, cfg.civitai_api_host, vid) + except CloudError as exc: + log(str(exc)) + continue + info = pick_primary_file(version) + if not info: + log(f"version {vid}: нет files[]") + continue + name = str(info.get("name") or f"{vid}.safetensors") + folder = MODEL_DIRS.get(entry.kind, entry.kind) + dest = f"{DATA}/Models/{folder}/{name}" + hashes = info.get("hashes") or {} + sha = str((hashes.get("SHA256") or hashes.get("sha256") or "")).lower() + stem = Path(name).stem + civitai_json = json.dumps(version, ensure_ascii=False, indent=2) + trained = version.get("trainedWords") or [] + phrase = trained[0] if isinstance(trained, list) and trained else "" + swarm = { + "name": stem, + "title": version.get("name") or stem, + "description": (version.get("description") or "")[:2000], + "trigger_phrase": phrase, + "author": ((version.get("model") or {}) if isinstance(version.get("model"), dict) else {}).get("name"), + "tags": version.get("tags") or [], + } + jobs.append( + { + "dest": dest, + "url": _download_url(api_host, vid, info), + "sha256": sha, + "sidecars": { + f"{stem}.civitai.json": civitai_json, + f"{stem}.swarm.json": json.dumps(swarm, ensure_ascii=False, indent=2), + }, + } + ) + if not jobs: + log("Civitai-seed: ни одной скачиваемой строки") + return + if not any(e.kind == "checkpoint" for e in entries): + log("в манифесте нет checkpoint — генерация может не стартовать") + put_text(cfg, host, "/tmp/gpu-rent-civitai-jobs.json", json.dumps(jobs, indent=2)) + put_text(cfg, host, "/tmp/gpu-rent-civitai.token", cfg.civitai_api_token + "\n", mode=0o600) + log(f"Civitai: качаю {len(jobs)} файл(ов) на VM") + run_python( + cfg, + host, + _pkg_text("civitai_fetch.py"), + remote_path="/tmp/gpu-rent-civitai_fetch.py", + timeout=7200, + log=log, + ) + + +def ensure_swarmui_running(cfg: Config, host: str, log: Log, restart: bool) -> None: + active = run_ssh(cfg, host, "systemctl is-active swarmui 2>/dev/null || true", check=False).strip() + if restart and active == "active": + log("systemctl restart swarmui (новые extensions/autocomplete)") + run_ssh(cfg, host, "sudo -n systemctl restart swarmui", timeout=120) + return + if active != "active": + log("systemctl start swarmui") + run_ssh(cfg, host, "sudo -n systemctl start swarmui", timeout=120) + run_ssh(cfg, host, "sudo -n systemctl enable swarmui", check=False) + + +def provision_vm(cfg: Config, host: str, log: Log) -> None: + restart = False + try: + if seed_extensions(cfg, host, log): + restart = True + except GpuRentError as exc: + log(f"extensions: {exc}") + raise + try: + if seed_autocomplete(cfg, host, log): + restart = True + except GpuRentError as exc: + log(f"autocomplete: {exc}") + seed_civitai(cfg, host, log) + push_tree(cfg, host, cfg.local_models_dir, f"{DATA}/Models", log, models=True) + push_tree(cfg, host, cfg.local_wildcards_dir, f"{DATA}/Data/Wildcards", log, models=False) + push_tree(cfg, host, cfg.local_workflows_dir, f"{DATA}/CustomWorkflows", log, models=False) + if cfg.pull_output: + pull_tree(cfg, host, f"{DATA}/Output", cfg.local_output_dir, log) + ensure_swarmui_running(cfg, host, log, restart=restart) + log("SwarmUI слушает 127.0.0.1:7801 — gpu-rent tunnel") diff --git a/src/gpu_rent/remote/__init__.py b/src/gpu_rent/remote/__init__.py new file mode 100644 index 0000000..1ecf614 --- /dev/null +++ b/src/gpu_rent/remote/__init__.py @@ -0,0 +1 @@ +"""Scripts uploaded to the GPU VM.""" diff --git a/src/gpu_rent/remote/bootstrap.sh b/src/gpu_rent/remote/bootstrap.sh new file mode 100644 index 0000000..cbbe62a --- /dev/null +++ b/src/gpu_rent/remote/bootstrap.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Idempotent gpu-rent first-boot on the VM. No Docker. Do not apt-upgrade the kernel. +set -euo pipefail + +SWARM_USER="${SWARM_USER:-ubuntu}" +SWARM_ROOT="/opt/swarmui" +DATA_ROOT="/mnt/swarm_data" +MARKER_DATA="${DATA_ROOT}/.gpu-rent-ready" +MARKER_BOOT="${SWARM_ROOT}/.gpu-rent-bootstrapped" +SWARM_REPO="https://github.com/mcmonkeyprojects/SwarmUI.git" + +log() { echo "[gpu-rent] $*"; } + +if [[ "$(id -u)" -ne 0 ]]; then + echo "нужен root (sudo -n bash $0)" >&2 + exit 1 +fi + +export DEBIAN_FRONTEND=noninteractive + +pick_data_disk() { + local root_src root_pk name type + root_src="$(findmnt -n -o SOURCE / || true)" + root_pk="$(lsblk -no PKNAME "$root_src" 2>/dev/null | head -n1 || true)" + if [[ -z "$root_pk" && -n "$root_src" ]]; then + root_pk="$(lsblk -no NAME "$root_src" 2>/dev/null | head -n1 | sed 's/[0-9]*$//' || true)" + fi + while read -r name type; do + [[ "$type" == "disk" ]] || continue + [[ -n "$root_pk" && "$name" == "$root_pk" ]] && continue + if findmnt "/dev/${name}" >/dev/null 2>&1; then + continue + fi + echo "/dev/${name}" + return 0 + done < <(lsblk -dn -o NAME,TYPE) + return 1 +} + +ensure_data_mount() { + mkdir -p "$DATA_ROOT" + if findmnt "$DATA_ROOT" >/dev/null 2>&1; then + log "data volume уже на ${DATA_ROOT}" + return 0 + fi + local disk fstype uuid + disk="$(pick_data_disk)" || { + log "нет второго диска — проверь BDM data volume" + exit 1 + } + fstype="$(blkid -s TYPE -o value "$disk" 2>/dev/null || true)" + if [[ -z "$fstype" ]]; then + if [[ -e "$MARKER_DATA" ]]; then + log "маркер есть, а FS на ${disk} нет — не mkfs, разбери вручную" + exit 1 + fi + log "mkfs.ext4 ${disk} (пустой data volume)" + mkfs.ext4 -F -L swarm-data "$disk" + fi + mount "$disk" "$DATA_ROOT" + uuid="$(blkid -s UUID -o value "$disk")" + if [[ -n "$uuid" ]] && ! grep -q "UUID=${uuid}" /etc/fstab; then + echo "UUID=${uuid} ${DATA_ROOT} ext4 defaults,nofail 0 2" >> /etc/fstab + fi + log "смонтирован ${disk} -> ${DATA_ROOT}" +} + +ensure_bind() { + local src="$1" dst="$2" + mkdir -p "$src" "$dst" + if ! findmnt "$dst" >/dev/null 2>&1; then + mount --bind "$src" "$dst" + fi + if ! grep -Fq " ${dst} " /etc/fstab; then + echo "${src} ${dst} none bind,nofail 0 0" >> /etc/fstab + fi +} + +log "пакеты (без upgrade ядра)" +apt-get update -qq +apt-get install -y -qq git python3 python3-venv python3-pip python3-full curl ca-certificates + +ensure_data_mount + +mkdir -p \ + "${DATA_ROOT}/Models" \ + "${DATA_ROOT}/Output" \ + "${DATA_ROOT}/Data" \ + "${DATA_ROOT}/Data/Autocompletions" \ + "${DATA_ROOT}/Data/Wildcards" \ + "${DATA_ROOT}/dlbackend" \ + "${DATA_ROOT}/Extensions" \ + "${DATA_ROOT}/DLNodes" \ + "${DATA_ROOT}/CustomWorkflows" + +if [[ ! -d "${SWARM_ROOT}/.git" ]]; then + log "clone SwarmUI -> ${SWARM_ROOT}" + mkdir -p "$(dirname "$SWARM_ROOT")" + git clone --depth 1 "$SWARM_REPO" "$SWARM_ROOT" +else + log "SwarmUI уже в ${SWARM_ROOT}" +fi + +if [[ ! -x /usr/share/dotnet/dotnet && ! -x "/home/${SWARM_USER}/.dotnet/dotnet" ]]; then + if [[ -x "${SWARM_ROOT}/launchtools/linux-dotnet-install.sh" ]]; then + log "ставим .NET SDK (скрипт SwarmUI)" + sudo -u "$SWARM_USER" bash "${SWARM_ROOT}/launchtools/linux-dotnet-install.sh" || true + fi +fi + +mkdir -p \ + "${SWARM_ROOT}/Models" \ + "${SWARM_ROOT}/Output" \ + "${SWARM_ROOT}/Data" \ + "${SWARM_ROOT}/dlbackend" \ + "${SWARM_ROOT}/src/Extensions" \ + "${SWARM_ROOT}/src/BuiltinExtensions/ComfyUIBackend/DLNodes" \ + "${SWARM_ROOT}/src/BuiltinExtensions/ComfyUIBackend/CustomWorkflows" + +ensure_bind "${DATA_ROOT}/Models" "${SWARM_ROOT}/Models" +ensure_bind "${DATA_ROOT}/Output" "${SWARM_ROOT}/Output" +ensure_bind "${DATA_ROOT}/Data" "${SWARM_ROOT}/Data" +ensure_bind "${DATA_ROOT}/dlbackend" "${SWARM_ROOT}/dlbackend" +ensure_bind "${DATA_ROOT}/Extensions" "${SWARM_ROOT}/src/Extensions" +ensure_bind "${DATA_ROOT}/DLNodes" "${SWARM_ROOT}/src/BuiltinExtensions/ComfyUIBackend/DLNodes" +ensure_bind "${DATA_ROOT}/CustomWorkflows" "${SWARM_ROOT}/src/BuiltinExtensions/ComfyUIBackend/CustomWorkflows" + +chown -R "${SWARM_USER}:${SWARM_USER}" "$DATA_ROOT" "$SWARM_ROOT" + +cat >/etc/systemd/system/swarmui.service <"$MARKER_DATA" +date -u +"%Y-%m-%dT%H:%M:%SZ" >"$MARKER_BOOT" +chown "${SWARM_USER}:${SWARM_USER}" "$MARKER_DATA" "$MARKER_BOOT" + +if command -v nvidia-smi >/dev/null 2>&1; then + nvidia-smi -L || true +fi +log "bootstrap ok (unit готов; swarmui старт после seed)" diff --git a/src/gpu_rent/remote/civitai_fetch.py b/src/gpu_rent/remote/civitai_fetch.py new file mode 100644 index 0000000..4cd9eb0 --- /dev/null +++ b/src/gpu_rent/remote/civitai_fetch.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Download Civitai files on the VM. Stdlib only. Token in a 600 file.""" +from __future__ import annotations + +import hashlib +import json +import sys +import urllib.parse +import urllib.request +from pathlib import Path + +TOKEN_PATH = Path("/tmp/gpu-rent-civitai.token") +JOBS_PATH = Path("/tmp/gpu-rent-civitai-jobs.json") +MARKER = Path("/mnt/swarm_data/.gpu-rent-models-seeded") + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + while True: + block = fh.read(1024 * 1024) + if not block: + break + digest.update(block) + return digest.hexdigest() + + +def download(url: str, dest: Path, token: str) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + partial = dest.with_suffix(dest.suffix + ".partial") + + class StripAuthRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + new = urllib.request.HTTPRedirectHandler.redirect_request( + self, req, fp, code, msg, headers, newurl + ) + if new is None: + return None + host = (urllib.parse.urlparse(new.full_url).hostname or "").lower() + if host.endswith("civitai.com") or host.endswith("civitai.red") or host.endswith("civitai.green"): + return new + # presigned S3 / CDN: token must not leave civitai + return urllib.request.Request(new.full_url, headers={"User-Agent": "gpu-rent/0.1"}) + + opener = urllib.request.build_opener(StripAuthRedirect) + req = urllib.request.Request( + url, headers={"Authorization": f"Bearer {token}", "User-Agent": "gpu-rent/0.1"} + ) + with opener.open(req, timeout=600) as response, partial.open("wb") as out: + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + out.write(chunk) + partial.replace(dest) + + +def main() -> int: + if not TOKEN_PATH.is_file(): + print("нет токена", file=sys.stderr) + return 1 + token = TOKEN_PATH.read_text(encoding="utf-8").strip() + jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8")) + failed = 0 + for job in jobs: + dest = Path(job["dest"]) + expect = (job.get("sha256") or "").lower() + if dest.is_file() and expect and sha256_path(dest).lower() == expect: + print(f"skip {dest}") + continue + try: + print(f"download {dest.name}") + download(job["url"], dest, token) + if expect: + got = sha256_path(dest).lower() + if got != expect: + dest.unlink(missing_ok=True) + raise RuntimeError(f"sha256 {got} != {expect}") + for extra_name, extra_text in (job.get("sidecars") or {}).items(): + extra = dest.parent / extra_name + extra.write_text(extra_text, encoding="utf-8") + except Exception as exc: + failed += 1 + print(f"FAIL {dest}: {exc}", file=sys.stderr) + TOKEN_PATH.unlink(missing_ok=True) + if failed: + return 1 + MARKER.parent.mkdir(parents=True, exist_ok=True) + MARKER.write_text("ok\n", encoding="utf-8") + print("civitai seed ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/gpu_rent/remote/clone_ext.py b/src/gpu_rent/remote/clone_ext.py new file mode 100644 index 0000000..7e8018b --- /dev/null +++ b/src/gpu_rent/remote/clone_ext.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Clone git extensions on the VM. Stdlib only. Token file optional.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +TOKEN_PATH = Path("/tmp/gpu-rent-git.token") +JOBS_PATH = Path("/tmp/gpu-rent-ext.json") +MARKER = Path("/mnt/swarm_data/.gpu-rent-extensions-seeded") + + +def strip_auth(url: str) -> str: + parts = urlsplit(url) + host = parts.hostname or "" + if parts.port: + host = f"{host}:{parts.port}" + return urlunsplit((parts.scheme, host, parts.path, parts.query, parts.fragment)) + + +def with_token(url: str, token: str) -> str: + if not token: + return url + if url.startswith("https://github.com/"): + return "https://x-access-token:" + token + "@github.com/" + url[len("https://github.com/") :] + if url.startswith("https://gitlab.com/"): + return "https://oauth2:" + token + "@gitlab.com/" + url[len("https://gitlab.com/") :] + return url + + +def run(argv: list[str], cwd: str | None = None) -> None: + subprocess.check_call(argv, cwd=cwd) + + +def is_sha(ref: str) -> bool: + ref = ref.strip() + return len(ref) == 40 and all(c in "0123456789abcdefABCDEF" for c in ref) + + +def clone_one(job: dict, token: str) -> None: + dest = Path(job["dest"]) + url = job["url"].strip() + ref = (job.get("ref") or "main").strip() + dest.parent.mkdir(parents=True, exist_ok=True) + authed = with_token(url, token) + if dest.is_dir() and (dest / ".git").is_dir(): + origin = subprocess.check_output(["git", "-C", str(dest), "remote", "get-url", "origin"], text=True).strip() + if strip_auth(origin) != strip_auth(url): + print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr) + raise SystemExit(2) + run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "origin"]) + run(["git", "-C", str(dest), "checkout", ref]) + print(f"updated {dest}") + return + if dest.exists(): + print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr) + raise SystemExit(2) + if is_sha(ref): + run(["git", "clone", "--recurse-submodules", authed, str(dest)]) + run(["git", "-C", str(dest), "fetch", "origin", ref]) + run(["git", "-C", str(dest), "checkout", ref]) + else: + try: + run(["git", "clone", "--recurse-submodules", "--depth", "1", "--branch", ref, authed, str(dest)]) + except subprocess.CalledProcessError: + run(["git", "clone", "--recurse-submodules", authed, str(dest)]) + run(["git", "-C", str(dest), "checkout", ref]) + print(f"cloned {dest}") + + +def main() -> int: + token = TOKEN_PATH.read_text(encoding="utf-8").strip() if TOKEN_PATH.is_file() else "" + jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8")) + failed = 0 + for job in jobs: + try: + clone_one(job, token) + except Exception as exc: + failed += 1 + print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr) + if TOKEN_PATH.is_file(): + TOKEN_PATH.unlink() + if failed: + return 1 + MARKER.parent.mkdir(parents=True, exist_ok=True) + MARKER.write_text("ok\n", encoding="utf-8") + print("extensions ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py new file mode 100644 index 0000000..440c674 --- /dev/null +++ b/src/gpu_rent/session.py @@ -0,0 +1,282 @@ +"""up / stop / destroy: GPU lifetime; SwarmUI bootstrap after SSH.""" + +from __future__ import annotations + +from collections.abc import Callable + +from gpu_rent.cloud import ( + create_gpu_server, + delete_floating_ip, + delete_server, + ensure_boot_volume, + ensure_data_volume, + ensure_floating_ip, + ensure_keypair, + ensure_network, + ensure_security_group, + guess_operator_cidr, + pick_existing_server, + server_status, + unshelve, +) +from gpu_rent.bootstrap import run_bootstrap +from gpu_rent.provision import provision_vm +from gpu_rent.config import Config +from gpu_rent.errors import CloudError, GpuRentError +from gpu_rent.inventory import ( + gpu_quota_from_compute, + pick_boot_image, + pick_volume_type, + resolve_flavor, +) +from gpu_rent.lock import SessionLock +from gpu_rent.os_client import ( + KEYPAIR_NAME, + SG_NAME, + compute_quotas, + connect, + iter_flavors, + iter_images, + iter_volume_types, +) +from gpu_rent.ssh_keys import ensure_ed25519 +from gpu_rent.ssh_ops import wait_ssh +from gpu_rent.state import SessionState, load_state, save_state, utc_now + +Log = Callable[[str], None] + + +def _log_default(msg: str) -> None: + print(msg) + + +def _require_gpu_quota(conn) -> None: + quota = compute_quotas(conn) + limit = gpu_quota_from_compute(quota) + if limit is not None and limit <= 0: + raise CloudError( + "квота GPU = 0. Напиши в поддержку Selectel (текст в docs/setup.md). " + "CLI не создаст сервер." + ) + + +def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> SessionState: + ip, fip_id = ensure_floating_ip( + conn, server, state.floating_ip_id, state.floating_ip, log + ) + state.floating_ip = ip + if fip_id: + state.floating_ip_id = fip_id + save_state(state) + wait_ssh(cfg, ip) + log(f"SSH {cfg.ssh_user}@{ip}") + run_bootstrap(cfg, ip, log) + provision_vm(cfg, ip, log) + state.bootstrapped = True + save_state(state) + log("gpu-rent tunnel — UI на localhost:17801") + log("gpu-rent stop — удалить compute, диски оставить") + return state + + +def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState: + conn = connect(cfg) + server = pick_existing_server(conn) + if not server: + raise CloudError("нечего adopt: нет сервера с тегом/именем gpu-rent") + state = load_state() + state.server_id = server.id + state.server_name = getattr(server, "name", None) + state.flavor_id = getattr(server, "flavor_id", None) or ( + (server.flavor or {}).get("id") if isinstance(getattr(server, "flavor", None), dict) else None + ) + state.phase = "ready_cloud" + state.availability_zone = cfg.gpu_rent_az + save_state(state) + log(f"подхватили {server.id} статус {server_status(server)}") + if server_status(server) == "ACTIVE": + _bind_access(conn, server, state, cfg, log) + return state + + +def cmd_up( + cfg: Config, + *, + no_spot: bool = False, + flavor: str | None = None, + yes: bool = False, + adopt: bool = False, + confirm: Callable[[str], bool] | None = None, + log: Log = _log_default, +) -> SessionState: + with SessionLock(): + if adopt: + return adopt_server(cfg, log=log) + conn = connect(cfg) + _require_gpu_quota(conn) + state = load_state() + existing = pick_existing_server(conn) + if existing: + status = server_status(existing) + state.server_id = existing.id + state.server_name = getattr(existing, "name", None) + if status == "ACTIVE": + log("сервер уже ACTIVE — второй GPU не создаём") + state.phase = "ready_cloud" + save_state(state) + _bind_access(conn, existing, state, cfg, log) + return state + if status in {"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"}: + existing = unshelve(conn, existing, log) + state.server_id = existing.id + state.phase = "ready_cloud" + state.unshelved_at = utc_now() + save_state(state) + _bind_access(conn, existing, state, cfg, log) + return state + raise CloudError(f"сервер gpu-rent в статусе {status} — разбери в панели") + + flavors = list(iter_flavors(conn)) + try: + picked = resolve_flavor( + flavors, + cfg.flavor_preference, + explicit=flavor, + default_id=cfg.default_flavor_id or None, + fallback=cfg.flavor_fallback, + ) + except ValueError as exc: + raise CloudError(str(exc)) from exc + + image = pick_boot_image(list(iter_images(conn))) + if not image: + raise CloudError("нет GPU-образа Ubuntu Driver 580/535 в Glance") + vtype = pick_volume_type(list(iter_volume_types(conn)), cfg.gpu_rent_az) + spot = cfg.default_spot and not no_spot + + prompt = ( + f"Создать {'preemptible ' if spot else ''}GPU {picked.name} " + f"в {cfg.gpu_rent_az}, образ {getattr(image, 'name', image.id)}, " + f"data {cfg.data_volume_size_gb} GB. Диск тарифицируется всегда. Продолжить?" + ) + if not yes: + ok = confirm(prompt) if confirm else False + if not ok: + raise GpuRentError("отменено") + + private_path, pub = ensure_ed25519(cfg.ssh_private_key_path) + del private_path + public_key = pub.read_text(encoding="utf-8") + ensure_keypair(conn, public_key, log) + + net, _subnet = ensure_network(conn, log) + cidr = guess_operator_cidr() + if cidr == "0.0.0.0/0": + log("не удалось узнать твой IP — SG откроет SSH с 0.0.0.0/0") + sg = ensure_security_group(conn, cidr, log) + + boot = ensure_boot_volume( + conn, + az=cfg.gpu_rent_az, + volume_type=vtype, + image_id=image.id, + snapshot_name=cfg.boot_snapshot_name, + existing_id=state.boot_volume_id or cfg.boot_volume_id, + log=log, + ) + data = ensure_data_volume( + conn, + az=cfg.gpu_rent_az, + volume_type=vtype, + size_gb=cfg.data_volume_size_gb, + existing_id=state.data_volume_id or cfg.data_volume_id, + log=log, + ) + + state.phase = "provisioning" + state.flavor_id = picked.id + state.flavor_name = picked.name + state.boot_volume_id = boot.id + state.data_volume_id = data.id + state.image_id = image.id + state.network_id = net.id + state.security_group_id = sg.id + state.availability_zone = cfg.gpu_rent_az + state.spot = spot + state.keypair_name = KEYPAIR_NAME + save_state(state) + + server = create_gpu_server( + conn, + flavor_id=picked.id, + net_id=net.id, + sg_name=SG_NAME, + boot_volume_id=boot.id, + data_volume_id=data.id, + az=cfg.gpu_rent_az, + spot=spot, + log=log, + ) + state.server_id = server.id + state.server_name = getattr(server, "name", None) + state.created_at = utc_now() + state.unshelved_at = None + state.phase = "ready_cloud" + save_state(state) + _bind_access(conn, server, state, cfg, log) + return state + + +def cmd_stop( + cfg: Config, + *, + destroy_disks: bool = False, + no_pull: bool = False, + log: Log = _log_default, +) -> SessionState: + with SessionLock(): + conn = connect(cfg) + state = load_state() + if cfg.pull_output and not no_pull and state.floating_ip: + try: + from gpu_rent.sync_files import pull_tree + + pull_tree(cfg, state.floating_ip, "/mnt/swarm_data/Output", cfg.local_output_dir, log) + except Exception as exc: + log(f"pull Output не удался: {exc}") + server = None + if state.server_id: + try: + server = conn.compute.get_server(state.server_id) + except Exception: + server = None + if server is None: + server = pick_existing_server(conn) + if server: + delete_server(conn, server, log) + else: + log("compute уже нет") + + if not cfg.keep_floating_ip: + delete_floating_ip(conn, state.floating_ip_id, state.floating_ip, log) + state.floating_ip = None + state.floating_ip_id = None + + if destroy_disks: + for vid in (state.data_volume_id, state.boot_volume_id): + if not vid: + continue + try: + conn.block_storage.delete_volume(vid, ignore_missing=True) + log(f"удалён том {vid}") + except Exception as exc: + log(f"том {vid} не удалился: {exc}") + state.boot_volume_id = None + state.data_volume_id = None + + state.server_id = None + state.server_name = None + state.phase = "idle" + save_state(state) + log("фаза idle" + ("" if destroy_disks else " (диски на месте)")) + return state diff --git a/src/gpu_rent/ssh_ops.py b/src/gpu_rent/ssh_ops.py new file mode 100644 index 0000000..202d7db --- /dev/null +++ b/src/gpu_rent/ssh_ops.py @@ -0,0 +1,274 @@ +"""Wait for SSH and run remote commands.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +import shlex +import socket +import subprocess +import time + +import paramiko + +from gpu_rent.config import Config +from gpu_rent.errors import CloudError + + +def wait_tcp(host: str, port: int, timeout: float = 300.0) -> None: + deadline = time.time() + timeout + last = None + while time.time() < deadline: + try: + with socket.create_connection((host, port), timeout=8): + return + except OSError as exc: + last = exc + time.sleep(4) + raise CloudError(f"TCP {host}:{port} не открылся за {int(timeout)} с ({last})") + + +def wait_ssh(cfg: Config, host: str, timeout: float = 420.0) -> None: + wait_tcp(host, 22, timeout=min(timeout, 240)) + deadline = time.time() + timeout + key = str(cfg.ssh_private_key_path) + last = None + while time.time() < deadline: + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + client.connect( + hostname=host, + username=cfg.ssh_user, + key_filename=key, + timeout=12, + banner_timeout=12, + auth_timeout=12, + ) + client.close() + return + except Exception as exc: + last = exc + try: + client.close() + except Exception: + pass + time.sleep(5) + raise CloudError(f"SSH {cfg.ssh_user}@{host} не принял ключ ({last})") + + +def ssh_argv(cfg: Config, host: str, remote: list[str] | None = None) -> list[str]: + cmd = [ + "ssh", + "-i", + str(cfg.ssh_private_key_path), + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "IdentitiesOnly=yes", + f"{cfg.ssh_user}@{host}", + ] + if remote: + cmd.append(" ".join(remote)) + return cmd + + +def interactive_ssh(cfg: Config, host: str) -> int: + argv = ssh_argv(cfg, host) + try: + return subprocess.call(argv) + except FileNotFoundError: + raise CloudError( + "Нет клиента ssh в PATH. Windows: установи OpenSSH Client " + f"или зайди так: ssh -i {cfg.ssh_private_key_path} {cfg.ssh_user}@{host}" + ) from None + + +def run_ssh(cfg: Config, host: str, command: str, timeout: int = 60, check: bool = True) -> str: + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + client.connect( + hostname=host, + username=cfg.ssh_user, + key_filename=str(cfg.ssh_private_key_path), + timeout=15, + ) + _stdin, stdout, stderr = client.exec_command(command, timeout=timeout) + out = stdout.read().decode("utf-8", errors="replace") + err = stderr.read().decode("utf-8", errors="replace") + code = stdout.channel.recv_exit_status() + if check and code != 0: + raise CloudError(f"SSH `{command}` exit {code}: {err or out}") + return out + finally: + client.close() + + +def run_script_sudo( + cfg: Config, + host: str, + script: str, + *, + remote_path: str, + timeout: int = 1800, + env: dict[str, str] | None = None, + log: Callable[[str], None] | None = None, +) -> str: + """Upload a script and run it with passwordless sudo, streaming stdout.""" + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + chunks: list[str] = [] + try: + client.connect( + hostname=host, + username=cfg.ssh_user, + key_filename=str(cfg.ssh_private_key_path), + timeout=20, + ) + sftp = client.open_sftp() + with sftp.file(remote_path, "w") as fh: + fh.write(script) + sftp.chmod(remote_path, 0o755) + sftp.close() + env_s = " ".join(f"{key}={value}" for key, value in (env or {}).items()) + prefix = f"sudo -n env {env_s} " if env_s else "sudo -n " + command = f"{prefix}bash {remote_path}" + _stdin, stdout, stderr = client.exec_command(command, timeout=timeout, get_pty=True) + while True: + line = stdout.readline() + if not line: + break + chunks.append(line) + if log: + log(line.rstrip("\n\r")) + code = stdout.channel.recv_exit_status() + err = stderr.read().decode("utf-8", errors="replace") if not stdout.channel.closed else "" + out = "".join(chunks) + if code != 0: + raise CloudError(f"remote script exit {code}: {err or out[-2000:]}") + return out + finally: + client.close() + + +def run_python( + cfg: Config, + host: str, + script: str, + *, + remote_path: str, + timeout: int = 1800, + log: Callable[[str], None] | None = None, +) -> str: + """Upload a Python script and run it as SSH user (not root).""" + client = _connect(cfg, host) + chunks: list[str] = [] + try: + sftp = client.open_sftp() + with sftp.file(remote_path, "w") as fh: + fh.write(script) + sftp.chmod(remote_path, 0o755) + sftp.close() + command = f"python3 {shlex.quote(remote_path)}" + _stdin, stdout, stderr = client.exec_command(command, timeout=timeout, get_pty=True) + while True: + line = stdout.readline() + if not line: + break + chunks.append(line) + if log: + log(line.rstrip("\n\r")) + code = stdout.channel.recv_exit_status() + err = stderr.read().decode("utf-8", errors="replace") if not stdout.channel.closed else "" + out = "".join(chunks) + if code != 0: + raise CloudError(f"remote python exit {code}: {err or out[-2000:]}") + return out + finally: + client.close() + + +def _connect(cfg: Config, host: str) -> paramiko.SSHClient: + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=host, + username=cfg.ssh_user, + key_filename=str(cfg.ssh_private_key_path), + timeout=20, + ) + return client + + +def put_text(cfg: Config, host: str, remote_path: str, text: str, mode: int = 0o644) -> None: + client = _connect(cfg, host) + try: + sftp = client.open_sftp() + _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/")) + with sftp.file(remote_path, "w") as fh: + fh.write(text) + sftp.chmod(remote_path, mode) + sftp.close() + finally: + client.close() + + +def put_file(cfg: Config, host: str, local: Path, remote_path: str) -> None: + client = _connect(cfg, host) + try: + sftp = client.open_sftp() + _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/")) + sftp.put(str(local), remote_path) + sftp.close() + finally: + client.close() + + +def get_file(cfg: Config, host: str, remote_path: str, local: Path) -> None: + local.parent.mkdir(parents=True, exist_ok=True) + client = _connect(cfg, host) + try: + sftp = client.open_sftp() + sftp.get(remote_path, str(local)) + sftp.close() + finally: + client.close() + + +def remote_exists(cfg: Config, host: str, remote_path: str) -> bool: + client = _connect(cfg, host) + try: + sftp = client.open_sftp() + try: + sftp.stat(remote_path) + return True + except FileNotFoundError: + return False + finally: + sftp.close() + finally: + client.close() + + +def remote_sha256(cfg: Config, host: str, remote_path: str) -> str | None: + out = run_ssh( + cfg, + host, + f"sha256sum {shlex.quote(remote_path)} 2>/dev/null | awk '{{print $1}}'", + check=False, + ).strip() + return out or None + + +def _sftp_mkdirs(sftp, remote_dir: str) -> None: + if not remote_dir or remote_dir == "/": + return + parts = [p for p in remote_dir.split("/") if p] + cur = "" + for part in parts: + cur += "/" + part + try: + sftp.stat(cur) + except FileNotFoundError: + sftp.mkdir(cur) diff --git a/src/gpu_rent/state.py b/src/gpu_rent/state.py index b1fa431..7e08ce1 100644 --- a/src/gpu_rent/state.py +++ b/src/gpu_rent/state.py @@ -28,6 +28,13 @@ class SessionState: created_at: str | None = None unshelved_at: str | None = None notes: dict[str, Any] = field(default_factory=dict) + floating_ip_id: str | None = None + network_id: str | None = None + security_group_id: str | None = None + image_id: str | None = None + availability_zone: str | None = None + spot: bool = True + bootstrapped: bool = False def to_dict(self) -> dict[str, Any]: return asdict(self) diff --git a/src/gpu_rent/sync_files.py b/src/gpu_rent/sync_files.py new file mode 100644 index 0000000..b16cf3b --- /dev/null +++ b/src/gpu_rent/sync_files.py @@ -0,0 +1,76 @@ +"""SFTP push/pull of local app folders. Never delete remote extras.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from gpu_rent.config import Config +from gpu_rent.payload import has_payload, iter_payload_files, model_push_set, sha256_file +from gpu_rent.ssh_ops import get_file, put_file, remote_sha256, run_ssh + +Log = Callable[[str], None] + + +def push_tree( + cfg: Config, + host: str, + local_root: Path, + remote_root: str, + log: Log, + *, + models: bool, +) -> int: + if not has_payload(local_root): + log(f"push {local_root.name}: пусто — skip") + return 0 + files = model_push_set(local_root) if models else iter_payload_files(local_root) + sent = 0 + for path in files: + rel = path.relative_to(local_root).as_posix() + remote = f"{remote_root.rstrip('/')}/{rel}" + local_hash = sha256_file(path) + remote_hash = remote_sha256(cfg, host, remote) + if remote_hash and remote_hash.lower() == local_hash.lower(): + continue + if models and not _is_weight_name(path.name): + # sidecar: warn if we somehow got here without weight — still send + pass + log(f"push {rel}") + put_file(cfg, host, path, remote) + sent += 1 + if sent == 0: + log(f"push {local_root.name}: всё уже на VM") + else: + log(f"push {local_root.name}: {sent} файл(ов)") + return sent + + +def _is_weight_name(name: str) -> bool: + lower = name.lower() + return lower.endswith((".safetensors", ".ckpt", ".pt", ".pth", ".bin", ".gguf", ".sft", ".onnx")) + + +def pull_tree(cfg: Config, host: str, remote_root: str, local_root: Path, log: Log) -> int: + listing = run_ssh( + cfg, + host, + f"find {remote_root} -type f 2>/dev/null | sed 's|^{remote_root}/||'", + check=False, + timeout=120, + ) + names = [line.strip() for line in listing.splitlines() if line.strip()] + pulled = 0 + for rel in names: + if rel.endswith("/.gitkeep") or rel.rsplit("/", 1)[-1] in {".gitkeep", "README.md"}: + continue + remote = f"{remote_root.rstrip('/')}/{rel}" + local = local_root / rel + remote_hash = remote_sha256(cfg, host, remote) + if local.is_file() and remote_hash and sha256_file(local).lower() == remote_hash.lower(): + continue + log(f"pull {rel}") + get_file(cfg, host, remote, local) + pulled += 1 + log(f"pull Output: {pulled} файл(ов)" if pulled else "pull Output: нечего забирать") + return pulled diff --git a/src/gpu_rent/tunnel.py b/src/gpu_rent/tunnel.py new file mode 100644 index 0000000..5b06b04 --- /dev/null +++ b/src/gpu_rent/tunnel.py @@ -0,0 +1,62 @@ +"""SSH local forward. Closing the tunnel does not stop the GPU.""" + +from __future__ import annotations + +import time +import webbrowser +from collections.abc import Callable + +from gpu_rent.config import Config +from gpu_rent.errors import CloudError + +Log = Callable[[str], None] + + +def run_tunnel( + cfg: Config, + host: str, + *, + open_browser: bool = False, + log: Log = print, + wait: Callable[[], None] | None = None, +) -> None: + try: + from sshtunnel import SSHTunnelForwarder + except ImportError as exc: + raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc + + local_port = cfg.swarmui_local_port + log(f"туннель 127.0.0.1:{local_port} -> {host}:7801") + log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop") + server = SSHTunnelForwarder( + (host, 22), + ssh_username=cfg.ssh_user, + ssh_pkey=str(cfg.ssh_private_key_path), + remote_bind_address=("127.0.0.1", 7801), + local_bind_address=("127.0.0.1", local_port), + set_keepalive=30, + ) + try: + server.start() + except Exception as exc: + raise CloudError( + f"не открыть туннель на {local_port}: {exc}. Порт занят локальным SwarmUI? " + "17801 должен быть свободен." + ) from exc + + url = f"http://127.0.0.1:{local_port}" + log(f"UI {url}") + log(f"API {url}/API/") + log(f"MCP {url}/mcp") + if open_browser: + webbrowser.open(url) + try: + if wait is not None: + wait() + return + while server.is_active: + time.sleep(1) + except KeyboardInterrupt: + log("туннель закрыт. GPU жив.") + finally: + server.stop() diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py new file mode 100644 index 0000000..e96b587 --- /dev/null +++ b/tests/test_bootstrap.py @@ -0,0 +1,16 @@ +from gpu_rent.bootstrap import bootstrap_script + + +def test_bootstrap_script_is_native_swarmui(): + script = bootstrap_script() + assert "docker.io" not in script + assert "docker run" not in script + assert "nvidia-container" not in script + assert "/opt/swarmui" in script + assert "/mnt/swarm_data" in script + assert "launch-linux.sh --launch_mode none --host 127.0.0.1 --port 7801" in script + assert "systemctl enable swarmui" in script + assert "mkfs.ext4" in script + assert "apt-get upgrade" not in script + assert ".gpu-rent-ready" in script + assert "src/BuiltinExtensions/ComfyUIBackend/DLNodes" in script diff --git a/tests/test_cloud.py b/tests/test_cloud.py new file mode 100644 index 0000000..1f69fa7 --- /dev/null +++ b/tests/test_cloud.py @@ -0,0 +1,43 @@ +from gpu_rent.cloud import ensure_floating_ip, server_floating_ip, server_status +from gpu_rent.lock import SessionLock +from gpu_rent.paths import lock_path + + +class Obj: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def test_server_floating_ip(): + server = Obj( + status="ACTIVE", + addresses={"gpu-rent": [{"addr": "192.168.77.10", "OS-EXT-IPS:type": "fixed"}, {"addr": "203.0.113.9", "OS-EXT-IPS:type": "floating"}]}, + ) + assert server_status(server) == "ACTIVE" + assert server_floating_ip(server) == "203.0.113.9" + + +def test_lock_roundtrip(): + with SessionLock(): + assert lock_path().is_file() + assert not lock_path().is_file() + + +def test_ensure_floating_ip_reuses_existing(): + class Net: + def ports(self, device_id=None): + return [Obj(id="p1")] + + def update_ip(self, existing_id, port_id=None): + assert existing_id == "old-fip" + assert port_id == "p1" + return Obj(floating_ip_address="9.9.9.9") + + def create_ip(self, floating_network_id=None): + raise AssertionError("must reuse FIP") + + conn = Obj(network=Net()) + server = Obj(id="s1", addresses={}) + ip, fip_id = ensure_floating_ip(conn, server, "old-fip", "9.9.9.9", lambda m: None) + assert ip == "9.9.9.9" + assert fip_id == "old-fip" diff --git a/tests/test_inventory.py b/tests/test_inventory.py index 092bc28..6567880 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -4,6 +4,7 @@ from gpu_rent.inventory import ( match_label, pick_boot_image, rank_flavors, + resolve_flavor, ) @@ -41,6 +42,38 @@ def test_gpu_quota_from_compute(): assert gpu_quota_from_compute({"GPU_limit": 2}) == 2 +def test_resolve_flavor_explicit_and_preference(): + flavors = [ + FakeFlavor("d", "RTX 4090 24GB", disabled=True), + FakeFlavor("a5", "RTX A5000 24GB"), + FakeFlavor("ok", "GPU 1x RTX 4090 24GB"), + ] + picked = resolve_flavor(flavors, ("4090-24", "a5000")) + assert picked.id == "ok" + explicit = resolve_flavor(flavors, ("a5000",), explicit="a5") + assert explicit.id == "a5" + + +def test_resolve_flavor_prefers_list_over_default_id(): + flavors = [ + FakeFlavor("a5", "RTX A5000 24GB"), + FakeFlavor("ok", "GPU 1x RTX 4090 24GB"), + ] + picked = resolve_flavor(flavors, ("4090-24", "a5000"), default_id="a5", fallback=True) + assert picked.id == "ok" + + +def test_resolve_flavor_no_fallback_requires_default(): + flavors = [FakeFlavor("ok", "GPU 1x RTX 4090 24GB")] + try: + resolve_flavor(flavors, ("4090-24",), fallback=False) + raise AssertionError("expected ValueError") + except ValueError: + pass + picked = resolve_flavor(flavors, ("a5000",), default_id="ok", fallback=False) + assert picked.id == "ok" + + def test_pick_boot_image_prefers_24_580_without_docker(): class Img: def __init__(self, name): diff --git a/tests/test_launchers.py b/tests/test_launchers.py new file mode 100644 index 0000000..8d80902 --- /dev/null +++ b/tests/test_launchers.py @@ -0,0 +1,19 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_launcher_files_exist(): + for name in ("gpu-rent.bat", "gpu-rent.ps1", "gpu-rent.sh"): + path = ROOT / name + assert path.is_file(), name + text = path.read_text(encoding="utf-8") + assert ".venv" in text or "gpu-rent.ps1" in text + assert "gpu_rent" in text or "gpu-rent.ps1" in text + + +def test_launchers_require_python_311(): + ps1 = (ROOT / "gpu-rent.ps1").read_text(encoding="utf-8") + sh = (ROOT / "gpu-rent.sh").read_text(encoding="utf-8") + assert "3, 11" in ps1 or "3.11" in ps1 + assert "3, 11" in sh diff --git a/tests/test_session.py b/tests/test_session.py new file mode 100644 index 0000000..1bf08e4 --- /dev/null +++ b/tests/test_session.py @@ -0,0 +1,132 @@ +import pytest + +from gpu_rent.config import load_config +from gpu_rent.errors import CloudError, GpuRentError +from gpu_rent.lock import SessionLock +from gpu_rent.paths import lock_path +from gpu_rent.session import cmd_stop, cmd_up +from gpu_rent.state import SessionState, save_state + + +class Server: + def __init__(self, status="ACTIVE", server_id="s1"): + self.id = server_id + self.name = "gpu-rent" + self.status = status + self.flavor = {"id": "f1"} + self.addresses = { + "gpu-rent": [{"addr": "203.0.113.9", "OS-EXT-IPS:type": "floating"}] + } + + +def _cfg(monkeypatch): + monkeypatch.setenv("OS_AUTH_URL", "https://example.invalid/identity/v3") + monkeypatch.setenv("OS_USER_DOMAIN_NAME", "999") + monkeypatch.setenv("OS_USERNAME", "svc") + monkeypatch.setenv("OS_PASSWORD", "secret") + monkeypatch.setenv("OS_PROJECT_ID", "proj") + monkeypatch.setenv("OS_REGION_NAME", "ru-7") + monkeypatch.setenv("GPU_RENT_AZ", "ru-7a") + return load_config(require_auth=True) + + +def test_cmd_up_refuses_zero_gpu_quota(monkeypatch): + monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) + monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 0}) + with pytest.raises(CloudError, match="квота GPU"): + cmd_up(_cfg(monkeypatch), yes=True) + + +def test_cmd_up_does_not_create_second_gpu(monkeypatch): + created = [] + monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) + monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 1}) + monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server()) + monkeypatch.setattr( + "gpu_rent.session.ensure_floating_ip", + lambda conn, server, existing_id, existing_addr, log: ("203.0.113.9", "fip1"), + ) + monkeypatch.setattr("gpu_rent.session.wait_ssh", lambda cfg, host, timeout=420.0: None) + monkeypatch.setattr("gpu_rent.session.run_bootstrap", lambda cfg, host, log: None) + monkeypatch.setattr("gpu_rent.session.provision_vm", lambda cfg, host, log: None) + monkeypatch.setattr( + "gpu_rent.session.create_gpu_server", + lambda *a, **k: created.append("created") or Server(), + ) + state = cmd_up(_cfg(monkeypatch), yes=True) + assert created == [] + assert state.server_id == "s1" + assert state.phase == "ready_cloud" + assert state.floating_ip == "203.0.113.9" + assert state.bootstrapped is True + + +def test_cmd_up_unshelves_expired(monkeypatch): + unshelved = [] + created = [] + monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) + monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 1}) + monkeypatch.setattr( + "gpu_rent.session.pick_existing_server", lambda conn: Server(status="EXPIRED") + ) + monkeypatch.setattr( + "gpu_rent.session.unshelve", + lambda conn, server, log: unshelved.append(server.id) or Server(status="ACTIVE"), + ) + monkeypatch.setattr( + "gpu_rent.session.ensure_floating_ip", + lambda conn, server, existing_id, existing_addr, log: ("203.0.113.9", "fip1"), + ) + monkeypatch.setattr("gpu_rent.session.wait_ssh", lambda cfg, host, timeout=420.0: None) + monkeypatch.setattr("gpu_rent.session.run_bootstrap", lambda cfg, host, log: None) + monkeypatch.setattr("gpu_rent.session.provision_vm", lambda cfg, host, log: None) + monkeypatch.setattr( + "gpu_rent.session.create_gpu_server", + lambda *a, **k: created.append("created"), + ) + state = cmd_up(_cfg(monkeypatch), yes=True) + assert unshelved == ["s1"] + assert created == [] + assert state.unshelved_at + assert state.phase == "ready_cloud" + assert state.bootstrapped is True + + +def test_cmd_stop_deletes_compute_keeps_disks(monkeypatch): + deleted = [] + monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) + monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server()) + monkeypatch.setattr( + "gpu_rent.session.delete_server", + lambda conn, server, log: deleted.append(server.id), + ) + monkeypatch.setattr( + "gpu_rent.session.delete_floating_ip", + lambda conn, fip_id, address, log: deleted.append("fip"), + ) + save_state( + SessionState(server_id="s1", boot_volume_id="b1", data_volume_id="d1", floating_ip="1.1.1.1") + ) + + class Conn: + class compute: + @staticmethod + def get_server(sid): + return Server(server_id=sid) + + monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: Conn()) + state = cmd_stop(_cfg(monkeypatch)) + assert "s1" in deleted + assert state.phase == "idle" + assert state.server_id is None + assert state.boot_volume_id == "b1" + assert state.data_volume_id == "d1" + + +def test_lock_busy(monkeypatch): + lock_path().parent.mkdir(parents=True, exist_ok=True) + lock_path().write_text("1", encoding="utf-8") + monkeypatch.setattr("gpu_rent.lock._pid_alive", lambda pid: True) + monkeypatch.setattr("gpu_rent.lock.os.getpid", lambda: 99) + with pytest.raises(GpuRentError, match="уже работает"): + SessionLock().__enter__()