diff --git a/.gitignore b/.gitignore index 3c3a224..9fdb1f3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .env .env.* !.env.example +gpu-rent.vars models.yaml extensions.yaml diff --git a/docs/cli.md b/docs/cli.md index 29df31e..f365497 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,7 +8,8 @@ | --- | --- | | `gpu-rent flavors` | Скан `SCAN_POOLS` (ru-6 multizone…) × `FLAVOR_PREFERENCE`, затем список в текущем `OS_REGION_NAME` | | `gpu-rent doctor` | Preflight **без** create: Keystone, квота GPU, flavor в AZ, диски, Civitai token+`.red`, манифесты, SSH-ключ. Код выхода ≠ 0, если сессию нельзя начать | -| `gpu-rent up` / `up --yes` | Preflight → create/unshelve → bootstrap → **туннель** `localhost:17801` + URL панели; Ctrl+C закрывает туннель | +| `gpu-rent up` / `up --yes` | Preflight → create/unshelve → bootstrap → git update SwarmUI/extensions → **туннель** `localhost:17801`; Ctrl+C закрывает туннель | +| `gpu-rent up --no-update` | Без `git pull` SwarmUI и extensions (только недостающие clone) | | `gpu-rent up --no-tunnel` | Только облако + bootstrap, без локального проброса | | `gpu-rent up --no-spot` | Обычный (не preemptible) сервер | | `gpu-rent up --flavor … --yes` | Без вопросов. `--flavor` бьёт список фоллбека. `--yes` без `--flavor` берёт первый доступный из `FLAVOR_PREFERENCE` | @@ -100,7 +101,25 @@ Hold killer: gpu-rent hold ## Конфигурация -`.env` в корне репозитория (рядом с `env.example`): +| Файл | Назначение | +| --- | --- | +| `.env` | Секреты и OpenStack (`OS_*`, токены). Не в git | +| `gpu-rent.vars` | Параметры запуска / несекретные дефолты. Читают `gpu-rent.ps1` / `.sh` / `.bat` и CLI. Пример: `gpu-rent.vars.example` | +| `models.yaml` / `extensions.yaml` | Манифесты | + +`.env` в корне репозитория (рядом с `env.example`). + +В `gpu-rent.vars` (создаётся из example при первом запуске): + +```env +# Двойной клик / запуск без аргументов: +GPU_RENT_DEFAULT_ARGS=up --yes + +# Дописать ко всем вызовам: +# GPU_RENT_EXTRA_ARGS=--no-update + +UPDATE_GIT=true +``` ```env OS_AUTH_URL=https://cloud.api.selcloud.ru/identity/v3 diff --git a/docs/decisions.md b/docs/decisions.md index 15cebd7..3bccc13 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -20,6 +20,7 @@ | Idle-killer | 30 минут пустой очереди генерации. Открытый браузер не продлевает жизнь | | Льгота после boot | Killer молчит во время clone/seed/push, hold, качалки в UI SwarmUI, пока backend не Idle, и 45 мин после ACTIVE/unshelve | | `up` / `tunnel` | `up` по умолчанию после ready открывает туннель `:17801`, печатает URL и ждёт. `--no-tunnel` — только облако. Ctrl+C на туннеле GPU не гасит (`stop` отдельно). Команда `tunnel` остаётся для повторного входа | +| Git update | На каждом `up` по умолчанию: `git pull` SwarmUI + репы из `extensions.yaml` + уже установленные на data (`Extensions`/`DLNodes`). `--no-update` или `UPDATE_GIT=false` — не тянуть | | Data-диск | Старт **100 GB**, рост через resize вверх (вниз Selectel не умеет) | | SSH | CLI генерирует `/.gpu-rent/id_ed25519` без passphrase и сам регистрирует keypair | | Локальные файлы | Всё в корне репозитория: `.env`, `models.yaml`, `extensions.yaml`; runtime (`state.json`, lock, SSH) в `/.gpu-rent/`. Не `%USERPROFILE%\.gpu-rent` | diff --git a/docs/extensions.md b/docs/extensions.md index 2424e6f..d35f177 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -1,6 +1,12 @@ # Расширения: git-репы на первый bootstrap -Срабатывает вместе с первым диском и первым инстансом, **до** первого старта SwarmUI (C# extensions компилируются при запуске). Повторный `up` на уже засеянный диск только догоняет новые строки манифеста (идемпотентный `git fetch`). +Срабатывает на каждом `up` (и `seed-extensions`): + +- нет каталога → `git clone`; +- каталог есть → по умолчанию `fetch` + `reset --hard` на `ref` (или текущую ветку для уже установленных вне yaml); +- `--no-update` / `UPDATE_GIT=false` → существующие репы не трогаем, только недостающие clone. + +Также обновляется сам SwarmUI в `/opt/swarmui` (тот же флаг). Два разных мира — не путать: diff --git a/env.example b/env.example index fbb75af..214ddab 100644 --- a/env.example +++ b/env.example @@ -37,6 +37,8 @@ AUTOCOMPLETE_GITHUB_REF=main AUTOCOMPLETE_FILENAME=danbooru.csv SWARMUI_LOCAL_PORT=17801 +# git pull SwarmUI + extensions on each up (default true). CLI: --no-update +UPDATE_GIT=true DEFAULT_FLAVOR_ID= FLAVOR_PREFERENCE=4090-24,4090-48,a5000,a100-40 diff --git a/gpu-rent.ps1 b/gpu-rent.ps1 index a3375d0..6f6d87e 100644 --- a/gpu-rent.ps1 +++ b/gpu-rent.ps1 @@ -12,6 +12,33 @@ try { } catch { } +function Import-GpuRentVars { + param([Parameter(Mandatory = $true)][string]$Path) + if (-not (Test-Path -LiteralPath $Path)) { return } + Get-Content -LiteralPath $Path -Encoding UTF8 | ForEach-Object { + $line = $_.Trim() + if (-not $line -or $line.StartsWith("#")) { return } + if ($line -match '^(?i)export\s+') { + $line = $line.Substring($Matches[0].Length).Trim() + } + $eq = $line.IndexOf("=") + if ($eq -lt 1) { return } + $key = $line.Substring(0, $eq).Trim() + $val = $line.Substring($eq + 1).Trim() + if ($val.Length -ge 2) { + $q = $val[0] + if (($q -eq '"' -or $q -eq "'") -and $val[-1] -eq $q) { + $val = $val.Substring(1, $val.Length - 2) + } + } + if (-not $key) { return } + $existing = [Environment]::GetEnvironmentVariable($key, "Process") + if ([string]::IsNullOrEmpty($existing)) { + Set-Item -Path "Env:$key" -Value $val + } + } +} + function Test-Python311 { param( [Parameter(Mandatory = $true)][string]$Exe, @@ -91,7 +118,18 @@ function Copy-IfMissing { } Copy-IfMissing (Join-Path $Root "models.example.yaml") (Join-Path $Root "models.yaml") "models.yaml" Copy-IfMissing (Join-Path $Root "extensions.example.yaml") (Join-Path $Root "extensions.yaml") "extensions.yaml" +Copy-IfMissing (Join-Path $Root "gpu-rent.vars.example") (Join-Path $Root "gpu-rent.vars") "gpu-rent.vars" + +Import-GpuRentVars (Join-Path $Root "gpu-rent.vars") + +$InvokeArgs = @($args) +if ($InvokeArgs.Count -eq 0 -and $env:GPU_RENT_DEFAULT_ARGS) { + $InvokeArgs = @($env:GPU_RENT_DEFAULT_ARGS -split '\s+' | Where-Object { $_ }) +} +if ($env:GPU_RENT_EXTRA_ARGS) { + $InvokeArgs += @($env:GPU_RENT_EXTRA_ARGS -split '\s+' | Where-Object { $_ }) +} $ErrorActionPreference = "Continue" -& $VenvPy -m gpu_rent @args +& $VenvPy -m gpu_rent @InvokeArgs exit $LASTEXITCODE diff --git a/gpu-rent.sh b/gpu-rent.sh index cfe56b0..a3aceb9 100644 --- a/gpu-rent.sh +++ b/gpu-rent.sh @@ -5,6 +5,37 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$ROOT" +load_vars_file() { + local file="$1" + [[ -f "$file" ]] || return 0 + while IFS= read -r line || [[ -n "$line" ]]; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "$line" || "$line" == \#* ]] && continue + if [[ "$line" == [eE][xX][pP][oO][rR][tT][[:space:]]* ]]; then + line="${line#*[eE][xX][pP][oO][rR][tT]}" + line="${line#"${line%%[![:space:]]*}"}" + fi + [[ "$line" != *=* ]] && continue + local key="${line%%=*}" + local val="${line#*=}" + key="${key%"${key##*[![:space:]]}"}" + key="${key#"${key%%[![:space:]]*}"}" + val="${val#"${val%%[![:space:]]*}"}" + val="${val%"${val##*[![:space:]]}"}" + if [[ ${#val} -ge 2 ]]; then + local q="${val:0:1}" + if [[ ( "$q" == '"' || "$q" == "'" ) && "${val: -1}" == "$q" ]]; then + val="${val:1:${#val}-2}" + fi + fi + [[ -z "$key" ]] && continue + if [[ -z "${!key+x}" || -z "${!key}" ]]; then + export "$key=$val" + fi + done <"$file" +} + ok_py() { local exe="$1" shift || true @@ -64,5 +95,21 @@ if [[ ! -f "$ROOT/extensions.yaml" && -f "$ROOT/extensions.example.yaml" ]]; the cp "$ROOT/extensions.example.yaml" "$ROOT/extensions.yaml" echo "gpu-rent: created extensions.yaml" fi +if [[ ! -f "$ROOT/gpu-rent.vars" && -f "$ROOT/gpu-rent.vars.example" ]]; then + cp "$ROOT/gpu-rent.vars.example" "$ROOT/gpu-rent.vars" + echo "gpu-rent: created gpu-rent.vars" +fi -exec "$VENV_PY" -m gpu_rent "$@" +load_vars_file "$ROOT/gpu-rent.vars" + +ARGS=("$@") +if [[ ${#ARGS[@]} -eq 0 && -n "${GPU_RENT_DEFAULT_ARGS:-}" ]]; then + # shellcheck disable=SC2206 + ARGS=(${GPU_RENT_DEFAULT_ARGS}) +fi +if [[ -n "${GPU_RENT_EXTRA_ARGS:-}" ]]; then + # shellcheck disable=SC2206 + ARGS+=(${GPU_RENT_EXTRA_ARGS}) +fi + +exec "$VENV_PY" -m gpu_rent "${ARGS[@]}" diff --git a/gpu-rent.vars.example b/gpu-rent.vars.example new file mode 100644 index 0000000..3fddc39 --- /dev/null +++ b/gpu-rent.vars.example @@ -0,0 +1,22 @@ +# gpu-rent.vars — локальные параметры запуска (не секреты). +# Скопируй в gpu-rent.vars и правь. Файл в .gitignore. +# Читают: gpu-rent.ps1 / .bat / .sh и python -m gpu_rent (после .env). +# Секреты (OS_PASSWORD, CIVITAI_*) — только в .env. + +# Если запуск без аргументов (двойной клик / .\gpu-rent.ps1): +# GPU_RENT_DEFAULT_ARGS=up --yes + +# Доп. флаги ко ВСЕМ вызовам (после твоих аргументов): +# GPU_RENT_EXTRA_ARGS= + +# Те же имена, что в .env / env.example (несекретные дефолты): +# UPDATE_GIT=true +# DEFAULT_SPOT=true +# SCAN_POOLS=ru-6,ru-7 +# FLAVOR_PREFERENCE=4090-24,4090-48,a5000,a100-40 +# SWARMUI_LOCAL_PORT=17801 +# IDLE_MINUTES=30 +# IDLE_GRACE_MINUTES=45 +# NOTIFY_READY=true +# PULL_OUTPUT=false +# GPU_RENT_SSH_CIDR=0.0.0.0/0 diff --git a/src/gpu_rent/bootstrap.py b/src/gpu_rent/bootstrap.py index d1619db..fdd86f4 100644 --- a/src/gpu_rent/bootstrap.py +++ b/src/gpu_rent/bootstrap.py @@ -16,8 +16,12 @@ def bootstrap_script() -> str: return files("gpu_rent.remote").joinpath("bootstrap.sh").read_text(encoding="utf-8") -def run_bootstrap(cfg: Config, host: str, log: Log) -> None: +def run_bootstrap(cfg: Config, host: str, log: Log, *, update: bool = True) -> None: log("bootstrap SwarmUI на VM (идемпотентно, без Docker)") + if update: + log("git update: SwarmUI on") + else: + log("git update: SwarmUI off (--no-update)") script = bootstrap_script() out = run_script_sudo( cfg, @@ -25,7 +29,10 @@ def run_bootstrap(cfg: Config, host: str, log: Log) -> None: script, remote_path="/tmp/gpu-rent-bootstrap.sh", timeout=1800, - env={"SWARM_USER": cfg.ssh_user}, + env={ + "SWARM_USER": cfg.ssh_user, + "GPU_RENT_UPDATE_GIT": "1" if update else "0", + }, log=log, ) if "bootstrap ok" not in out: diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index c817ff0..1fd2f50 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -301,6 +301,11 @@ def up( "--open/--no-open", help="После туннеля открыть браузер на 17801 (по умолчанию да)", ), + no_update: bool = typer.Option( + False, + "--no-update", + help="Не делать git pull SwarmUI и установленных extensions", + ), ) -> None: """Create/unshelve GPU, bootstrap SwarmUI, по умолчанию туннель на :17801.""" try: @@ -319,6 +324,7 @@ def up( flavor=flavor, yes=yes, adopt=adopt, + update=False if no_update else None, confirm=confirm, log=lambda m: console.print(m), ) diff --git a/src/gpu_rent/config.py b/src/gpu_rent/config.py index ef88e14..f181c25 100644 --- a/src/gpu_rent/config.py +++ b/src/gpu_rent/config.py @@ -17,7 +17,9 @@ from gpu_rent.paths import ( migrate_legacy_if_needed, models_manifest_path, runtime_dir, + vars_path, ) +from gpu_rent.varsfile import apply_vars_file def _as_bool(value: str | None, default: bool) -> bool: @@ -76,6 +78,7 @@ class Config: swarmui_local_port: int swarmui_image: str + update_git: bool default_flavor_id: str flavor_preference: tuple[str, ...] @@ -108,6 +111,9 @@ def load_config(*, require_auth: bool = True) -> Config: if env_file.is_file(): load_dotenv(env_file, override=False) + # Non-secret launch defaults (gpu-rent.vars). Do not override .env / real env. + apply_vars_file(vars_path(), override=False) + missing: list[str] = [] required = ( "OS_AUTH_URL", @@ -180,6 +186,7 @@ def load_config(*, require_auth: bool = True) -> Config: autocomplete_filename=(os.environ.get("AUTOCOMPLETE_FILENAME") or "danbooru.csv").strip(), swarmui_local_port=_as_int(os.environ.get("SWARMUI_LOCAL_PORT"), 17801), swarmui_image=(os.environ.get("SWARMUI_IMAGE") or "").strip(), + update_git=_as_bool(os.environ.get("UPDATE_GIT"), True), default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(), flavor_preference=_csv( os.environ.get("FLAVOR_PREFERENCE"), diff --git a/src/gpu_rent/paths.py b/src/gpu_rent/paths.py index 0b46eb1..ff9b14f 100644 --- a/src/gpu_rent/paths.py +++ b/src/gpu_rent/paths.py @@ -40,6 +40,14 @@ def extensions_manifest_path() -> Path: return app_root() / "extensions.yaml" +def vars_path() -> Path: + return app_root() / "gpu-rent.vars" + + +def vars_example_path() -> Path: + return app_root() / "gpu-rent.vars.example" + + def state_path() -> Path: return runtime_dir() / "state.json" diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index c90c443..f18c230 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -33,9 +33,9 @@ def _pkg_text(name: str) -> str: return files("gpu_rent.remote").joinpath(name).read_text(encoding="utf-8") -def seed_extensions(cfg: Config, host: str, log: Log) -> bool: +def seed_extensions(cfg: Config, host: str, log: Log, *, update: bool = True) -> bool: repos = parse_extensions(cfg.extensions_manifest) - if not repos: + if not repos and not update: log("extensions.yaml пуст — стоковый SwarmUI") return False jobs = [] @@ -49,9 +49,13 @@ def seed_extensions(cfg: Config, host: str, log: Log) -> bool: } ) put_text(cfg, host, "/tmp/gpu-rent-ext.json", json.dumps(jobs, indent=2)) + put_text(cfg, host, "/tmp/gpu-rent-update-git", "1\n" if update else "0\n") if cfg.git_token: put_text(cfg, host, "/tmp/gpu-rent-git.token", cfg.git_token + "\n", mode=0o600) - log(f"clone {len(jobs)} git-реп на data volume") + if update: + log(f"extensions: clone/update {len(jobs)} из yaml + установленные на data") + else: + log(f"extensions: только недостающие из yaml ({len(jobs)}), без git pull (--no-update)") out = run_python( cfg, host, @@ -224,7 +228,10 @@ def seed_civitai(cfg: Config, host: str, log: Log) -> None: log("в манифесте нет checkpoint — генерация может не стартовать") put_text(cfg, host, "/tmp/gpu-rent-civitai-jobs.json", json.dumps(jobs, indent=2)) put_text(cfg, host, "/tmp/gpu-rent-civitai.token", cfg.civitai_api_token + "\n", mode=0o600) - log(f"Civitai: качаю {len(jobs)} файл(ов) на VM") + log( + f"Civitai: {len(jobs)} в манифесте — на VM качаю отсутствующие " + f"(уже есть + sha → skip; прогресс [N/{len(jobs)}])" + ) run_python( cfg, host, @@ -254,10 +261,11 @@ def provision_vm( *, conn=None, server_id: str | None = None, + update: bool = True, ) -> None: - restart = False + restart = bool(update) try: - if seed_extensions(cfg, host, log): + if seed_extensions(cfg, host, log, update=update): restart = True except GpuRentError as exc: log(f"extensions: {exc}") diff --git a/src/gpu_rent/remote/bootstrap.sh b/src/gpu_rent/remote/bootstrap.sh index cbbe62a..4e940ab 100644 --- a/src/gpu_rent/remote/bootstrap.sh +++ b/src/gpu_rent/remote/bootstrap.sh @@ -97,8 +97,21 @@ if [[ ! -d "${SWARM_ROOT}/.git" ]]; then log "clone SwarmUI -> ${SWARM_ROOT}" mkdir -p "$(dirname "$SWARM_ROOT")" git clone --depth 1 "$SWARM_REPO" "$SWARM_ROOT" +elif [[ "${GPU_RENT_UPDATE_GIT:-1}" == "1" ]]; then + log "обновляю SwarmUI в ${SWARM_ROOT}" + branch="$(git -C "$SWARM_ROOT" remote show origin 2>/dev/null | sed -n '/HEAD branch/s/.*: //p' || true)" + branch="${branch:-master}" + # shallow clone: deepen tip of default branch + git -C "$SWARM_ROOT" fetch --depth 1 origin "$branch" || git -C "$SWARM_ROOT" fetch --depth 1 origin + if git -C "$SWARM_ROOT" rev-parse --verify -q "origin/${branch}" >/dev/null; then + git -C "$SWARM_ROOT" checkout -B "$branch" "origin/${branch}" + git -C "$SWARM_ROOT" reset --hard "origin/${branch}" + else + git -C "$SWARM_ROOT" pull --ff-only || true + fi + log "SwarmUI @ $(git -C "$SWARM_ROOT" rev-parse --short HEAD)" else - log "SwarmUI уже в ${SWARM_ROOT}" + log "SwarmUI уже в ${SWARM_ROOT} (update off)" fi if [[ ! -x /usr/share/dotnet/dotnet && ! -x "/home/${SWARM_USER}/.dotnet/dotnet" ]]; then diff --git a/src/gpu_rent/remote/civitai_fetch.py b/src/gpu_rent/remote/civitai_fetch.py index 4cd9eb0..4907bfe 100644 --- a/src/gpu_rent/remote/civitai_fetch.py +++ b/src/gpu_rent/remote/civitai_fetch.py @@ -25,6 +25,19 @@ def sha256_path(path: Path) -> str: return digest.hexdigest() +def should_skip(dest: Path, expect_sha: str) -> tuple[bool, str]: + """Return (skip, reason). Existing file with matching sha — or any non-empty if no sha.""" + if not dest.is_file() or dest.stat().st_size <= 0: + return False, "" + expect = (expect_sha or "").lower() + if not expect: + return True, "уже есть" + got = sha256_path(dest).lower() + if got == expect: + return True, "уже есть (sha ok)" + return False, "sha не совпал — перекачиваю" + + def download(url: str, dest: Path, token: str) -> None: dest.parent.mkdir(parents=True, exist_ok=True) partial = dest.with_suffix(dest.suffix + ".partial") @@ -61,16 +74,30 @@ def main() -> int: return 1 token = TOKEN_PATH.read_text(encoding="utf-8").strip() jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8")) + total = len(jobs) failed = 0 - for job in jobs: + skipped = 0 + downloaded = 0 + for index, job in enumerate(jobs, start=1): dest = Path(job["dest"]) expect = (job.get("sha256") or "").lower() - if dest.is_file() and expect and sha256_path(dest).lower() == expect: - print(f"skip {dest}") + prefix = f"[{index}/{total}]" + skip, reason = should_skip(dest, expect) + if skip: + skipped += 1 + print(f"{prefix} {reason}: {dest.name}") + # refresh sidecars even on skip + for extra_name, extra_text in (job.get("sidecars") or {}).items(): + extra = dest.parent / extra_name + extra.parent.mkdir(parents=True, exist_ok=True) + extra.write_text(extra_text, encoding="utf-8") continue + if reason: + print(f"{prefix} {reason}: {dest.name}") try: - print(f"download {dest.name}") + print(f"{prefix} качаю: {dest.name}") download(job["url"], dest, token) + downloaded += 1 if expect: got = sha256_path(dest).lower() if got != expect: @@ -81,8 +108,9 @@ def main() -> int: extra.write_text(extra_text, encoding="utf-8") except Exception as exc: failed += 1 - print(f"FAIL {dest}: {exc}", file=sys.stderr) + print(f"{prefix} FAIL {dest.name}: {exc}", file=sys.stderr) TOKEN_PATH.unlink(missing_ok=True) + print(f"Civitai итог: скачано {downloaded}, пропущено {skipped}, ошибок {failed} (из {total})") if failed: return 1 MARKER.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/gpu_rent/remote/clone_ext.py b/src/gpu_rent/remote/clone_ext.py index de06373..6dac1f4 100644 --- a/src/gpu_rent/remote/clone_ext.py +++ b/src/gpu_rent/remote/clone_ext.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 -"""Clone git extensions on the VM. Stdlib only. Token file optional.""" +"""Clone/update git extensions on the VM. Stdlib only. Token file optional.""" from __future__ import annotations import json +import os import subprocess import sys from pathlib import Path @@ -10,7 +11,12 @@ from urllib.parse import urlsplit, urlunsplit TOKEN_PATH = Path("/tmp/gpu-rent-git.token") JOBS_PATH = Path("/tmp/gpu-rent-ext.json") +UPDATE_PATH = Path("/tmp/gpu-rent-update-git") MARKER = Path("/mnt/swarm_data/.gpu-rent-extensions-seeded") +EXTRA_ROOTS = ( + Path("/mnt/swarm_data/Extensions"), + Path("/mnt/swarm_data/DLNodes"), +) def strip_auth(url: str) -> str: @@ -35,25 +41,64 @@ def run(argv: list[str], cwd: str | None = None) -> None: subprocess.check_call(argv, cwd=cwd) +def out(argv: list[str], cwd: str | None = None) -> str: + return subprocess.check_output(argv, cwd=cwd, text=True).strip() + + def is_sha(ref: str) -> bool: ref = ref.strip() return len(ref) == 40 and all(c in "0123456789abcdefABCDEF" for c in ref) -def clone_one(job: dict, token: str) -> None: +def do_update() -> bool: + if UPDATE_PATH.is_file(): + return UPDATE_PATH.read_text(encoding="utf-8").strip() not in {"0", "false", "no", "off"} + return (os.environ.get("GPU_RENT_UPDATE_GIT") or "1").strip() not in {"0", "false", "no", "off"} + + +def fetch_and_checkout(dest: Path, ref: str) -> None: + run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"]) + if is_sha(ref): + run(["git", "-C", str(dest), "checkout", "--detach", ref]) + print(f"updated {dest} @ {ref[:12]}") + return + run(["git", "-C", str(dest), "checkout", ref]) + # Move branch tip to remote (shallow-friendly). + try: + run(["git", "-C", str(dest), "reset", "--hard", f"origin/{ref}"]) + except subprocess.CalledProcessError: + run(["git", "-C", str(dest), "pull", "--ff-only", "origin", ref]) + print(f"updated {dest} ({ref})") + + +def update_tracking_branch(dest: Path) -> None: + branch = out(["git", "-C", str(dest), "rev-parse", "--abbrev-ref", "HEAD"]) + if not branch or branch == "HEAD": + print(f"skip detached {dest}") + return + run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"]) + try: + run(["git", "-C", str(dest), "reset", "--hard", f"origin/{branch}"]) + except subprocess.CalledProcessError: + run(["git", "-C", str(dest), "pull", "--ff-only"]) + print(f"updated installed {dest} ({branch})") + + +def clone_one(job: dict, token: str, update: bool) -> None: dest = Path(job["dest"]) url = job["url"].strip() ref = (job.get("ref") or "main").strip() dest.parent.mkdir(parents=True, exist_ok=True) authed = with_token(url, token) if dest.is_dir() and (dest / ".git").is_dir(): - origin = subprocess.check_output(["git", "-C", str(dest), "remote", "get-url", "origin"], text=True).strip() + origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"]) if strip_auth(origin) != strip_auth(url): print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr) raise SystemExit(2) - run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "origin"]) - run(["git", "-C", str(dest), "checkout", ref]) - print(f"updated {dest}") + if not update: + print(f"skip update {dest}") + return + fetch_and_checkout(dest, ref) return if dest.exists(): print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr) @@ -71,16 +116,44 @@ def clone_one(job: dict, token: str) -> None: print(f"cloned {dest}") +def update_installed_extras(known: set[str], update: bool) -> None: + if not update: + return + for root in EXTRA_ROOTS: + if not root.is_dir(): + continue + for child in sorted(root.iterdir()): + if not child.is_dir() or not (child / ".git").is_dir(): + continue + key = str(child) + if key in known: + continue + try: + update_tracking_branch(child) + except Exception as exc: + print(f"FAIL installed {child}: {exc}", file=sys.stderr) + raise + + def main() -> int: token = TOKEN_PATH.read_text(encoding="utf-8").strip() if TOKEN_PATH.is_file() else "" jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8")) + update = do_update() + print(f"extensions update={'on' if update else 'off'}") failed = 0 + known: set[str] = set() for job in jobs: try: - clone_one(job, token) + dest = str(Path(job["dest"])) + known.add(dest) + clone_one(job, token, update) except Exception as exc: failed += 1 print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr) + try: + update_installed_extras(known, update) + except Exception: + failed += 1 if TOKEN_PATH.is_file(): TOKEN_PATH.unlink() if failed: diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py index e146ec3..9020527 100644 --- a/src/gpu_rent/session.py +++ b/src/gpu_rent/session.py @@ -46,7 +46,7 @@ from gpu_rent.os_client import ( iter_volume_types, ) from gpu_rent.ssh_keys import ensure_ed25519 -from gpu_rent.ssh_ops import probe_ssh, wait_ssh +from gpu_rent.ssh_ops import probe_ssh, run_ssh, wait_ssh from gpu_rent.state import SessionState, load_state, save_state, utc_now Log = Callable[[str], None] @@ -66,7 +66,15 @@ def _require_gpu_quota(conn) -> None: ) -def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> SessionState: +def _bind_access( + conn, + server, + state: SessionState, + cfg: Config, + log: Log, + *, + update: bool = True, +) -> SessionState: ip, fip_id = ensure_floating_ip( conn, server, state.floating_ip_id, state.floating_ip, log ) @@ -76,8 +84,20 @@ def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> Se save_state(state) wait_ssh(cfg, ip) log(f"SSH {cfg.ssh_user}@{ip}") - run_bootstrap(cfg, ip, log) - provision_vm(cfg, ip, log, conn=conn, server_id=getattr(server, "id", None) or state.server_id) + if update: + active = run_ssh(cfg, ip, "systemctl is-active swarmui 2>/dev/null || true", check=False).strip() + if active == "active": + log("systemctl stop swarmui перед git update") + run_ssh(cfg, ip, "sudo -n systemctl stop swarmui", timeout=120, check=False) + run_bootstrap(cfg, ip, log, update=update) + provision_vm( + cfg, + ip, + log, + conn=conn, + server_id=getattr(server, "id", None) or state.server_id, + update=update, + ) try: wait_backend_idle(cfg, ip, log) except CloudError as exc: @@ -99,7 +119,7 @@ def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> Se return state -def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState: +def adopt_server(cfg: Config, log: Log = _log_default, *, update: bool = True) -> SessionState: conn = connect(cfg) server = pick_existing_server(conn) if not server: @@ -115,7 +135,7 @@ def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState: save_state(state) log(f"подхватили {server.id} статус {server_status(server)}") if server_status(server) == "ACTIVE": - _bind_access(conn, server, state, cfg, log) + _bind_access(conn, server, state, cfg, log, update=update) return state @@ -126,12 +146,14 @@ def cmd_up( flavor: str | None = None, yes: bool = False, adopt: bool = False, + update: bool | None = None, confirm: Callable[[str], bool] | None = None, log: Log = _log_default, ) -> SessionState: + do_update = cfg.update_git if update is None else update with SessionLock(): if adopt: - return adopt_server(cfg, log=log) + return adopt_server(cfg, log=log, update=do_update) conn = connect(cfg) _require_gpu_quota(conn) state = load_state() @@ -145,7 +167,7 @@ def cmd_up( log("сервер уже ACTIVE — второй GPU не создаём") state.phase = "ready_cloud" save_state(state) - _bind_access(conn, existing, state, cfg, log) + _bind_access(conn, existing, state, cfg, log, update=do_update) return state # Bootstrap не завершён: почти всегда VM без authorized_keys. @@ -167,7 +189,7 @@ def cmd_up( log("сервер ACTIVE, SSH ок — продолжаем bootstrap") state.phase = "ready_cloud" save_state(state) - _bind_access(conn, existing, state, cfg, log) + _bind_access(conn, existing, state, cfg, log, update=do_update) return state log( @@ -194,7 +216,7 @@ def cmd_up( state.phase = "ready_cloud" state.unshelved_at = utc_now() save_state(state) - _bind_access(conn, existing, state, cfg, log) + _bind_access(conn, existing, state, cfg, log, update=do_update) return state if existing is not None: raise CloudError(f"сервер gpu-rent в статусе {status} — разбери в панели") @@ -321,7 +343,7 @@ def cmd_up( state.unshelved_at = None state.phase = "ready_cloud" save_state(state) - _bind_access(conn, server, state, cfg, log) + _bind_access(conn, server, state, cfg, log, update=do_update) return state diff --git a/src/gpu_rent/tunnel.py b/src/gpu_rent/tunnel.py index ed4daaa..91a12b3 100644 --- a/src/gpu_rent/tunnel.py +++ b/src/gpu_rent/tunnel.py @@ -27,6 +27,27 @@ EXIT_STATUSES = frozenset( SHELVED_STATUSES = frozenset({"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"}) +def _patch_paramiko_for_sshtunnel() -> None: + """sshtunnel 0.4 still refs paramiko.DSSKey; Paramiko 4+ removed it.""" + import paramiko + + if hasattr(paramiko, "DSSKey"): + return + + class _DSSKeyRemoved(paramiko.PKey): + def __init__(self, *args, **kwargs): + raise paramiko.SSHException("DSA keys unsupported (paramiko>=4)") + + paramiko.DSSKey = _DSSKeyRemoved # type: ignore[attr-defined, assignment] + + +def _ssh_tunnel_forwarder(): + _patch_paramiko_for_sshtunnel() + from sshtunnel import SSHTunnelForwarder + + return SSHTunnelForwarder + + @dataclass class WatchDecision: kind: str # ok | reconnect | unshelve | exit @@ -51,7 +72,7 @@ def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision: def _start_forwarder(cfg: Config, host: str, local_port: int): - from sshtunnel import SSHTunnelForwarder + SSHTunnelForwarder = _ssh_tunnel_forwarder() server = SSHTunnelForwarder( (host, 22), @@ -131,7 +152,7 @@ def run_tunnel( poll_seconds: float = 30.0, ) -> None: try: - from sshtunnel import SSHTunnelForwarder # noqa: F401 + _ssh_tunnel_forwarder() except ImportError as exc: raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc diff --git a/src/gpu_rent/varsfile.py b/src/gpu_rent/varsfile.py new file mode 100644 index 0000000..83cc4e1 --- /dev/null +++ b/src/gpu_rent/varsfile.py @@ -0,0 +1,47 @@ +"""Load KEY=VALUE files without overriding existing environment.""" + +from __future__ import annotations + +from pathlib import Path +import os +import shlex + + +def parse_vars_file(path: Path) -> dict[str, str]: + """Parse simple KEY=VALUE lines (# comments). No shell expansion.""" + out: dict[str, str] = {} + if not path.is_file(): + return out + text = path.read_text(encoding="utf-8-sig") + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.lower().startswith("export "): + line = line[7:].strip() + if "=" not in line: + continue + key, _, value = line.partition("=") + key = key.strip() + if not key or not key.replace("_", "").isalnum(): + continue + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}: + value = value[1:-1] + out[key] = value + return out + + +def apply_vars_file(path: Path, *, override: bool = False) -> dict[str, str]: + """Set os.environ from vars file. By default do not override existing keys.""" + loaded = parse_vars_file(path) + for key, value in loaded.items(): + if override or key not in os.environ or os.environ.get(key) == "": + os.environ[key] = value + return loaded + + +def split_args(value: str | None) -> list[str]: + if not value or not value.strip(): + return [] + return shlex.split(value, posix=True) diff --git a/tests/test_civitai_fetch.py b/tests/test_civitai_fetch.py new file mode 100644 index 0000000..7ef7a78 --- /dev/null +++ b/tests/test_civitai_fetch.py @@ -0,0 +1,35 @@ +from pathlib import Path + +from gpu_rent.remote.civitai_fetch import should_skip + + +def test_should_skip_missing(tmp_path: Path): + skip, _ = should_skip(tmp_path / "nope.safetensors", "abc") + assert skip is False + + +def test_should_skip_exists_no_sha(tmp_path: Path): + dest = tmp_path / "a.safetensors" + dest.write_bytes(b"weights") + skip, reason = should_skip(dest, "") + assert skip is True + assert "уже есть" in reason + + +def test_should_skip_sha_match(tmp_path: Path): + dest = tmp_path / "a.safetensors" + dest.write_bytes(b"weights") + import hashlib + + digest = hashlib.sha256(b"weights").hexdigest() + skip, reason = should_skip(dest, digest) + assert skip is True + assert "sha ok" in reason + + +def test_should_not_skip_bad_sha(tmp_path: Path): + dest = tmp_path / "a.safetensors" + dest.write_bytes(b"weights") + skip, reason = should_skip(dest, "deadbeef" * 8) + assert skip is False + assert "перекачиваю" in reason diff --git a/tests/test_clone_ext_update.py b/tests/test_clone_ext_update.py new file mode 100644 index 0000000..91cd5c1 --- /dev/null +++ b/tests/test_clone_ext_update.py @@ -0,0 +1,15 @@ +from gpu_rent.remote.clone_ext import do_update, is_sha + + +def test_is_sha(): + assert is_sha("a" * 40) + assert not is_sha("main") + + +def test_do_update_reads_flag(tmp_path, monkeypatch): + flag = tmp_path / "gpu-rent-update-git" + monkeypatch.setattr("gpu_rent.remote.clone_ext.UPDATE_PATH", flag) + flag.write_text("0\n", encoding="utf-8") + assert do_update() is False + flag.write_text("1\n", encoding="utf-8") + assert do_update() is True diff --git a/tests/test_sshtunnel_shim.py b/tests/test_sshtunnel_shim.py new file mode 100644 index 0000000..5cfe8bb --- /dev/null +++ b/tests/test_sshtunnel_shim.py @@ -0,0 +1,18 @@ +import paramiko + +from gpu_rent.tunnel import _patch_paramiko_for_sshtunnel, _ssh_tunnel_forwarder + + +def test_dsskey_shim_allows_sshtunnel_import(): + # Simulate paramiko>=4 (no DSSKey) then ensure sshtunnel can load. + had = getattr(paramiko, "DSSKey", None) + if had is not None: + delattr(paramiko, "DSSKey") + try: + _patch_paramiko_for_sshtunnel() + assert hasattr(paramiko, "DSSKey") + cls = _ssh_tunnel_forwarder() + assert cls is not None + finally: + if had is not None: + paramiko.DSSKey = had diff --git a/tests/test_varsfile.py b/tests/test_varsfile.py new file mode 100644 index 0000000..f72b938 --- /dev/null +++ b/tests/test_varsfile.py @@ -0,0 +1,32 @@ +from pathlib import Path + +from gpu_rent.varsfile import apply_vars_file, parse_vars_file, split_args + + +def test_parse_vars_file(tmp_path: Path): + path = tmp_path / "gpu-rent.vars" + path.write_text( + "# comment\n" + "GPU_RENT_DEFAULT_ARGS=up --yes\n" + "export UPDATE_GIT=false\n" + "QUOTED=\"a b\"\n", + encoding="utf-8", + ) + data = parse_vars_file(path) + assert data["GPU_RENT_DEFAULT_ARGS"] == "up --yes" + assert data["UPDATE_GIT"] == "false" + assert data["QUOTED"] == "a b" + + +def test_apply_does_not_override(tmp_path: Path, monkeypatch): + path = tmp_path / "gpu-rent.vars" + path.write_text("FOO=from-file\n", encoding="utf-8") + monkeypatch.setenv("FOO", "from-env") + apply_vars_file(path, override=False) + import os + + assert os.environ["FOO"] == "from-env" + + +def test_split_args(): + assert split_args("up --yes --no-update") == ["up", "--yes", "--no-update"]