Add support for gpu-rent.vars and enhance git update functionality
- Updated .gitignore to include gpu-rent.vars. - Modified env.example to introduce the UPDATE_GIT variable for controlling git updates during execution. - Implemented Import-GpuRentVars function in gpu-rent.ps1 to load environment variables from gpu-rent.vars. - Enhanced gpu-rent.sh to support loading variables from gpu-rent.vars and added logic for handling default and extra arguments. - Updated CLI documentation to reflect the new gpu-rent.vars file and its usage in configuration. - Improved bootstrap and provisioning logic to conditionally perform git updates based on the new configuration.
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
gpu-rent.vars
|
||||
models.yaml
|
||||
extensions.yaml
|
||||
|
||||
|
||||
+21
-2
@@ -8,7 +8,8 @@
|
||||
| --- | --- |
|
||||
| `gpu-rent flavors` | Скан `SCAN_POOLS` (ru-6 multizone…) × `FLAVOR_PREFERENCE`, затем список в текущем `OS_REGION_NAME` |
|
||||
| `gpu-rent doctor` | Preflight **без** create: Keystone, квота GPU, flavor в AZ, диски, Civitai token+`.red`, манифесты, SSH-ключ. Код выхода ≠ 0, если сессию нельзя начать |
|
||||
| `gpu-rent up` / `up --yes` | Preflight → create/unshelve → bootstrap → **туннель** `localhost:17801` + URL панели; Ctrl+C закрывает туннель |
|
||||
| `gpu-rent up` / `up --yes` | Preflight → create/unshelve → bootstrap → git update SwarmUI/extensions → **туннель** `localhost:17801`; Ctrl+C закрывает туннель |
|
||||
| `gpu-rent up --no-update` | Без `git pull` SwarmUI и extensions (только недостающие clone) |
|
||||
| `gpu-rent up --no-tunnel` | Только облако + bootstrap, без локального проброса |
|
||||
| `gpu-rent up --no-spot` | Обычный (не preemptible) сервер |
|
||||
| `gpu-rent up --flavor … --yes` | Без вопросов. `--flavor` бьёт список фоллбека. `--yes` без `--flavor` берёт первый доступный из `FLAVOR_PREFERENCE` |
|
||||
@@ -100,7 +101,25 @@ Hold killer: gpu-rent hold
|
||||
|
||||
## Конфигурация
|
||||
|
||||
`.env` в корне репозитория (рядом с `env.example`):
|
||||
| Файл | Назначение |
|
||||
| --- | --- |
|
||||
| `.env` | Секреты и OpenStack (`OS_*`, токены). Не в git |
|
||||
| `gpu-rent.vars` | Параметры запуска / несекретные дефолты. Читают `gpu-rent.ps1` / `.sh` / `.bat` и CLI. Пример: `gpu-rent.vars.example` |
|
||||
| `models.yaml` / `extensions.yaml` | Манифесты |
|
||||
|
||||
`.env` в корне репозитория (рядом с `env.example`).
|
||||
|
||||
В `gpu-rent.vars` (создаётся из example при первом запуске):
|
||||
|
||||
```env
|
||||
# Двойной клик / запуск без аргументов:
|
||||
GPU_RENT_DEFAULT_ARGS=up --yes
|
||||
|
||||
# Дописать ко всем вызовам:
|
||||
# GPU_RENT_EXTRA_ARGS=--no-update
|
||||
|
||||
UPDATE_GIT=true
|
||||
```
|
||||
|
||||
```env
|
||||
OS_AUTH_URL=https://cloud.api.selcloud.ru/identity/v3
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
| Idle-killer | 30 минут пустой очереди генерации. Открытый браузер не продлевает жизнь |
|
||||
| Льгота после boot | Killer молчит во время clone/seed/push, hold, качалки в UI SwarmUI, пока backend не Idle, и 45 мин после ACTIVE/unshelve |
|
||||
| `up` / `tunnel` | `up` по умолчанию после ready открывает туннель `:17801`, печатает URL и ждёт. `--no-tunnel` — только облако. Ctrl+C на туннеле GPU не гасит (`stop` отдельно). Команда `tunnel` остаётся для повторного входа |
|
||||
| Git update | На каждом `up` по умолчанию: `git pull` SwarmUI + репы из `extensions.yaml` + уже установленные на data (`Extensions`/`DLNodes`). `--no-update` или `UPDATE_GIT=false` — не тянуть |
|
||||
| Data-диск | Старт **100 GB**, рост через resize вверх (вниз Selectel не умеет) |
|
||||
| SSH | CLI генерирует `<repo>/.gpu-rent/id_ed25519` без passphrase и сам регистрирует keypair |
|
||||
| Локальные файлы | Всё в корне репозитория: `.env`, `models.yaml`, `extensions.yaml`; runtime (`state.json`, lock, SSH) в `<repo>/.gpu-rent/`. Не `%USERPROFILE%\.gpu-rent` |
|
||||
|
||||
+7
-1
@@ -1,6 +1,12 @@
|
||||
# Расширения: git-репы на первый bootstrap
|
||||
|
||||
Срабатывает вместе с первым диском и первым инстансом, **до** первого старта SwarmUI (C# extensions компилируются при запуске). Повторный `up` на уже засеянный диск только догоняет новые строки манифеста (идемпотентный `git fetch`).
|
||||
Срабатывает на каждом `up` (и `seed-extensions`):
|
||||
|
||||
- нет каталога → `git clone`;
|
||||
- каталог есть → по умолчанию `fetch` + `reset --hard` на `ref` (или текущую ветку для уже установленных вне yaml);
|
||||
- `--no-update` / `UPDATE_GIT=false` → существующие репы не трогаем, только недостающие clone.
|
||||
|
||||
Также обновляется сам SwarmUI в `/opt/swarmui` (тот же флаг).
|
||||
|
||||
Два разных мира — не путать:
|
||||
|
||||
|
||||
@@ -37,6 +37,8 @@ AUTOCOMPLETE_GITHUB_REF=main
|
||||
AUTOCOMPLETE_FILENAME=danbooru.csv
|
||||
|
||||
SWARMUI_LOCAL_PORT=17801
|
||||
# git pull SwarmUI + extensions on each up (default true). CLI: --no-update
|
||||
UPDATE_GIT=true
|
||||
|
||||
DEFAULT_FLAVOR_ID=
|
||||
FLAVOR_PREFERENCE=4090-24,4090-48,a5000,a100-40
|
||||
|
||||
+39
-1
@@ -12,6 +12,33 @@ try {
|
||||
} catch {
|
||||
}
|
||||
|
||||
function Import-GpuRentVars {
|
||||
param([Parameter(Mandatory = $true)][string]$Path)
|
||||
if (-not (Test-Path -LiteralPath $Path)) { return }
|
||||
Get-Content -LiteralPath $Path -Encoding UTF8 | ForEach-Object {
|
||||
$line = $_.Trim()
|
||||
if (-not $line -or $line.StartsWith("#")) { return }
|
||||
if ($line -match '^(?i)export\s+') {
|
||||
$line = $line.Substring($Matches[0].Length).Trim()
|
||||
}
|
||||
$eq = $line.IndexOf("=")
|
||||
if ($eq -lt 1) { return }
|
||||
$key = $line.Substring(0, $eq).Trim()
|
||||
$val = $line.Substring($eq + 1).Trim()
|
||||
if ($val.Length -ge 2) {
|
||||
$q = $val[0]
|
||||
if (($q -eq '"' -or $q -eq "'") -and $val[-1] -eq $q) {
|
||||
$val = $val.Substring(1, $val.Length - 2)
|
||||
}
|
||||
}
|
||||
if (-not $key) { return }
|
||||
$existing = [Environment]::GetEnvironmentVariable($key, "Process")
|
||||
if ([string]::IsNullOrEmpty($existing)) {
|
||||
Set-Item -Path "Env:$key" -Value $val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Test-Python311 {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$Exe,
|
||||
@@ -91,7 +118,18 @@ function Copy-IfMissing {
|
||||
}
|
||||
Copy-IfMissing (Join-Path $Root "models.example.yaml") (Join-Path $Root "models.yaml") "models.yaml"
|
||||
Copy-IfMissing (Join-Path $Root "extensions.example.yaml") (Join-Path $Root "extensions.yaml") "extensions.yaml"
|
||||
Copy-IfMissing (Join-Path $Root "gpu-rent.vars.example") (Join-Path $Root "gpu-rent.vars") "gpu-rent.vars"
|
||||
|
||||
Import-GpuRentVars (Join-Path $Root "gpu-rent.vars")
|
||||
|
||||
$InvokeArgs = @($args)
|
||||
if ($InvokeArgs.Count -eq 0 -and $env:GPU_RENT_DEFAULT_ARGS) {
|
||||
$InvokeArgs = @($env:GPU_RENT_DEFAULT_ARGS -split '\s+' | Where-Object { $_ })
|
||||
}
|
||||
if ($env:GPU_RENT_EXTRA_ARGS) {
|
||||
$InvokeArgs += @($env:GPU_RENT_EXTRA_ARGS -split '\s+' | Where-Object { $_ })
|
||||
}
|
||||
|
||||
$ErrorActionPreference = "Continue"
|
||||
& $VenvPy -m gpu_rent @args
|
||||
& $VenvPy -m gpu_rent @InvokeArgs
|
||||
exit $LASTEXITCODE
|
||||
|
||||
+48
-1
@@ -5,6 +5,37 @@ set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
load_vars_file() {
|
||||
local file="$1"
|
||||
[[ -f "$file" ]] || return 0
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
line="${line%"${line##*[![:space:]]}"}"
|
||||
[[ -z "$line" || "$line" == \#* ]] && continue
|
||||
if [[ "$line" == [eE][xX][pP][oO][rR][tT][[:space:]]* ]]; then
|
||||
line="${line#*[eE][xX][pP][oO][rR][tT]}"
|
||||
line="${line#"${line%%[![:space:]]*}"}"
|
||||
fi
|
||||
[[ "$line" != *=* ]] && continue
|
||||
local key="${line%%=*}"
|
||||
local val="${line#*=}"
|
||||
key="${key%"${key##*[![:space:]]}"}"
|
||||
key="${key#"${key%%[![:space:]]*}"}"
|
||||
val="${val#"${val%%[![:space:]]*}"}"
|
||||
val="${val%"${val##*[![:space:]]}"}"
|
||||
if [[ ${#val} -ge 2 ]]; then
|
||||
local q="${val:0:1}"
|
||||
if [[ ( "$q" == '"' || "$q" == "'" ) && "${val: -1}" == "$q" ]]; then
|
||||
val="${val:1:${#val}-2}"
|
||||
fi
|
||||
fi
|
||||
[[ -z "$key" ]] && continue
|
||||
if [[ -z "${!key+x}" || -z "${!key}" ]]; then
|
||||
export "$key=$val"
|
||||
fi
|
||||
done <"$file"
|
||||
}
|
||||
|
||||
ok_py() {
|
||||
local exe="$1"
|
||||
shift || true
|
||||
@@ -64,5 +95,21 @@ if [[ ! -f "$ROOT/extensions.yaml" && -f "$ROOT/extensions.example.yaml" ]]; the
|
||||
cp "$ROOT/extensions.example.yaml" "$ROOT/extensions.yaml"
|
||||
echo "gpu-rent: created extensions.yaml"
|
||||
fi
|
||||
if [[ ! -f "$ROOT/gpu-rent.vars" && -f "$ROOT/gpu-rent.vars.example" ]]; then
|
||||
cp "$ROOT/gpu-rent.vars.example" "$ROOT/gpu-rent.vars"
|
||||
echo "gpu-rent: created gpu-rent.vars"
|
||||
fi
|
||||
|
||||
exec "$VENV_PY" -m gpu_rent "$@"
|
||||
load_vars_file "$ROOT/gpu-rent.vars"
|
||||
|
||||
ARGS=("$@")
|
||||
if [[ ${#ARGS[@]} -eq 0 && -n "${GPU_RENT_DEFAULT_ARGS:-}" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
ARGS=(${GPU_RENT_DEFAULT_ARGS})
|
||||
fi
|
||||
if [[ -n "${GPU_RENT_EXTRA_ARGS:-}" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
ARGS+=(${GPU_RENT_EXTRA_ARGS})
|
||||
fi
|
||||
|
||||
exec "$VENV_PY" -m gpu_rent "${ARGS[@]}"
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# gpu-rent.vars — локальные параметры запуска (не секреты).
|
||||
# Скопируй в gpu-rent.vars и правь. Файл в .gitignore.
|
||||
# Читают: gpu-rent.ps1 / .bat / .sh и python -m gpu_rent (после .env).
|
||||
# Секреты (OS_PASSWORD, CIVITAI_*) — только в .env.
|
||||
|
||||
# Если запуск без аргументов (двойной клик / .\gpu-rent.ps1):
|
||||
# GPU_RENT_DEFAULT_ARGS=up --yes
|
||||
|
||||
# Доп. флаги ко ВСЕМ вызовам (после твоих аргументов):
|
||||
# GPU_RENT_EXTRA_ARGS=
|
||||
|
||||
# Те же имена, что в .env / env.example (несекретные дефолты):
|
||||
# UPDATE_GIT=true
|
||||
# DEFAULT_SPOT=true
|
||||
# SCAN_POOLS=ru-6,ru-7
|
||||
# FLAVOR_PREFERENCE=4090-24,4090-48,a5000,a100-40
|
||||
# SWARMUI_LOCAL_PORT=17801
|
||||
# IDLE_MINUTES=30
|
||||
# IDLE_GRACE_MINUTES=45
|
||||
# NOTIFY_READY=true
|
||||
# PULL_OUTPUT=false
|
||||
# GPU_RENT_SSH_CIDR=0.0.0.0/0
|
||||
@@ -16,8 +16,12 @@ def bootstrap_script() -> str:
|
||||
return files("gpu_rent.remote").joinpath("bootstrap.sh").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def run_bootstrap(cfg: Config, host: str, log: Log) -> None:
|
||||
def run_bootstrap(cfg: Config, host: str, log: Log, *, update: bool = True) -> None:
|
||||
log("bootstrap SwarmUI на VM (идемпотентно, без Docker)")
|
||||
if update:
|
||||
log("git update: SwarmUI on")
|
||||
else:
|
||||
log("git update: SwarmUI off (--no-update)")
|
||||
script = bootstrap_script()
|
||||
out = run_script_sudo(
|
||||
cfg,
|
||||
@@ -25,7 +29,10 @@ def run_bootstrap(cfg: Config, host: str, log: Log) -> None:
|
||||
script,
|
||||
remote_path="/tmp/gpu-rent-bootstrap.sh",
|
||||
timeout=1800,
|
||||
env={"SWARM_USER": cfg.ssh_user},
|
||||
env={
|
||||
"SWARM_USER": cfg.ssh_user,
|
||||
"GPU_RENT_UPDATE_GIT": "1" if update else "0",
|
||||
},
|
||||
log=log,
|
||||
)
|
||||
if "bootstrap ok" not in out:
|
||||
|
||||
@@ -301,6 +301,11 @@ def up(
|
||||
"--open/--no-open",
|
||||
help="После туннеля открыть браузер на 17801 (по умолчанию да)",
|
||||
),
|
||||
no_update: bool = typer.Option(
|
||||
False,
|
||||
"--no-update",
|
||||
help="Не делать git pull SwarmUI и установленных extensions",
|
||||
),
|
||||
) -> None:
|
||||
"""Create/unshelve GPU, bootstrap SwarmUI, по умолчанию туннель на :17801."""
|
||||
try:
|
||||
@@ -319,6 +324,7 @@ def up(
|
||||
flavor=flavor,
|
||||
yes=yes,
|
||||
adopt=adopt,
|
||||
update=False if no_update else None,
|
||||
confirm=confirm,
|
||||
log=lambda m: console.print(m),
|
||||
)
|
||||
|
||||
@@ -17,7 +17,9 @@ from gpu_rent.paths import (
|
||||
migrate_legacy_if_needed,
|
||||
models_manifest_path,
|
||||
runtime_dir,
|
||||
vars_path,
|
||||
)
|
||||
from gpu_rent.varsfile import apply_vars_file
|
||||
|
||||
|
||||
def _as_bool(value: str | None, default: bool) -> bool:
|
||||
@@ -76,6 +78,7 @@ class Config:
|
||||
|
||||
swarmui_local_port: int
|
||||
swarmui_image: str
|
||||
update_git: bool
|
||||
|
||||
default_flavor_id: str
|
||||
flavor_preference: tuple[str, ...]
|
||||
@@ -108,6 +111,9 @@ def load_config(*, require_auth: bool = True) -> Config:
|
||||
if env_file.is_file():
|
||||
load_dotenv(env_file, override=False)
|
||||
|
||||
# Non-secret launch defaults (gpu-rent.vars). Do not override .env / real env.
|
||||
apply_vars_file(vars_path(), override=False)
|
||||
|
||||
missing: list[str] = []
|
||||
required = (
|
||||
"OS_AUTH_URL",
|
||||
@@ -180,6 +186,7 @@ def load_config(*, require_auth: bool = True) -> Config:
|
||||
autocomplete_filename=(os.environ.get("AUTOCOMPLETE_FILENAME") or "danbooru.csv").strip(),
|
||||
swarmui_local_port=_as_int(os.environ.get("SWARMUI_LOCAL_PORT"), 17801),
|
||||
swarmui_image=(os.environ.get("SWARMUI_IMAGE") or "").strip(),
|
||||
update_git=_as_bool(os.environ.get("UPDATE_GIT"), True),
|
||||
default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(),
|
||||
flavor_preference=_csv(
|
||||
os.environ.get("FLAVOR_PREFERENCE"),
|
||||
|
||||
@@ -40,6 +40,14 @@ def extensions_manifest_path() -> Path:
|
||||
return app_root() / "extensions.yaml"
|
||||
|
||||
|
||||
def vars_path() -> Path:
|
||||
return app_root() / "gpu-rent.vars"
|
||||
|
||||
|
||||
def vars_example_path() -> Path:
|
||||
return app_root() / "gpu-rent.vars.example"
|
||||
|
||||
|
||||
def state_path() -> Path:
|
||||
return runtime_dir() / "state.json"
|
||||
|
||||
|
||||
@@ -33,9 +33,9 @@ def _pkg_text(name: str) -> str:
|
||||
return files("gpu_rent.remote").joinpath(name).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def seed_extensions(cfg: Config, host: str, log: Log) -> bool:
|
||||
def seed_extensions(cfg: Config, host: str, log: Log, *, update: bool = True) -> bool:
|
||||
repos = parse_extensions(cfg.extensions_manifest)
|
||||
if not repos:
|
||||
if not repos and not update:
|
||||
log("extensions.yaml пуст — стоковый SwarmUI")
|
||||
return False
|
||||
jobs = []
|
||||
@@ -49,9 +49,13 @@ def seed_extensions(cfg: Config, host: str, log: Log) -> bool:
|
||||
}
|
||||
)
|
||||
put_text(cfg, host, "/tmp/gpu-rent-ext.json", json.dumps(jobs, indent=2))
|
||||
put_text(cfg, host, "/tmp/gpu-rent-update-git", "1\n" if update else "0\n")
|
||||
if cfg.git_token:
|
||||
put_text(cfg, host, "/tmp/gpu-rent-git.token", cfg.git_token + "\n", mode=0o600)
|
||||
log(f"clone {len(jobs)} git-реп на data volume")
|
||||
if update:
|
||||
log(f"extensions: clone/update {len(jobs)} из yaml + установленные на data")
|
||||
else:
|
||||
log(f"extensions: только недостающие из yaml ({len(jobs)}), без git pull (--no-update)")
|
||||
out = run_python(
|
||||
cfg,
|
||||
host,
|
||||
@@ -224,7 +228,10 @@ def seed_civitai(cfg: Config, host: str, log: Log) -> None:
|
||||
log("в манифесте нет checkpoint — генерация может не стартовать")
|
||||
put_text(cfg, host, "/tmp/gpu-rent-civitai-jobs.json", json.dumps(jobs, indent=2))
|
||||
put_text(cfg, host, "/tmp/gpu-rent-civitai.token", cfg.civitai_api_token + "\n", mode=0o600)
|
||||
log(f"Civitai: качаю {len(jobs)} файл(ов) на VM")
|
||||
log(
|
||||
f"Civitai: {len(jobs)} в манифесте — на VM качаю отсутствующие "
|
||||
f"(уже есть + sha → skip; прогресс [N/{len(jobs)}])"
|
||||
)
|
||||
run_python(
|
||||
cfg,
|
||||
host,
|
||||
@@ -254,10 +261,11 @@ def provision_vm(
|
||||
*,
|
||||
conn=None,
|
||||
server_id: str | None = None,
|
||||
update: bool = True,
|
||||
) -> None:
|
||||
restart = False
|
||||
restart = bool(update)
|
||||
try:
|
||||
if seed_extensions(cfg, host, log):
|
||||
if seed_extensions(cfg, host, log, update=update):
|
||||
restart = True
|
||||
except GpuRentError as exc:
|
||||
log(f"extensions: {exc}")
|
||||
|
||||
@@ -97,8 +97,21 @@ if [[ ! -d "${SWARM_ROOT}/.git" ]]; then
|
||||
log "clone SwarmUI -> ${SWARM_ROOT}"
|
||||
mkdir -p "$(dirname "$SWARM_ROOT")"
|
||||
git clone --depth 1 "$SWARM_REPO" "$SWARM_ROOT"
|
||||
elif [[ "${GPU_RENT_UPDATE_GIT:-1}" == "1" ]]; then
|
||||
log "обновляю SwarmUI в ${SWARM_ROOT}"
|
||||
branch="$(git -C "$SWARM_ROOT" remote show origin 2>/dev/null | sed -n '/HEAD branch/s/.*: //p' || true)"
|
||||
branch="${branch:-master}"
|
||||
# shallow clone: deepen tip of default branch
|
||||
git -C "$SWARM_ROOT" fetch --depth 1 origin "$branch" || git -C "$SWARM_ROOT" fetch --depth 1 origin
|
||||
if git -C "$SWARM_ROOT" rev-parse --verify -q "origin/${branch}" >/dev/null; then
|
||||
git -C "$SWARM_ROOT" checkout -B "$branch" "origin/${branch}"
|
||||
git -C "$SWARM_ROOT" reset --hard "origin/${branch}"
|
||||
else
|
||||
git -C "$SWARM_ROOT" pull --ff-only || true
|
||||
fi
|
||||
log "SwarmUI @ $(git -C "$SWARM_ROOT" rev-parse --short HEAD)"
|
||||
else
|
||||
log "SwarmUI уже в ${SWARM_ROOT}"
|
||||
log "SwarmUI уже в ${SWARM_ROOT} (update off)"
|
||||
fi
|
||||
|
||||
if [[ ! -x /usr/share/dotnet/dotnet && ! -x "/home/${SWARM_USER}/.dotnet/dotnet" ]]; then
|
||||
|
||||
@@ -25,6 +25,19 @@ def sha256_path(path: Path) -> str:
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def should_skip(dest: Path, expect_sha: str) -> tuple[bool, str]:
|
||||
"""Return (skip, reason). Existing file with matching sha — or any non-empty if no sha."""
|
||||
if not dest.is_file() or dest.stat().st_size <= 0:
|
||||
return False, ""
|
||||
expect = (expect_sha or "").lower()
|
||||
if not expect:
|
||||
return True, "уже есть"
|
||||
got = sha256_path(dest).lower()
|
||||
if got == expect:
|
||||
return True, "уже есть (sha ok)"
|
||||
return False, "sha не совпал — перекачиваю"
|
||||
|
||||
|
||||
def download(url: str, dest: Path, token: str) -> None:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
partial = dest.with_suffix(dest.suffix + ".partial")
|
||||
@@ -61,16 +74,30 @@ def main() -> int:
|
||||
return 1
|
||||
token = TOKEN_PATH.read_text(encoding="utf-8").strip()
|
||||
jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8"))
|
||||
total = len(jobs)
|
||||
failed = 0
|
||||
for job in jobs:
|
||||
skipped = 0
|
||||
downloaded = 0
|
||||
for index, job in enumerate(jobs, start=1):
|
||||
dest = Path(job["dest"])
|
||||
expect = (job.get("sha256") or "").lower()
|
||||
if dest.is_file() and expect and sha256_path(dest).lower() == expect:
|
||||
print(f"skip {dest}")
|
||||
prefix = f"[{index}/{total}]"
|
||||
skip, reason = should_skip(dest, expect)
|
||||
if skip:
|
||||
skipped += 1
|
||||
print(f"{prefix} {reason}: {dest.name}")
|
||||
# refresh sidecars even on skip
|
||||
for extra_name, extra_text in (job.get("sidecars") or {}).items():
|
||||
extra = dest.parent / extra_name
|
||||
extra.parent.mkdir(parents=True, exist_ok=True)
|
||||
extra.write_text(extra_text, encoding="utf-8")
|
||||
continue
|
||||
if reason:
|
||||
print(f"{prefix} {reason}: {dest.name}")
|
||||
try:
|
||||
print(f"download {dest.name}")
|
||||
print(f"{prefix} качаю: {dest.name}")
|
||||
download(job["url"], dest, token)
|
||||
downloaded += 1
|
||||
if expect:
|
||||
got = sha256_path(dest).lower()
|
||||
if got != expect:
|
||||
@@ -81,8 +108,9 @@ def main() -> int:
|
||||
extra.write_text(extra_text, encoding="utf-8")
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
print(f"FAIL {dest}: {exc}", file=sys.stderr)
|
||||
print(f"{prefix} FAIL {dest.name}: {exc}", file=sys.stderr)
|
||||
TOKEN_PATH.unlink(missing_ok=True)
|
||||
print(f"Civitai итог: скачано {downloaded}, пропущено {skipped}, ошибок {failed} (из {total})")
|
||||
if failed:
|
||||
return 1
|
||||
MARKER.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Clone git extensions on the VM. Stdlib only. Token file optional."""
|
||||
"""Clone/update git extensions on the VM. Stdlib only. Token file optional."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@@ -10,7 +11,12 @@ from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
TOKEN_PATH = Path("/tmp/gpu-rent-git.token")
|
||||
JOBS_PATH = Path("/tmp/gpu-rent-ext.json")
|
||||
UPDATE_PATH = Path("/tmp/gpu-rent-update-git")
|
||||
MARKER = Path("/mnt/swarm_data/.gpu-rent-extensions-seeded")
|
||||
EXTRA_ROOTS = (
|
||||
Path("/mnt/swarm_data/Extensions"),
|
||||
Path("/mnt/swarm_data/DLNodes"),
|
||||
)
|
||||
|
||||
|
||||
def strip_auth(url: str) -> str:
|
||||
@@ -35,25 +41,64 @@ def run(argv: list[str], cwd: str | None = None) -> None:
|
||||
subprocess.check_call(argv, cwd=cwd)
|
||||
|
||||
|
||||
def out(argv: list[str], cwd: str | None = None) -> str:
|
||||
return subprocess.check_output(argv, cwd=cwd, text=True).strip()
|
||||
|
||||
|
||||
def is_sha(ref: str) -> bool:
|
||||
ref = ref.strip()
|
||||
return len(ref) == 40 and all(c in "0123456789abcdefABCDEF" for c in ref)
|
||||
|
||||
|
||||
def clone_one(job: dict, token: str) -> None:
|
||||
def do_update() -> bool:
|
||||
if UPDATE_PATH.is_file():
|
||||
return UPDATE_PATH.read_text(encoding="utf-8").strip() not in {"0", "false", "no", "off"}
|
||||
return (os.environ.get("GPU_RENT_UPDATE_GIT") or "1").strip() not in {"0", "false", "no", "off"}
|
||||
|
||||
|
||||
def fetch_and_checkout(dest: Path, ref: str) -> None:
|
||||
run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"])
|
||||
if is_sha(ref):
|
||||
run(["git", "-C", str(dest), "checkout", "--detach", ref])
|
||||
print(f"updated {dest} @ {ref[:12]}")
|
||||
return
|
||||
run(["git", "-C", str(dest), "checkout", ref])
|
||||
# Move branch tip to remote (shallow-friendly).
|
||||
try:
|
||||
run(["git", "-C", str(dest), "reset", "--hard", f"origin/{ref}"])
|
||||
except subprocess.CalledProcessError:
|
||||
run(["git", "-C", str(dest), "pull", "--ff-only", "origin", ref])
|
||||
print(f"updated {dest} ({ref})")
|
||||
|
||||
|
||||
def update_tracking_branch(dest: Path) -> None:
|
||||
branch = out(["git", "-C", str(dest), "rev-parse", "--abbrev-ref", "HEAD"])
|
||||
if not branch or branch == "HEAD":
|
||||
print(f"skip detached {dest}")
|
||||
return
|
||||
run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"])
|
||||
try:
|
||||
run(["git", "-C", str(dest), "reset", "--hard", f"origin/{branch}"])
|
||||
except subprocess.CalledProcessError:
|
||||
run(["git", "-C", str(dest), "pull", "--ff-only"])
|
||||
print(f"updated installed {dest} ({branch})")
|
||||
|
||||
|
||||
def clone_one(job: dict, token: str, update: bool) -> None:
|
||||
dest = Path(job["dest"])
|
||||
url = job["url"].strip()
|
||||
ref = (job.get("ref") or "main").strip()
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
authed = with_token(url, token)
|
||||
if dest.is_dir() and (dest / ".git").is_dir():
|
||||
origin = subprocess.check_output(["git", "-C", str(dest), "remote", "get-url", "origin"], text=True).strip()
|
||||
origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"])
|
||||
if strip_auth(origin) != strip_auth(url):
|
||||
print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "origin"])
|
||||
run(["git", "-C", str(dest), "checkout", ref])
|
||||
print(f"updated {dest}")
|
||||
if not update:
|
||||
print(f"skip update {dest}")
|
||||
return
|
||||
fetch_and_checkout(dest, ref)
|
||||
return
|
||||
if dest.exists():
|
||||
print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr)
|
||||
@@ -71,16 +116,44 @@ def clone_one(job: dict, token: str) -> None:
|
||||
print(f"cloned {dest}")
|
||||
|
||||
|
||||
def update_installed_extras(known: set[str], update: bool) -> None:
|
||||
if not update:
|
||||
return
|
||||
for root in EXTRA_ROOTS:
|
||||
if not root.is_dir():
|
||||
continue
|
||||
for child in sorted(root.iterdir()):
|
||||
if not child.is_dir() or not (child / ".git").is_dir():
|
||||
continue
|
||||
key = str(child)
|
||||
if key in known:
|
||||
continue
|
||||
try:
|
||||
update_tracking_branch(child)
|
||||
except Exception as exc:
|
||||
print(f"FAIL installed {child}: {exc}", file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def main() -> int:
|
||||
token = TOKEN_PATH.read_text(encoding="utf-8").strip() if TOKEN_PATH.is_file() else ""
|
||||
jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8"))
|
||||
update = do_update()
|
||||
print(f"extensions update={'on' if update else 'off'}")
|
||||
failed = 0
|
||||
known: set[str] = set()
|
||||
for job in jobs:
|
||||
try:
|
||||
clone_one(job, token)
|
||||
dest = str(Path(job["dest"]))
|
||||
known.add(dest)
|
||||
clone_one(job, token, update)
|
||||
except Exception as exc:
|
||||
failed += 1
|
||||
print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr)
|
||||
try:
|
||||
update_installed_extras(known, update)
|
||||
except Exception:
|
||||
failed += 1
|
||||
if TOKEN_PATH.is_file():
|
||||
TOKEN_PATH.unlink()
|
||||
if failed:
|
||||
|
||||
+33
-11
@@ -46,7 +46,7 @@ from gpu_rent.os_client import (
|
||||
iter_volume_types,
|
||||
)
|
||||
from gpu_rent.ssh_keys import ensure_ed25519
|
||||
from gpu_rent.ssh_ops import probe_ssh, wait_ssh
|
||||
from gpu_rent.ssh_ops import probe_ssh, run_ssh, wait_ssh
|
||||
from gpu_rent.state import SessionState, load_state, save_state, utc_now
|
||||
|
||||
Log = Callable[[str], None]
|
||||
@@ -66,7 +66,15 @@ def _require_gpu_quota(conn) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> SessionState:
|
||||
def _bind_access(
|
||||
conn,
|
||||
server,
|
||||
state: SessionState,
|
||||
cfg: Config,
|
||||
log: Log,
|
||||
*,
|
||||
update: bool = True,
|
||||
) -> SessionState:
|
||||
ip, fip_id = ensure_floating_ip(
|
||||
conn, server, state.floating_ip_id, state.floating_ip, log
|
||||
)
|
||||
@@ -76,8 +84,20 @@ def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> Se
|
||||
save_state(state)
|
||||
wait_ssh(cfg, ip)
|
||||
log(f"SSH {cfg.ssh_user}@{ip}")
|
||||
run_bootstrap(cfg, ip, log)
|
||||
provision_vm(cfg, ip, log, conn=conn, server_id=getattr(server, "id", None) or state.server_id)
|
||||
if update:
|
||||
active = run_ssh(cfg, ip, "systemctl is-active swarmui 2>/dev/null || true", check=False).strip()
|
||||
if active == "active":
|
||||
log("systemctl stop swarmui перед git update")
|
||||
run_ssh(cfg, ip, "sudo -n systemctl stop swarmui", timeout=120, check=False)
|
||||
run_bootstrap(cfg, ip, log, update=update)
|
||||
provision_vm(
|
||||
cfg,
|
||||
ip,
|
||||
log,
|
||||
conn=conn,
|
||||
server_id=getattr(server, "id", None) or state.server_id,
|
||||
update=update,
|
||||
)
|
||||
try:
|
||||
wait_backend_idle(cfg, ip, log)
|
||||
except CloudError as exc:
|
||||
@@ -99,7 +119,7 @@ def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> Se
|
||||
return state
|
||||
|
||||
|
||||
def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState:
|
||||
def adopt_server(cfg: Config, log: Log = _log_default, *, update: bool = True) -> SessionState:
|
||||
conn = connect(cfg)
|
||||
server = pick_existing_server(conn)
|
||||
if not server:
|
||||
@@ -115,7 +135,7 @@ def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState:
|
||||
save_state(state)
|
||||
log(f"подхватили {server.id} статус {server_status(server)}")
|
||||
if server_status(server) == "ACTIVE":
|
||||
_bind_access(conn, server, state, cfg, log)
|
||||
_bind_access(conn, server, state, cfg, log, update=update)
|
||||
return state
|
||||
|
||||
|
||||
@@ -126,12 +146,14 @@ def cmd_up(
|
||||
flavor: str | None = None,
|
||||
yes: bool = False,
|
||||
adopt: bool = False,
|
||||
update: bool | None = None,
|
||||
confirm: Callable[[str], bool] | None = None,
|
||||
log: Log = _log_default,
|
||||
) -> SessionState:
|
||||
do_update = cfg.update_git if update is None else update
|
||||
with SessionLock():
|
||||
if adopt:
|
||||
return adopt_server(cfg, log=log)
|
||||
return adopt_server(cfg, log=log, update=do_update)
|
||||
conn = connect(cfg)
|
||||
_require_gpu_quota(conn)
|
||||
state = load_state()
|
||||
@@ -145,7 +167,7 @@ def cmd_up(
|
||||
log("сервер уже ACTIVE — второй GPU не создаём")
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
_bind_access(conn, existing, state, cfg, log, update=do_update)
|
||||
return state
|
||||
|
||||
# Bootstrap не завершён: почти всегда VM без authorized_keys.
|
||||
@@ -167,7 +189,7 @@ def cmd_up(
|
||||
log("сервер ACTIVE, SSH ок — продолжаем bootstrap")
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
_bind_access(conn, existing, state, cfg, log, update=do_update)
|
||||
return state
|
||||
|
||||
log(
|
||||
@@ -194,7 +216,7 @@ def cmd_up(
|
||||
state.phase = "ready_cloud"
|
||||
state.unshelved_at = utc_now()
|
||||
save_state(state)
|
||||
_bind_access(conn, existing, state, cfg, log)
|
||||
_bind_access(conn, existing, state, cfg, log, update=do_update)
|
||||
return state
|
||||
if existing is not None:
|
||||
raise CloudError(f"сервер gpu-rent в статусе {status} — разбери в панели")
|
||||
@@ -321,7 +343,7 @@ def cmd_up(
|
||||
state.unshelved_at = None
|
||||
state.phase = "ready_cloud"
|
||||
save_state(state)
|
||||
_bind_access(conn, server, state, cfg, log)
|
||||
_bind_access(conn, server, state, cfg, log, update=do_update)
|
||||
return state
|
||||
|
||||
|
||||
|
||||
+23
-2
@@ -27,6 +27,27 @@ EXIT_STATUSES = frozenset(
|
||||
SHELVED_STATUSES = frozenset({"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"})
|
||||
|
||||
|
||||
def _patch_paramiko_for_sshtunnel() -> None:
|
||||
"""sshtunnel 0.4 still refs paramiko.DSSKey; Paramiko 4+ removed it."""
|
||||
import paramiko
|
||||
|
||||
if hasattr(paramiko, "DSSKey"):
|
||||
return
|
||||
|
||||
class _DSSKeyRemoved(paramiko.PKey):
|
||||
def __init__(self, *args, **kwargs):
|
||||
raise paramiko.SSHException("DSA keys unsupported (paramiko>=4)")
|
||||
|
||||
paramiko.DSSKey = _DSSKeyRemoved # type: ignore[attr-defined, assignment]
|
||||
|
||||
|
||||
def _ssh_tunnel_forwarder():
|
||||
_patch_paramiko_for_sshtunnel()
|
||||
from sshtunnel import SSHTunnelForwarder
|
||||
|
||||
return SSHTunnelForwarder
|
||||
|
||||
|
||||
@dataclass
|
||||
class WatchDecision:
|
||||
kind: str # ok | reconnect | unshelve | exit
|
||||
@@ -51,7 +72,7 @@ def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
|
||||
|
||||
|
||||
def _start_forwarder(cfg: Config, host: str, local_port: int):
|
||||
from sshtunnel import SSHTunnelForwarder
|
||||
SSHTunnelForwarder = _ssh_tunnel_forwarder()
|
||||
|
||||
server = SSHTunnelForwarder(
|
||||
(host, 22),
|
||||
@@ -131,7 +152,7 @@ def run_tunnel(
|
||||
poll_seconds: float = 30.0,
|
||||
) -> None:
|
||||
try:
|
||||
from sshtunnel import SSHTunnelForwarder # noqa: F401
|
||||
_ssh_tunnel_forwarder()
|
||||
except ImportError as exc:
|
||||
raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Load KEY=VALUE files without overriding existing environment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
import shlex
|
||||
|
||||
|
||||
def parse_vars_file(path: Path) -> dict[str, str]:
|
||||
"""Parse simple KEY=VALUE lines (# comments). No shell expansion."""
|
||||
out: dict[str, str] = {}
|
||||
if not path.is_file():
|
||||
return out
|
||||
text = path.read_text(encoding="utf-8-sig")
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if line.lower().startswith("export "):
|
||||
line = line[7:].strip()
|
||||
if "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
key = key.strip()
|
||||
if not key or not key.replace("_", "").isalnum():
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
|
||||
value = value[1:-1]
|
||||
out[key] = value
|
||||
return out
|
||||
|
||||
|
||||
def apply_vars_file(path: Path, *, override: bool = False) -> dict[str, str]:
|
||||
"""Set os.environ from vars file. By default do not override existing keys."""
|
||||
loaded = parse_vars_file(path)
|
||||
for key, value in loaded.items():
|
||||
if override or key not in os.environ or os.environ.get(key) == "":
|
||||
os.environ[key] = value
|
||||
return loaded
|
||||
|
||||
|
||||
def split_args(value: str | None) -> list[str]:
|
||||
if not value or not value.strip():
|
||||
return []
|
||||
return shlex.split(value, posix=True)
|
||||
@@ -0,0 +1,35 @@
|
||||
from pathlib import Path
|
||||
|
||||
from gpu_rent.remote.civitai_fetch import should_skip
|
||||
|
||||
|
||||
def test_should_skip_missing(tmp_path: Path):
|
||||
skip, _ = should_skip(tmp_path / "nope.safetensors", "abc")
|
||||
assert skip is False
|
||||
|
||||
|
||||
def test_should_skip_exists_no_sha(tmp_path: Path):
|
||||
dest = tmp_path / "a.safetensors"
|
||||
dest.write_bytes(b"weights")
|
||||
skip, reason = should_skip(dest, "")
|
||||
assert skip is True
|
||||
assert "уже есть" in reason
|
||||
|
||||
|
||||
def test_should_skip_sha_match(tmp_path: Path):
|
||||
dest = tmp_path / "a.safetensors"
|
||||
dest.write_bytes(b"weights")
|
||||
import hashlib
|
||||
|
||||
digest = hashlib.sha256(b"weights").hexdigest()
|
||||
skip, reason = should_skip(dest, digest)
|
||||
assert skip is True
|
||||
assert "sha ok" in reason
|
||||
|
||||
|
||||
def test_should_not_skip_bad_sha(tmp_path: Path):
|
||||
dest = tmp_path / "a.safetensors"
|
||||
dest.write_bytes(b"weights")
|
||||
skip, reason = should_skip(dest, "deadbeef" * 8)
|
||||
assert skip is False
|
||||
assert "перекачиваю" in reason
|
||||
@@ -0,0 +1,15 @@
|
||||
from gpu_rent.remote.clone_ext import do_update, is_sha
|
||||
|
||||
|
||||
def test_is_sha():
|
||||
assert is_sha("a" * 40)
|
||||
assert not is_sha("main")
|
||||
|
||||
|
||||
def test_do_update_reads_flag(tmp_path, monkeypatch):
|
||||
flag = tmp_path / "gpu-rent-update-git"
|
||||
monkeypatch.setattr("gpu_rent.remote.clone_ext.UPDATE_PATH", flag)
|
||||
flag.write_text("0\n", encoding="utf-8")
|
||||
assert do_update() is False
|
||||
flag.write_text("1\n", encoding="utf-8")
|
||||
assert do_update() is True
|
||||
@@ -0,0 +1,18 @@
|
||||
import paramiko
|
||||
|
||||
from gpu_rent.tunnel import _patch_paramiko_for_sshtunnel, _ssh_tunnel_forwarder
|
||||
|
||||
|
||||
def test_dsskey_shim_allows_sshtunnel_import():
|
||||
# Simulate paramiko>=4 (no DSSKey) then ensure sshtunnel can load.
|
||||
had = getattr(paramiko, "DSSKey", None)
|
||||
if had is not None:
|
||||
delattr(paramiko, "DSSKey")
|
||||
try:
|
||||
_patch_paramiko_for_sshtunnel()
|
||||
assert hasattr(paramiko, "DSSKey")
|
||||
cls = _ssh_tunnel_forwarder()
|
||||
assert cls is not None
|
||||
finally:
|
||||
if had is not None:
|
||||
paramiko.DSSKey = had
|
||||
@@ -0,0 +1,32 @@
|
||||
from pathlib import Path
|
||||
|
||||
from gpu_rent.varsfile import apply_vars_file, parse_vars_file, split_args
|
||||
|
||||
|
||||
def test_parse_vars_file(tmp_path: Path):
|
||||
path = tmp_path / "gpu-rent.vars"
|
||||
path.write_text(
|
||||
"# comment\n"
|
||||
"GPU_RENT_DEFAULT_ARGS=up --yes\n"
|
||||
"export UPDATE_GIT=false\n"
|
||||
"QUOTED=\"a b\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
data = parse_vars_file(path)
|
||||
assert data["GPU_RENT_DEFAULT_ARGS"] == "up --yes"
|
||||
assert data["UPDATE_GIT"] == "false"
|
||||
assert data["QUOTED"] == "a b"
|
||||
|
||||
|
||||
def test_apply_does_not_override(tmp_path: Path, monkeypatch):
|
||||
path = tmp_path / "gpu-rent.vars"
|
||||
path.write_text("FOO=from-file\n", encoding="utf-8")
|
||||
monkeypatch.setenv("FOO", "from-env")
|
||||
apply_vars_file(path, override=False)
|
||||
import os
|
||||
|
||||
assert os.environ["FOO"] == "from-env"
|
||||
|
||||
|
||||
def test_split_args():
|
||||
assert split_args("up --yes --no-update") == ["up", "--yes", "--no-update"]
|
||||
Reference in New Issue
Block a user