Add support for gpu-rent.vars and enhance git update functionality

- Updated .gitignore to include gpu-rent.vars.
- Modified env.example to introduce the UPDATE_GIT variable for controlling git updates during execution.
- Implemented Import-GpuRentVars function in gpu-rent.ps1 to load environment variables from gpu-rent.vars.
- Enhanced gpu-rent.sh to support loading variables from gpu-rent.vars and added logic for handling default and extra arguments.
- Updated CLI documentation to reflect the new gpu-rent.vars file and its usage in configuration.
- Improved bootstrap and provisioning logic to conditionally perform git updates based on the new configuration.
This commit is contained in:
Leonid Pershin
2026-08-21 05:01:53 +03:00
parent ec42830579
commit a9cf2e0f90
23 changed files with 515 additions and 39 deletions
+9 -2
View File
@@ -16,8 +16,12 @@ def bootstrap_script() -> str:
return files("gpu_rent.remote").joinpath("bootstrap.sh").read_text(encoding="utf-8")
def run_bootstrap(cfg: Config, host: str, log: Log) -> None:
def run_bootstrap(cfg: Config, host: str, log: Log, *, update: bool = True) -> None:
log("bootstrap SwarmUI на VM (идемпотентно, без Docker)")
if update:
log("git update: SwarmUI on")
else:
log("git update: SwarmUI off (--no-update)")
script = bootstrap_script()
out = run_script_sudo(
cfg,
@@ -25,7 +29,10 @@ def run_bootstrap(cfg: Config, host: str, log: Log) -> None:
script,
remote_path="/tmp/gpu-rent-bootstrap.sh",
timeout=1800,
env={"SWARM_USER": cfg.ssh_user},
env={
"SWARM_USER": cfg.ssh_user,
"GPU_RENT_UPDATE_GIT": "1" if update else "0",
},
log=log,
)
if "bootstrap ok" not in out:
+6
View File
@@ -301,6 +301,11 @@ def up(
"--open/--no-open",
help="После туннеля открыть браузер на 17801 (по умолчанию да)",
),
no_update: bool = typer.Option(
False,
"--no-update",
help="Не делать git pull SwarmUI и установленных extensions",
),
) -> None:
"""Create/unshelve GPU, bootstrap SwarmUI, по умолчанию туннель на :17801."""
try:
@@ -319,6 +324,7 @@ def up(
flavor=flavor,
yes=yes,
adopt=adopt,
update=False if no_update else None,
confirm=confirm,
log=lambda m: console.print(m),
)
+7
View File
@@ -17,7 +17,9 @@ from gpu_rent.paths import (
migrate_legacy_if_needed,
models_manifest_path,
runtime_dir,
vars_path,
)
from gpu_rent.varsfile import apply_vars_file
def _as_bool(value: str | None, default: bool) -> bool:
@@ -76,6 +78,7 @@ class Config:
swarmui_local_port: int
swarmui_image: str
update_git: bool
default_flavor_id: str
flavor_preference: tuple[str, ...]
@@ -108,6 +111,9 @@ def load_config(*, require_auth: bool = True) -> Config:
if env_file.is_file():
load_dotenv(env_file, override=False)
# Non-secret launch defaults (gpu-rent.vars). Do not override .env / real env.
apply_vars_file(vars_path(), override=False)
missing: list[str] = []
required = (
"OS_AUTH_URL",
@@ -180,6 +186,7 @@ def load_config(*, require_auth: bool = True) -> Config:
autocomplete_filename=(os.environ.get("AUTOCOMPLETE_FILENAME") or "danbooru.csv").strip(),
swarmui_local_port=_as_int(os.environ.get("SWARMUI_LOCAL_PORT"), 17801),
swarmui_image=(os.environ.get("SWARMUI_IMAGE") or "").strip(),
update_git=_as_bool(os.environ.get("UPDATE_GIT"), True),
default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(),
flavor_preference=_csv(
os.environ.get("FLAVOR_PREFERENCE"),
+8
View File
@@ -40,6 +40,14 @@ def extensions_manifest_path() -> Path:
return app_root() / "extensions.yaml"
def vars_path() -> Path:
return app_root() / "gpu-rent.vars"
def vars_example_path() -> Path:
return app_root() / "gpu-rent.vars.example"
def state_path() -> Path:
return runtime_dir() / "state.json"
+14 -6
View File
@@ -33,9 +33,9 @@ def _pkg_text(name: str) -> str:
return files("gpu_rent.remote").joinpath(name).read_text(encoding="utf-8")
def seed_extensions(cfg: Config, host: str, log: Log) -> bool:
def seed_extensions(cfg: Config, host: str, log: Log, *, update: bool = True) -> bool:
repos = parse_extensions(cfg.extensions_manifest)
if not repos:
if not repos and not update:
log("extensions.yaml пуст — стоковый SwarmUI")
return False
jobs = []
@@ -49,9 +49,13 @@ def seed_extensions(cfg: Config, host: str, log: Log) -> bool:
}
)
put_text(cfg, host, "/tmp/gpu-rent-ext.json", json.dumps(jobs, indent=2))
put_text(cfg, host, "/tmp/gpu-rent-update-git", "1\n" if update else "0\n")
if cfg.git_token:
put_text(cfg, host, "/tmp/gpu-rent-git.token", cfg.git_token + "\n", mode=0o600)
log(f"clone {len(jobs)} git-реп на data volume")
if update:
log(f"extensions: clone/update {len(jobs)} из yaml + установленные на data")
else:
log(f"extensions: только недостающие из yaml ({len(jobs)}), без git pull (--no-update)")
out = run_python(
cfg,
host,
@@ -224,7 +228,10 @@ def seed_civitai(cfg: Config, host: str, log: Log) -> None:
log("в манифесте нет checkpoint — генерация может не стартовать")
put_text(cfg, host, "/tmp/gpu-rent-civitai-jobs.json", json.dumps(jobs, indent=2))
put_text(cfg, host, "/tmp/gpu-rent-civitai.token", cfg.civitai_api_token + "\n", mode=0o600)
log(f"Civitai: качаю {len(jobs)} файл(ов) на VM")
log(
f"Civitai: {len(jobs)} в манифесте — на VM качаю отсутствующие "
f"(уже есть + sha → skip; прогресс [N/{len(jobs)}])"
)
run_python(
cfg,
host,
@@ -254,10 +261,11 @@ def provision_vm(
*,
conn=None,
server_id: str | None = None,
update: bool = True,
) -> None:
restart = False
restart = bool(update)
try:
if seed_extensions(cfg, host, log):
if seed_extensions(cfg, host, log, update=update):
restart = True
except GpuRentError as exc:
log(f"extensions: {exc}")
+14 -1
View File
@@ -97,8 +97,21 @@ if [[ ! -d "${SWARM_ROOT}/.git" ]]; then
log "clone SwarmUI -> ${SWARM_ROOT}"
mkdir -p "$(dirname "$SWARM_ROOT")"
git clone --depth 1 "$SWARM_REPO" "$SWARM_ROOT"
elif [[ "${GPU_RENT_UPDATE_GIT:-1}" == "1" ]]; then
log "обновляю SwarmUI в ${SWARM_ROOT}"
branch="$(git -C "$SWARM_ROOT" remote show origin 2>/dev/null | sed -n '/HEAD branch/s/.*: //p' || true)"
branch="${branch:-master}"
# shallow clone: deepen tip of default branch
git -C "$SWARM_ROOT" fetch --depth 1 origin "$branch" || git -C "$SWARM_ROOT" fetch --depth 1 origin
if git -C "$SWARM_ROOT" rev-parse --verify -q "origin/${branch}" >/dev/null; then
git -C "$SWARM_ROOT" checkout -B "$branch" "origin/${branch}"
git -C "$SWARM_ROOT" reset --hard "origin/${branch}"
else
git -C "$SWARM_ROOT" pull --ff-only || true
fi
log "SwarmUI @ $(git -C "$SWARM_ROOT" rev-parse --short HEAD)"
else
log "SwarmUI уже в ${SWARM_ROOT}"
log "SwarmUI уже в ${SWARM_ROOT} (update off)"
fi
if [[ ! -x /usr/share/dotnet/dotnet && ! -x "/home/${SWARM_USER}/.dotnet/dotnet" ]]; then
+33 -5
View File
@@ -25,6 +25,19 @@ def sha256_path(path: Path) -> str:
return digest.hexdigest()
def should_skip(dest: Path, expect_sha: str) -> tuple[bool, str]:
"""Return (skip, reason). Existing file with matching sha — or any non-empty if no sha."""
if not dest.is_file() or dest.stat().st_size <= 0:
return False, ""
expect = (expect_sha or "").lower()
if not expect:
return True, "уже есть"
got = sha256_path(dest).lower()
if got == expect:
return True, "уже есть (sha ok)"
return False, "sha не совпал — перекачиваю"
def download(url: str, dest: Path, token: str) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
partial = dest.with_suffix(dest.suffix + ".partial")
@@ -61,16 +74,30 @@ def main() -> int:
return 1
token = TOKEN_PATH.read_text(encoding="utf-8").strip()
jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8"))
total = len(jobs)
failed = 0
for job in jobs:
skipped = 0
downloaded = 0
for index, job in enumerate(jobs, start=1):
dest = Path(job["dest"])
expect = (job.get("sha256") or "").lower()
if dest.is_file() and expect and sha256_path(dest).lower() == expect:
print(f"skip {dest}")
prefix = f"[{index}/{total}]"
skip, reason = should_skip(dest, expect)
if skip:
skipped += 1
print(f"{prefix} {reason}: {dest.name}")
# refresh sidecars even on skip
for extra_name, extra_text in (job.get("sidecars") or {}).items():
extra = dest.parent / extra_name
extra.parent.mkdir(parents=True, exist_ok=True)
extra.write_text(extra_text, encoding="utf-8")
continue
if reason:
print(f"{prefix} {reason}: {dest.name}")
try:
print(f"download {dest.name}")
print(f"{prefix} качаю: {dest.name}")
download(job["url"], dest, token)
downloaded += 1
if expect:
got = sha256_path(dest).lower()
if got != expect:
@@ -81,8 +108,9 @@ def main() -> int:
extra.write_text(extra_text, encoding="utf-8")
except Exception as exc:
failed += 1
print(f"FAIL {dest}: {exc}", file=sys.stderr)
print(f"{prefix} FAIL {dest.name}: {exc}", file=sys.stderr)
TOKEN_PATH.unlink(missing_ok=True)
print(f"Civitai итог: скачано {downloaded}, пропущено {skipped}, ошибок {failed} (из {total})")
if failed:
return 1
MARKER.parent.mkdir(parents=True, exist_ok=True)
+80 -7
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env python3
"""Clone git extensions on the VM. Stdlib only. Token file optional."""
"""Clone/update git extensions on the VM. Stdlib only. Token file optional."""
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
@@ -10,7 +11,12 @@ from urllib.parse import urlsplit, urlunsplit
TOKEN_PATH = Path("/tmp/gpu-rent-git.token")
JOBS_PATH = Path("/tmp/gpu-rent-ext.json")
UPDATE_PATH = Path("/tmp/gpu-rent-update-git")
MARKER = Path("/mnt/swarm_data/.gpu-rent-extensions-seeded")
EXTRA_ROOTS = (
Path("/mnt/swarm_data/Extensions"),
Path("/mnt/swarm_data/DLNodes"),
)
def strip_auth(url: str) -> str:
@@ -35,25 +41,64 @@ def run(argv: list[str], cwd: str | None = None) -> None:
subprocess.check_call(argv, cwd=cwd)
def out(argv: list[str], cwd: str | None = None) -> str:
return subprocess.check_output(argv, cwd=cwd, text=True).strip()
def is_sha(ref: str) -> bool:
ref = ref.strip()
return len(ref) == 40 and all(c in "0123456789abcdefABCDEF" for c in ref)
def clone_one(job: dict, token: str) -> None:
def do_update() -> bool:
if UPDATE_PATH.is_file():
return UPDATE_PATH.read_text(encoding="utf-8").strip() not in {"0", "false", "no", "off"}
return (os.environ.get("GPU_RENT_UPDATE_GIT") or "1").strip() not in {"0", "false", "no", "off"}
def fetch_and_checkout(dest: Path, ref: str) -> None:
run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"])
if is_sha(ref):
run(["git", "-C", str(dest), "checkout", "--detach", ref])
print(f"updated {dest} @ {ref[:12]}")
return
run(["git", "-C", str(dest), "checkout", ref])
# Move branch tip to remote (shallow-friendly).
try:
run(["git", "-C", str(dest), "reset", "--hard", f"origin/{ref}"])
except subprocess.CalledProcessError:
run(["git", "-C", str(dest), "pull", "--ff-only", "origin", ref])
print(f"updated {dest} ({ref})")
def update_tracking_branch(dest: Path) -> None:
branch = out(["git", "-C", str(dest), "rev-parse", "--abbrev-ref", "HEAD"])
if not branch or branch == "HEAD":
print(f"skip detached {dest}")
return
run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"])
try:
run(["git", "-C", str(dest), "reset", "--hard", f"origin/{branch}"])
except subprocess.CalledProcessError:
run(["git", "-C", str(dest), "pull", "--ff-only"])
print(f"updated installed {dest} ({branch})")
def clone_one(job: dict, token: str, update: bool) -> None:
dest = Path(job["dest"])
url = job["url"].strip()
ref = (job.get("ref") or "main").strip()
dest.parent.mkdir(parents=True, exist_ok=True)
authed = with_token(url, token)
if dest.is_dir() and (dest / ".git").is_dir():
origin = subprocess.check_output(["git", "-C", str(dest), "remote", "get-url", "origin"], text=True).strip()
origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"])
if strip_auth(origin) != strip_auth(url):
print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr)
raise SystemExit(2)
run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "origin"])
run(["git", "-C", str(dest), "checkout", ref])
print(f"updated {dest}")
if not update:
print(f"skip update {dest}")
return
fetch_and_checkout(dest, ref)
return
if dest.exists():
print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr)
@@ -71,16 +116,44 @@ def clone_one(job: dict, token: str) -> None:
print(f"cloned {dest}")
def update_installed_extras(known: set[str], update: bool) -> None:
if not update:
return
for root in EXTRA_ROOTS:
if not root.is_dir():
continue
for child in sorted(root.iterdir()):
if not child.is_dir() or not (child / ".git").is_dir():
continue
key = str(child)
if key in known:
continue
try:
update_tracking_branch(child)
except Exception as exc:
print(f"FAIL installed {child}: {exc}", file=sys.stderr)
raise
def main() -> int:
token = TOKEN_PATH.read_text(encoding="utf-8").strip() if TOKEN_PATH.is_file() else ""
jobs = json.loads(JOBS_PATH.read_text(encoding="utf-8"))
update = do_update()
print(f"extensions update={'on' if update else 'off'}")
failed = 0
known: set[str] = set()
for job in jobs:
try:
clone_one(job, token)
dest = str(Path(job["dest"]))
known.add(dest)
clone_one(job, token, update)
except Exception as exc:
failed += 1
print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr)
try:
update_installed_extras(known, update)
except Exception:
failed += 1
if TOKEN_PATH.is_file():
TOKEN_PATH.unlink()
if failed:
+33 -11
View File
@@ -46,7 +46,7 @@ from gpu_rent.os_client import (
iter_volume_types,
)
from gpu_rent.ssh_keys import ensure_ed25519
from gpu_rent.ssh_ops import probe_ssh, wait_ssh
from gpu_rent.ssh_ops import probe_ssh, run_ssh, wait_ssh
from gpu_rent.state import SessionState, load_state, save_state, utc_now
Log = Callable[[str], None]
@@ -66,7 +66,15 @@ def _require_gpu_quota(conn) -> None:
)
def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> SessionState:
def _bind_access(
conn,
server,
state: SessionState,
cfg: Config,
log: Log,
*,
update: bool = True,
) -> SessionState:
ip, fip_id = ensure_floating_ip(
conn, server, state.floating_ip_id, state.floating_ip, log
)
@@ -76,8 +84,20 @@ def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> Se
save_state(state)
wait_ssh(cfg, ip)
log(f"SSH {cfg.ssh_user}@{ip}")
run_bootstrap(cfg, ip, log)
provision_vm(cfg, ip, log, conn=conn, server_id=getattr(server, "id", None) or state.server_id)
if update:
active = run_ssh(cfg, ip, "systemctl is-active swarmui 2>/dev/null || true", check=False).strip()
if active == "active":
log("systemctl stop swarmui перед git update")
run_ssh(cfg, ip, "sudo -n systemctl stop swarmui", timeout=120, check=False)
run_bootstrap(cfg, ip, log, update=update)
provision_vm(
cfg,
ip,
log,
conn=conn,
server_id=getattr(server, "id", None) or state.server_id,
update=update,
)
try:
wait_backend_idle(cfg, ip, log)
except CloudError as exc:
@@ -99,7 +119,7 @@ def _bind_access(conn, server, state: SessionState, cfg: Config, log: Log) -> Se
return state
def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState:
def adopt_server(cfg: Config, log: Log = _log_default, *, update: bool = True) -> SessionState:
conn = connect(cfg)
server = pick_existing_server(conn)
if not server:
@@ -115,7 +135,7 @@ def adopt_server(cfg: Config, log: Log = _log_default) -> SessionState:
save_state(state)
log(f"подхватили {server.id} статус {server_status(server)}")
if server_status(server) == "ACTIVE":
_bind_access(conn, server, state, cfg, log)
_bind_access(conn, server, state, cfg, log, update=update)
return state
@@ -126,12 +146,14 @@ def cmd_up(
flavor: str | None = None,
yes: bool = False,
adopt: bool = False,
update: bool | None = None,
confirm: Callable[[str], bool] | None = None,
log: Log = _log_default,
) -> SessionState:
do_update = cfg.update_git if update is None else update
with SessionLock():
if adopt:
return adopt_server(cfg, log=log)
return adopt_server(cfg, log=log, update=do_update)
conn = connect(cfg)
_require_gpu_quota(conn)
state = load_state()
@@ -145,7 +167,7 @@ def cmd_up(
log("сервер уже ACTIVE — второй GPU не создаём")
state.phase = "ready_cloud"
save_state(state)
_bind_access(conn, existing, state, cfg, log)
_bind_access(conn, existing, state, cfg, log, update=do_update)
return state
# Bootstrap не завершён: почти всегда VM без authorized_keys.
@@ -167,7 +189,7 @@ def cmd_up(
log("сервер ACTIVE, SSH ок — продолжаем bootstrap")
state.phase = "ready_cloud"
save_state(state)
_bind_access(conn, existing, state, cfg, log)
_bind_access(conn, existing, state, cfg, log, update=do_update)
return state
log(
@@ -194,7 +216,7 @@ def cmd_up(
state.phase = "ready_cloud"
state.unshelved_at = utc_now()
save_state(state)
_bind_access(conn, existing, state, cfg, log)
_bind_access(conn, existing, state, cfg, log, update=do_update)
return state
if existing is not None:
raise CloudError(f"сервер gpu-rent в статусе {status} — разбери в панели")
@@ -321,7 +343,7 @@ def cmd_up(
state.unshelved_at = None
state.phase = "ready_cloud"
save_state(state)
_bind_access(conn, server, state, cfg, log)
_bind_access(conn, server, state, cfg, log, update=do_update)
return state
+23 -2
View File
@@ -27,6 +27,27 @@ EXIT_STATUSES = frozenset(
SHELVED_STATUSES = frozenset({"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"})
def _patch_paramiko_for_sshtunnel() -> None:
"""sshtunnel 0.4 still refs paramiko.DSSKey; Paramiko 4+ removed it."""
import paramiko
if hasattr(paramiko, "DSSKey"):
return
class _DSSKeyRemoved(paramiko.PKey):
def __init__(self, *args, **kwargs):
raise paramiko.SSHException("DSA keys unsupported (paramiko>=4)")
paramiko.DSSKey = _DSSKeyRemoved # type: ignore[attr-defined, assignment]
def _ssh_tunnel_forwarder():
_patch_paramiko_for_sshtunnel()
from sshtunnel import SSHTunnelForwarder
return SSHTunnelForwarder
@dataclass
class WatchDecision:
kind: str # ok | reconnect | unshelve | exit
@@ -51,7 +72,7 @@ def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision:
def _start_forwarder(cfg: Config, host: str, local_port: int):
from sshtunnel import SSHTunnelForwarder
SSHTunnelForwarder = _ssh_tunnel_forwarder()
server = SSHTunnelForwarder(
(host, 22),
@@ -131,7 +152,7 @@ def run_tunnel(
poll_seconds: float = 30.0,
) -> None:
try:
from sshtunnel import SSHTunnelForwarder # noqa: F401
_ssh_tunnel_forwarder()
except ImportError as exc:
raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc
+47
View File
@@ -0,0 +1,47 @@
"""Load KEY=VALUE files without overriding existing environment."""
from __future__ import annotations
from pathlib import Path
import os
import shlex
def parse_vars_file(path: Path) -> dict[str, str]:
"""Parse simple KEY=VALUE lines (# comments). No shell expansion."""
out: dict[str, str] = {}
if not path.is_file():
return out
text = path.read_text(encoding="utf-8-sig")
for raw in text.splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
if line.lower().startswith("export "):
line = line[7:].strip()
if "=" not in line:
continue
key, _, value = line.partition("=")
key = key.strip()
if not key or not key.replace("_", "").isalnum():
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in {'"', "'"}:
value = value[1:-1]
out[key] = value
return out
def apply_vars_file(path: Path, *, override: bool = False) -> dict[str, str]:
"""Set os.environ from vars file. By default do not override existing keys."""
loaded = parse_vars_file(path)
for key, value in loaded.items():
if override or key not in os.environ or os.environ.get(key) == "":
os.environ[key] = value
return loaded
def split_args(value: str | None) -> list[str]:
if not value or not value.strip():
return []
return shlex.split(value, posix=True)