diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 116b56e..b9032d9 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -460,6 +460,11 @@ def up( "--no-update", help="Не делать git pull SwarmUI и установленных extensions", ), + force_update: bool = typer.Option( + False, + "--update", + help="Принудительно git pull даже на warm ACTIVE (по умолчанию warm без pull)", + ), keep_on_fail: bool = typer.Option( False, "--keep-on-fail", @@ -627,7 +632,7 @@ def up( flavor=flavor, yes=yes, adopt=adopt, - update=False if no_update else None, + update=(False if no_update else True if force_update else None), confirm=confirm, ask=None if yes else ask, log=log, diff --git a/src/gpu_rent/idle_killer.py b/src/gpu_rent/idle_killer.py index 7d78d91..84981ce 100644 --- a/src/gpu_rent/idle_killer.py +++ b/src/gpu_rent/idle_killer.py @@ -138,6 +138,57 @@ WantedBy=timers.target log(f"systemd timer {UNIT}.timer") +def _remote_killer_reusable(cfg: Config, host: str, server_id: str) -> bool: + """True if VM already has matching idle-killer creds + active timer.""" + cmd = ( + f"sid=$(cat {DATA}/.gpu-rent-server-id 2>/dev/null | tr -d '[:space:]'); " + f"echo sid=$sid; echo want={server_id}; " + f"test -f {DATA}/.gpu-rent-killer-creds-ok && echo creds=yes || echo creds=no; " + f"test -f {DATA}/.gpu-rent-killer-armed && echo armed=yes || echo armed=no; " + f"sudo -n test -f {CREDS_REMOTE} && echo rootcreds=yes || echo rootcreds=no; " + f"echo timer=$(systemctl is-active {UNIT}.timer 2>/dev/null || echo inactive)" + ) + try: + out = run_ssh(cfg, host, cmd, check=False, timeout=25) + except Exception: + return False + parsed: dict[str, str] = {} + for line in out.splitlines(): + if "=" in line: + k, v = line.strip().split("=", 1) + parsed[k] = v + return ( + parsed.get("sid") == server_id + and parsed.get("want") == server_id + and parsed.get("creds") == "yes" + and parsed.get("armed") == "yes" + and parsed.get("rootcreds") == "yes" + and parsed.get("timer") == "active" + ) + + +def _refresh_killer_grace(cfg: Config, host: str) -> None: + """Bump grace_from / idle settings on existing remote creds; clear idle-since.""" + run_ssh( + cfg, + host, + "sudo -n python3 -c \"" + "import json,time; from pathlib import Path; " + f"p=Path('{CREDS_REMOTE}'); d=json.loads(p.read_text()); " + f"d['idle_minutes']={int(cfg.idle_minutes)}; " + f"d['grace_minutes']={int(cfg.idle_grace_minutes)}; " + "d['grace_from']=int(time.time()); " + "p.write_text(json.dumps(d,indent=2)+chr(10)); p.chmod(0o600)" + "\" && " + f"sudo -n rm -f {DATA}/.gpu-rent-idle-since && " + f"date -u +%Y-%m-%dT%H:%M:%SZ | sudo -n tee {DATA}/.gpu-rent-killer-armed >/dev/null && " + f"sudo -n chmod 644 {DATA}/.gpu-rent-killer-armed " + f"{DATA}/.gpu-rent-killer-creds-ok {DATA}/.gpu-rent-server-id 2>/dev/null || true", + check=False, + timeout=30, + ) + + def arm_idle_killer( cfg: Config, host: str, @@ -147,6 +198,13 @@ def arm_idle_killer( ) -> None: if not server_id: raise CloudError("idle-killer: нет server_id") + if _remote_killer_reusable(cfg, host, server_id): + _refresh_killer_grace(cfg, host) + log( + f"idle-killer: reuse app cred (server {server_id[:12]}…) — " + f"льгота {cfg.idle_grace_minutes} мин, потом {cfg.idle_minutes} мин → delete" + ) + return creds = create_application_credential(conn, cfg, server_id, log) put_text( cfg, diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index 879280d..c512bd6 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -38,6 +38,18 @@ DATA = "/mnt/swarm_data" _OLLAMA_INSTALL_ENV = ("OLLAMA_VERSION", "OLLAMA_SHA256") +def _models_manifest_fp(cfg: Config) -> str: + """Stable fingerprint of models.yaml so warm seed can skip API+fetch.""" + import hashlib + + path = Path(cfg.models_manifest) + try: + raw = path.read_bytes() + except OSError: + raw = b"" + return hashlib.sha256(raw).hexdigest()[:16] + + def _remote_llm_env(cfg: Config, *keys: str) -> dict[str, str]: import os @@ -408,6 +420,41 @@ def seed_civitai(cfg: Config, host: str, log: Log) -> None: log("model-seed пропущен: манифест пуст — дефолт SwarmUI") return + # Warm short-circuit: last successful jobs list still present on disk. + cache_remote = f"{DATA}/.gpu-rent-models-jobs.json" + try: + cached_raw = run_ssh( + cfg, + host, + f"test -f {cache_remote} && cat {cache_remote} || true", + check=False, + timeout=20, + ).strip() + except Exception: + cached_raw = "" + if cached_raw.startswith("{"): + try: + cached = json.loads(cached_raw) + dests = [str(d) for d in (cached.get("dests") or []) if d] + fp = str(cached.get("fp") or "") + want_fp = _models_manifest_fp(cfg) + if dests and fp == want_fp: + check = " && ".join(f"test -f {shlex.quote(d)}" for d in dests) + ok = run_ssh( + cfg, + host, + f"if {check}; then echo ALL_OK; else echo MISSING; fi", + check=False, + timeout=30, + ).strip() + if "ALL_OK" in ok: + log( + f"model-seed: skip — {len(dests)} файл(ов) уже на диске (cache)" + ) + return + except (json.JSONDecodeError, TypeError, CloudError): + pass + jobs: list[dict] = [] for entry in entries: url = (entry.url or "").strip() @@ -514,6 +561,31 @@ def seed_civitai(cfg: Config, host: str, log: Log) -> None: if not any(e.kind == "checkpoint" for e in entries): log("в манифесте нет checkpoint — генерация может не стартовать") + + dests = [str(j.get("dest") or "") for j in jobs if j.get("dest")] + if dests: + check = " && ".join(f"test -f {shlex.quote(d)}" for d in dests) + present = run_ssh( + cfg, + host, + f"if {check}; then echo ALL_OK; else echo MISSING; fi", + check=False, + timeout=30, + ).strip() + if "ALL_OK" in present: + put_text( + cfg, + host, + cache_remote, + json.dumps({"fp": _models_manifest_fp(cfg), "dests": dests}, indent=2) + + "\n", + ) + log( + f"model-seed: skip fetch — {len(dests)} файл(ов) уже на диске " + f"(civitai={civ_n}, huggingface={hf_n})" + ) + return + put_text(cfg, host, "/tmp/gpu-rent-civitai-jobs.json", json.dumps(jobs, indent=2)) if cfg.civitai_api_token: put_text( @@ -533,6 +605,12 @@ def seed_civitai(cfg: Config, host: str, log: Log) -> None: timeout=7200, log=log, ) + put_text( + cfg, + host, + cache_remote, + json.dumps({"fp": _models_manifest_fp(cfg), "dests": dests}, indent=2) + "\n", + ) def ensure_swarmui_running(cfg: Config, host: str, log: Log, restart: bool) -> None: @@ -594,16 +672,39 @@ def provision_llm(cfg: Config, host: str, log: Log) -> None: if not names: log("ollama-models.yaml пуст — pull skip") else: - put_text(cfg, host, "/tmp/gpu-rent-ollama-models.json", json.dumps(names, indent=2)) - log(f"Ollama: pull {len(names)} из манифеста") - run_python( + # Fast path: all tags already present — skip upload/pull script. + listed = run_ssh( cfg, host, - _pkg_text("ollama_pull.py"), - remote_path="/tmp/gpu-rent-ollama_pull.py", - timeout=7200, - log=log, + "ollama list 2>/dev/null | awk 'NR>1 {print $1}' || true", + check=False, + timeout=30, ) + have = {ln.strip() for ln in listed.splitlines() if ln.strip()} + missing = [] + for name in names: + if name in have or ( + ":" not in name and f"{name}:latest" in have + ) or ( + name.endswith(":latest") and name.rsplit(":", 1)[0] in have + ): + continue + missing.append(name) + if not missing: + log(f"ollama pull: skip — все {len(names)} уже есть") + else: + put_text( + cfg, host, "/tmp/gpu-rent-ollama-models.json", json.dumps(missing, indent=2) + ) + log(f"Ollama: pull {len(missing)} из манифеста (нет: {len(missing)})") + run_python( + cfg, + host, + _pkg_text("ollama_pull.py"), + remote_path="/tmp/gpu-rent-ollama_pull.py", + timeout=7200, + log=log, + ) else: raise CloudError(f"неизвестный LLM_RUNTIME={runtime!r}") st = load_state() diff --git a/src/gpu_rent/remote/civitai_fetch.py b/src/gpu_rent/remote/civitai_fetch.py index baea0ef..37c1a0c 100644 --- a/src/gpu_rent/remote/civitai_fetch.py +++ b/src/gpu_rent/remote/civitai_fetch.py @@ -317,6 +317,19 @@ def main() -> int: return 1 MARKER.parent.mkdir(parents=True, exist_ok=True) MARKER.write_text("ok\n", encoding="utf-8") + # Dest list for warm seed short-circuit (local provision also writes fp). + try: + dests = [] + for job in json.loads(JOBS_PATH.read_text(encoding="utf-8")): + if isinstance(job, dict) and job.get("dest"): + dests.append(str(job["dest"])) + if dests: + Path("/mnt/swarm_data/.gpu-rent-models-jobs.json").write_text( + json.dumps({"fp": "", "dests": dests}, indent=2) + "\n", + encoding="utf-8", + ) + except Exception: + pass print("civitai seed ok") return 0 diff --git a/src/gpu_rent/remote/install_ollama.sh b/src/gpu_rent/remote/install_ollama.sh index 5ba1535..8f47793 100644 --- a/src/gpu_rent/remote/install_ollama.sh +++ b/src/gpu_rent/remote/install_ollama.sh @@ -143,7 +143,7 @@ while IFS= read -r line || [[ -n "$line" ]]; do ENV_LINES+="Environment=${line}"$'\n' done < "$OLLAMA_ENV_FILE" -cat >/etc/systemd/system/${UNIT}.service </tmp/${UNIT}.service.new <