Update documentation and CLI behavior for GPU management
- Clarified the behavior of `Ctrl+C` and `Ctrl+D` in the README and other documentation, specifying that `Ctrl+C` only stops the tunnel while keeping the GPU active, and `Ctrl+D` stops the GPU while preserving disk data. - Enhanced the CLI documentation to reflect these changes, ensuring users understand the implications of these commands during GPU operations. - Improved the handling of data bindings and remounting logic in the codebase to prevent issues with empty model tabs in the UI. - Added tests to validate the new command behaviors and ensure proper documentation alignment.
This commit is contained in:
Leonid Pershin
+157
-90
@@ -28,7 +28,7 @@ from gpu_rent.manifests import (
|
||||
repo_dirname,
|
||||
repo_matches_runtime,
|
||||
)
|
||||
from gpu_rent.ssh_ops import put_text, remote_exists, run_python, run_ssh
|
||||
from gpu_rent.ssh_ops import put_text, remote_exists, run_python, run_script_sudo, run_ssh
|
||||
from gpu_rent.sync_files import pull_tree, push_tree
|
||||
|
||||
Log = Callable[[str], None]
|
||||
@@ -109,6 +109,26 @@ def tune_swarm_perf(cfg: Config, host: str, log: Log) -> bool:
|
||||
return "RESTART_SWARMUI=1" in out
|
||||
|
||||
|
||||
def ensure_data_binds(
|
||||
cfg: Config, host: str, log: Log, *, stop_swarm: bool = True
|
||||
) -> None:
|
||||
"""Re-bind Models/Data/Output/dlbackend without lazy umount.
|
||||
|
||||
``umount -l`` while Comfy holds files makes the Models tab go empty later
|
||||
(dropdown still shows the last checkpoint).
|
||||
"""
|
||||
env = None if stop_swarm else {"GPU_RENT_STOP_SWARM": "0"}
|
||||
run_script_sudo(
|
||||
cfg,
|
||||
host,
|
||||
_pkg_text("ensure_binds.sh"),
|
||||
remote_path="/tmp/gpu-rent-ensure_binds.sh",
|
||||
timeout=180,
|
||||
env=env,
|
||||
log=log,
|
||||
)
|
||||
|
||||
|
||||
def ensure_swarm_comfy_installed(cfg: Config, host: str, log: Log) -> None:
|
||||
"""Headless Comfy install / recover errored backends before ready wait."""
|
||||
# Diag script next to install so recover-fail can subprocess it on the VM.
|
||||
@@ -212,13 +232,11 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool:
|
||||
dest = f"{dest_dir}/{cfg.autocomplete_filename}"
|
||||
meta_path = f"{dest}.gpu-rent-meta.json"
|
||||
blob = _github_blob(cfg)
|
||||
if blob is None:
|
||||
return False
|
||||
if blob.get("error"):
|
||||
if blob and blob.get("error"):
|
||||
log(str(blob["error"]))
|
||||
return False
|
||||
sha = str(blob.get("sha") or "")
|
||||
download_url = str(blob.get("download_url") or "")
|
||||
github_ok = isinstance(blob, dict) and not blob.get("error")
|
||||
sha = str(blob.get("sha") or "") if github_ok else ""
|
||||
download_url = str(blob.get("download_url") or "") if github_ok else ""
|
||||
old_sha = ""
|
||||
if remote_exists(cfg, host, meta_path):
|
||||
raw = run_ssh(cfg, host, f"cat {meta_path}", check=False)
|
||||
@@ -226,54 +244,52 @@ def seed_autocomplete(cfg: Config, host: str, log: Log) -> bool:
|
||||
old_sha = str(json.loads(raw).get("github_blob_sha") or "")
|
||||
except json.JSONDecodeError:
|
||||
old_sha = ""
|
||||
changed = sha != old_sha or not remote_exists(cfg, host, dest)
|
||||
changed = bool(github_ok) and (sha != old_sha or not remote_exists(cfg, host, dest))
|
||||
if changed:
|
||||
if not download_url:
|
||||
log("GitHub не дал download_url")
|
||||
return False
|
||||
log(f"качаю {cfg.autocomplete_filename}")
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"mkdir -p {dir} && curl -fsSL -o {part} {url} && mv {part} {dest}".format(
|
||||
dir=shlex.quote(dest_dir),
|
||||
part=shlex.quote(dest + ".partial"),
|
||||
url=shlex.quote(download_url),
|
||||
dest=shlex.quote(dest),
|
||||
),
|
||||
timeout=180,
|
||||
)
|
||||
meta = {
|
||||
"repo": cfg.autocomplete_github_repo,
|
||||
"path": cfg.autocomplete_github_path,
|
||||
"ref": cfg.autocomplete_github_ref,
|
||||
"github_blob_sha": sha,
|
||||
"filename": cfg.autocomplete_filename,
|
||||
"fetched_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
|
||||
"settings_applied": True,
|
||||
}
|
||||
put_text(cfg, host, meta_path, json.dumps(meta, indent=2) + "\n")
|
||||
changed = False
|
||||
else:
|
||||
log(f"качаю {cfg.autocomplete_filename}")
|
||||
run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
"mkdir -p {dir} && curl -fsSL -o {part} {url} && mv {part} {dest}".format(
|
||||
dir=shlex.quote(dest_dir),
|
||||
part=shlex.quote(dest + ".partial"),
|
||||
url=shlex.quote(download_url),
|
||||
dest=shlex.quote(dest),
|
||||
),
|
||||
timeout=180,
|
||||
)
|
||||
meta = {
|
||||
"repo": cfg.autocomplete_github_repo,
|
||||
"path": cfg.autocomplete_github_path,
|
||||
"ref": cfg.autocomplete_github_ref,
|
||||
"github_blob_sha": sha,
|
||||
"filename": cfg.autocomplete_filename,
|
||||
"fetched_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
|
||||
"settings_applied": False,
|
||||
}
|
||||
put_text(cfg, host, meta_path, json.dumps(meta, indent=2) + "\n")
|
||||
settings = f"{DATA}/Data/Settings.fds"
|
||||
applied = False
|
||||
if remote_exists(cfg, host, meta_path):
|
||||
raw = run_ssh(cfg, host, f"cat {meta_path}", check=False)
|
||||
try:
|
||||
applied = bool(json.loads(raw).get("settings_applied"))
|
||||
except json.JSONDecodeError:
|
||||
applied = False
|
||||
if not applied:
|
||||
_merge_autocomplete_into_settings(cfg, host, settings, cfg.autocomplete_filename, log)
|
||||
if remote_exists(cfg, host, meta_path):
|
||||
try:
|
||||
meta_obj = json.loads(
|
||||
run_ssh(cfg, host, f"cat {meta_path}", check=False)
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
meta_obj = {}
|
||||
if isinstance(meta_obj, dict):
|
||||
meta_obj["settings_applied"] = True
|
||||
put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n")
|
||||
return changed
|
||||
settings_status = "noop"
|
||||
if remote_exists(cfg, host, dest):
|
||||
settings_status = _merge_autocomplete_into_settings(
|
||||
cfg, host, settings, cfg.autocomplete_filename, log
|
||||
)
|
||||
if settings_status in ("changed", "already", "skip_user"):
|
||||
if remote_exists(cfg, host, meta_path):
|
||||
try:
|
||||
meta_obj = json.loads(
|
||||
run_ssh(cfg, host, f"cat {meta_path}", check=False)
|
||||
)
|
||||
except json.JSONDecodeError:
|
||||
meta_obj = {}
|
||||
if isinstance(meta_obj, dict):
|
||||
meta_obj["settings_applied"] = True
|
||||
put_text(cfg, host, meta_path, json.dumps(meta_obj, indent=2) + "\n")
|
||||
return changed or settings_status == "changed"
|
||||
|
||||
|
||||
_AUTOCOMPLETE_MERGE_PY = r'''
|
||||
@@ -335,6 +351,77 @@ def sync_is_installed(text: str) -> tuple[str, str | None]:
|
||||
return text, None
|
||||
|
||||
|
||||
def set_autocomplete_source(text: str, fname: str) -> tuple[str, str]:
|
||||
"""Fill AutoComplete.Source when empty; keep a user-chosen non-empty file."""
|
||||
lines = text.splitlines(keepends=True)
|
||||
ac_idx = None
|
||||
ac_indent = ""
|
||||
for i, line in enumerate(lines):
|
||||
m = re.match(r"^([ \t]*)AutoComplete:\s*$", line)
|
||||
if m:
|
||||
ac_idx = i
|
||||
ac_indent = m.group(1)
|
||||
break
|
||||
if ac_idx is not None:
|
||||
child = ac_indent + " "
|
||||
src_idx = None
|
||||
src_val = None
|
||||
end = ac_idx + 1
|
||||
while end < len(lines):
|
||||
raw = lines[end]
|
||||
if raw.strip() == "":
|
||||
end += 1
|
||||
continue
|
||||
if (
|
||||
raw.startswith(ac_indent)
|
||||
and len(raw.rstrip("\n")) > len(ac_indent)
|
||||
and raw[len(ac_indent)] in " \t"
|
||||
):
|
||||
sm = re.match(r"^[ \t]*Source:\s*(.*)$", raw)
|
||||
if sm:
|
||||
src_idx = end
|
||||
src_val = sm.group(1).strip()
|
||||
end += 1
|
||||
continue
|
||||
break
|
||||
if src_val in ("\\x", "x"):
|
||||
src_val = ""
|
||||
if src_val == fname:
|
||||
return text, f"ALREADY AutoComplete.Source={fname}"
|
||||
if src_val:
|
||||
return text, f"SKIP_USER AutoComplete.Source={src_val}"
|
||||
src_line = f"{child}Source: {fname}\n"
|
||||
if src_idx is not None:
|
||||
nl = "\n" if lines[src_idx].endswith("\n") else ""
|
||||
lines[src_idx] = f"{child}Source: {fname}{nl}"
|
||||
return "".join(lines), f"CHANGED patched AutoComplete.Source={fname}"
|
||||
lines.insert(ac_idx + 1, src_line)
|
||||
return "".join(lines), f"CHANGED inserted AutoComplete.Source={fname}"
|
||||
if re.search(r"^DefaultUser:\s*$", text, re.M):
|
||||
new = re.sub(
|
||||
r"^(DefaultUser:\s*\n)",
|
||||
(
|
||||
r"\1 AutoComplete:\n"
|
||||
f" Source: {fname}\n"
|
||||
" EscapeParens: true\n"
|
||||
),
|
||||
text,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
return new, f"CHANGED inserted AutoComplete under DefaultUser Source={fname}"
|
||||
ac_block = (
|
||||
"DefaultUser:\n"
|
||||
" AutoComplete:\n"
|
||||
f" Source: {fname}\n"
|
||||
" EscapeParens: true\n"
|
||||
)
|
||||
return (
|
||||
text.rstrip() + "\n\n" + ac_block,
|
||||
f"CHANGED appended DefaultUser.AutoComplete Source={fname}",
|
||||
)
|
||||
|
||||
|
||||
p = Path(os.environ.get("GPU_RENT_SETTINGS_FDS") or "/mnt/swarm_data/Data/Settings.fds")
|
||||
fname = (os.environ.get("GPU_RENT_AUTOCOMPLETE_FILE") or "").strip()
|
||||
if not fname:
|
||||
@@ -358,7 +445,7 @@ if not p.is_file():
|
||||
prefix = installed_prefix if comfy_on_disk() else ""
|
||||
p.write_text(prefix + ac_block, encoding="utf-8")
|
||||
extra = "+IsInstalled" if prefix else "без IsInstalled (первый Comfy install)"
|
||||
print(f"created Settings.fds AutoComplete.Source={fname} {extra}")
|
||||
print(f"CHANGED created Settings.fds AutoComplete.Source={fname} {extra}")
|
||||
raise SystemExit(0)
|
||||
|
||||
text = p.read_text(encoding="utf-8", errors="replace")
|
||||
@@ -368,40 +455,10 @@ if note:
|
||||
print(note)
|
||||
text = p.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
if re.search(rf"^\s*Source:\s*{re.escape(fname)}\s*$", text, re.M):
|
||||
print(f"AutoComplete.Source already {fname}")
|
||||
raise SystemExit(0)
|
||||
|
||||
# Replace Source line if AutoComplete section exists
|
||||
new, n = re.subn(
|
||||
r"(^[ \t]*Source:\s*).*$",
|
||||
rf"\1{fname}",
|
||||
text,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
if n and "AutoComplete" in text:
|
||||
p.write_text(new, encoding="utf-8")
|
||||
print(f"patched AutoComplete.Source={fname}")
|
||||
raise SystemExit(0)
|
||||
|
||||
if re.search(r"^DefaultUser:\s*$", text, re.M):
|
||||
new = re.sub(
|
||||
r"^(DefaultUser:\s*\n)",
|
||||
(
|
||||
r"\1 AutoComplete:\n"
|
||||
f" Source: {fname}\n"
|
||||
" EscapeParens: true\n"
|
||||
),
|
||||
text,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
p.write_text(new, encoding="utf-8")
|
||||
print(f"inserted AutoComplete under DefaultUser Source={fname}")
|
||||
else:
|
||||
p.write_text(text.rstrip() + "\n\n" + ac_block, encoding="utf-8")
|
||||
print(f"appended DefaultUser.AutoComplete Source={fname}")
|
||||
text, src_note = set_autocomplete_source(text, fname)
|
||||
print(src_note)
|
||||
if src_note.startswith("CHANGED"):
|
||||
p.write_text(text if text.endswith("\n") else text + "\n", encoding="utf-8")
|
||||
'''
|
||||
|
||||
|
||||
@@ -548,8 +605,8 @@ print("CHANGED" if changed else "NOOP")
|
||||
|
||||
def _merge_autocomplete_into_settings(
|
||||
cfg: Config, host: str, settings_path: str, filename: str, log: Log
|
||||
) -> None:
|
||||
"""Patch AutoComplete.Source in Settings.fds without wiping the rest."""
|
||||
) -> str:
|
||||
"""Patch AutoComplete.Source. Returns changed|already|skip_user|noop."""
|
||||
out = run_python(
|
||||
cfg,
|
||||
host,
|
||||
@@ -562,9 +619,19 @@ def _merge_autocomplete_into_settings(
|
||||
"GPU_RENT_AUTOCOMPLETE_FILE": filename,
|
||||
},
|
||||
)
|
||||
status = "noop"
|
||||
for line in (out or "").splitlines():
|
||||
if line.strip():
|
||||
log(line.strip())
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
log(line)
|
||||
if line.startswith("CHANGED"):
|
||||
status = "changed"
|
||||
elif line.startswith("ALREADY") and status != "changed":
|
||||
status = "already"
|
||||
elif line.startswith("SKIP_USER") and status != "changed":
|
||||
status = "skip_user"
|
||||
return status
|
||||
|
||||
|
||||
def ensure_settings_is_installed(cfg: Config, host: str, log: Log) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user