From 2005b00175561c6d2f28559c544858a62051c025 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Fri, 21 Aug 2026 05:29:23 +0300 Subject: [PATCH] Update configuration and documentation for LLM support and local watchdog - Added `ollama-models.yaml` to .gitignore and implemented logic to copy it in gpu-rent.ps1 and gpu-rent.sh. - Enhanced env.example to include new variables for LLM runtime options and local watchdog configuration. - Updated CLI commands to support LLM options during setup and execution, including new flags for Ollama and llama.cpp. - Improved documentation in cli.md and README.md to reflect changes in LLM integration and local watchdog functionality. - Adjusted architecture and decisions documentation to clarify the role of LLMs and local watchdog in the system. --- .gitignore | 1 + docs/README.md | 1 + docs/architecture.md | 54 +-- docs/cli.md | 22 +- docs/concept.md | 2 +- docs/decisions.md | 5 +- docs/llm.md | 59 +++ docs/open-questions.md | 2 +- docs/reviews/2026-08-21-review-1.md | 111 ++++++ env.example | 11 +- gpu-rent.ps1 | 1 + gpu-rent.sh | 4 + gpu-rent.vars.example | 5 + ollama-models.example.yaml | 17 + src/gpu_rent/bootstrap.py | 15 +- src/gpu_rent/cli.py | 233 ++++++++++- src/gpu_rent/cloud.py | 28 +- src/gpu_rent/config.py | 20 + src/gpu_rent/doctor.py | 5 +- src/gpu_rent/idle_killer.py | 41 +- src/gpu_rent/llm_runtime.py | 166 ++++++++ src/gpu_rent/local_watchdog.py | 499 ++++++++++++++++++++++++ src/gpu_rent/os_client.py | 3 +- src/gpu_rent/paths.py | 20 + src/gpu_rent/provision.py | 66 ++++ src/gpu_rent/remote/bootstrap.sh | 8 +- src/gpu_rent/remote/clone_ext.py | 76 +++- src/gpu_rent/remote/idle_killer.py | 46 +++ src/gpu_rent/remote/install_llamacpp.sh | 103 +++++ src/gpu_rent/remote/install_ollama.sh | 57 +++ src/gpu_rent/remote/ollama_pull.py | 70 ++++ src/gpu_rent/session.py | 32 +- src/gpu_rent/setup_wizard.py | 100 +++++ src/gpu_rent/ssh_ops.py | 107 +++-- src/gpu_rent/sync_files.py | 84 ++-- src/gpu_rent/tunnel.py | 97 ++++- src/gpu_rent/ux.py | 4 +- tests/test_cli.py | 2 +- tests/test_llm_runtime.py | 59 +++ tests/test_local_watchdog.py | 87 +++++ tests/test_seed.py | 32 ++ tests/test_session.py | 76 ++-- tests/test_tunnel_watch.py | 24 +- 43 files changed, 2258 insertions(+), 197 deletions(-) create mode 100644 docs/llm.md create mode 100644 docs/reviews/2026-08-21-review-1.md create mode 100644 ollama-models.example.yaml create mode 100644 src/gpu_rent/llm_runtime.py create mode 100644 src/gpu_rent/local_watchdog.py create mode 100644 src/gpu_rent/remote/install_llamacpp.sh create mode 100644 src/gpu_rent/remote/install_ollama.sh create mode 100644 src/gpu_rent/remote/ollama_pull.py create mode 100644 src/gpu_rent/setup_wizard.py create mode 100644 tests/test_llm_runtime.py create mode 100644 tests/test_local_watchdog.py diff --git a/.gitignore b/.gitignore index 9fdb1f3..1661645 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .env.* !.env.example gpu-rent.vars +ollama-models.yaml models.yaml extensions.yaml diff --git a/docs/README.md b/docs/README.md index c964064..a4cc13c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,7 @@ CLI-оркестратор: поднимает прерываемый GPU-сер | [autocomplete.md](autocomplete.md) | Word-list промптов, Settings.fds, проверка версии | | [local-folders.md](local-folders.md) | `Models/` / `Wildcards/` / `CustomWorkflows/` push, optional `Output/` pull | | [swarmui.md](swarmui.md) | Нативный install, порт 7801, API, MCP | +| [llm.md](llm.md) | Opt-in Ollama / llama.cpp, `ollama-models.yaml`, порты 17811/17812 | | [cli.md](cli.md) | Команды, конфиг, локальный state | | [roadmap.md](roadmap.md) | Порядок реализации | | [open-questions.md](open-questions.md) | Ещё не закрыто | diff --git a/docs/architecture.md b/docs/architecture.md index f2b22bf..f865ff4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -34,23 +34,22 @@ | Модуль | Ответственность | | --- | --- | -| `cli` | Typer: `up`, `tunnel`, `open`, `status`, `stop`, `destroy`, `logs`, `ssh`, `doctor`, `hold`, `seed-*`, `push-models`, `pull-output`, `resize-data`, `dry-run` | -| `config` | `.env`, валидация, пути к ключам | +| `cli` | Typer: `up`, `setup`, `tunnel`, `open`, `status`, `stop`, `destroy`, `logs`, `ssh`, `doctor`, `hold`, `seed-*`, `push` / `pull-output`, `resize-data`, `watchdog`, `dry-run` | +| `config` / `varsfile` | `.env` + `gpu-rent.vars`, пути | | `state` | JSON сессии: ids, фаза, timestamps | -| `os_client` | `openstacksdk`, refresh IAM-токена | -| `inventory` | Flavors/images в сегменте, квоты, **фоллбек flavor** | -| `bootstrap` | Идемпотентный first-boot; после успеха — snapshot boot volume | +| `os_client` / `cloud` / `pools` | openstacksdk, ресурсы, скан пулов | +| `inventory` | Flavors/images, квоты, фоллбек flavor | +| `session` | `cmd_up` / `cmd_stop` / adopt | +| `bootstrap` + `remote/bootstrap.sh` | Идемпотентный first-boot; light без apt | +| `provision` | extensions, autocomplete, civitai seed, push, idle-killer arm, start SwarmUI | | `doctor` | Preflight без mutating compute | -| `civitai_seed` | Манифест + Civitai API на `.red` | -| `models_push` | SFTP `./Models`, `./Wildcards`, `./CustomWorkflows` | -| `output_pull` | Опциональный SFTP с VM `Output/` | -| `git_seed` | Clone `extensions.yaml` | -| `autocomplete_seed` | Word-list + Settings.fds | +| `sync_files` | SFTP `Models` / Wildcards / workflows / Output | | `notify` | Toast/звук при backend Idle | -| `tunnel` | paramiko / sshtunnel, порт **17801** | -| `watchdog` | EXPIRED → unshelve, пока туннель жив | -| `idle_killer` | systemd на VM + hold-файл + «качалка занята» | -| `reconcile` | Сироты по state и тегу `gpu-rent` | +| `tunnel` | sshtunnel + Nova EXPIRED watchdog | +| `local_watchdog` | Опциональный локальный тик → stop при unclean exit | +| `llm_runtime` / `setup_wizard` | Opt-in Ollama/llama.cpp + `ollama-models.yaml` | +| `idle_killer` / `hold` | systemd на VM + hold-файл | +| `ready` / `snapshot` | Idle backend + boot snapshot | Каталог сервисов — из Keystone, не из выдуманного `api.selectel.ru/v3/`. @@ -82,29 +81,32 @@ ## Стейт-машина +Фазы в `state.json` (код пишет только эти): + ``` idle │ gpu-rent up ▼ - provisioning → bootstrapping → seeding_extensions → seeding_autocomplete → seeding_models → waiting_ui + provisioning ← create volumes / server │ ▼ - ready_cloud ← compute жив, idle-killer вооружён + bootstrapping ← SSH + bootstrap + seed (пока не ready) │ - │ gpu-rent tunnel (опционально, пока ноут онлайн) ▼ - ready_tunneled ← localhost:17801 + ready_cloud ← compute жив, idle-killer вооружён │ - │ хостер: EXPIRED + │ gpu-rent tunnel (или up с туннелем) ▼ - restoring (unshelve) → ready_cloud / ready_tunneled + ready_tunneled ← localhost:17801 │ - │ stop | idle-killer | destroy + │ stop | idle-killer | local-watchdog | destroy ▼ - stopping → idle + idle ``` -`ready_cloud` ≠ Nova `ACTIVE`. ACTIVE бывает раньше SSH и UI. +Подшаги seed (extensions / autocomplete / models) идут внутри `bootstrapping`, отдельными фазами в state не пишутся. `ready_cloud` ≠ Nova `ACTIVE` (ACTIVE бывает раньше SSH и UI). + +На EXPIRED туннель сам делает unshelve → снова `bootstrapping`/`ready_*`. ## Idle-killer (на VM) @@ -118,7 +120,7 @@ Killer молчит: - пока существует hold: файл `/mnt/swarm_data/.gpu-rent-hold-until` с unix ts (пишет `gpu-rent hold`); - пока SwarmUI качает модель в UI (Model Downloader / активный download — точный JSON на spike). Нет сигнала — пользователь жмёт `hold`. -`gpu-rent hold` без аргументов = +`IDLE_MINUTES` от сейчас. `--minutes 90` — абсолютное продление. `--until` ISO опционально. `hold --clear` снимает. +`gpu-rent hold` без аргументов = +`IDLE_MINUTES` от сейчас. `--minutes 90` — hold до now+90 мин (заменяет предыдущий, не складывает). `--until` ISO опционально. `hold --clear` снимает. Потом счётчик 30 минут пустой очереди. @@ -167,7 +169,7 @@ Reconcile: сервер с тегом `gpu-rent` есть, локального | Слой | Выбор | | --- | --- | -| Python 3.11+ | openstacksdk, paramiko, Typer, Rich, questionary, dotenv | +| Python 3.11+ | openstacksdk, paramiko, sshtunnel, Typer, Rich, dotenv, httpx, pyyaml | | SSH | paramiko + sshtunnel (Windows без системного `ssh -L`) | -| Конфиг | `/.env` + runtime в `/.gpu-rent/` | +| Конфиг | `/.env` + `gpu-rent.vars` + runtime в `/.gpu-rent/` | | Лицензия | MIT | diff --git a/docs/cli.md b/docs/cli.md index f365497..ee1b2b0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,7 +8,9 @@ | --- | --- | | `gpu-rent flavors` | Скан `SCAN_POOLS` (ru-6 multizone…) × `FLAVOR_PREFERENCE`, затем список в текущем `OS_REGION_NAME` | | `gpu-rent doctor` | Preflight **без** create: Keystone, квота GPU, flavor в AZ, диски, Civitai token+`.red`, манифесты, SSH-ключ. Код выхода ≠ 0, если сессию нельзя начать | -| `gpu-rent up` / `up --yes` | Preflight → create/unshelve → bootstrap → git update SwarmUI/extensions → **туннель** `localhost:17801`; Ctrl+C закрывает туннель | +| `gpu-rent setup` | Wizard: `.env`/манифесты, `LLM_RUNTIME`, пресет ollama-models, опционально local-watchdog | +| `gpu-rent up` / `up --yes` | Preflight → create/unshelve → bootstrap → git update → optional LLM → **туннель** `localhost:17801`; Ctrl+C закрывает туннель | +| `gpu-rent up --ollama` / `--llamacpp` / `--llm …` | Поднять LLM рядом со SwarmUI (см. [llm.md](llm.md)) | | `gpu-rent up --no-update` | Без `git pull` SwarmUI и extensions (только недостающие clone) | | `gpu-rent up --no-tunnel` | Только облако + bootstrap, без локального проброса | | `gpu-rent up --no-spot` | Обычный (не preemptible) сервер | @@ -32,13 +34,29 @@ | `gpu-rent pull-output` | Забрать новые файлы с VM `Output/` в `./Output` (сервер не чистим) | | `gpu-rent seed-extensions` | Доклонировать/обновить git-репы; если VM жива — `systemctl restart swarmui` | | `gpu-rent resize-data --gb 400` | Увеличить data volume вверх (Selectel online resize). Вниз нельзя | +| `gpu-rent watchdog install` | Локальный тик (Task Scheduler / systemd user / launchd): аварийное закрытие туннеля → `stop` после grace | +| `gpu-rent watchdog uninstall` | Снять локальный сервис | +| `gpu-rent watchdog status` | Установлен ли сервис + local lease | +| `gpu-rent watchdog tick` | Один тик (для планировщика; `--dry-run` без delete) | | `gpu-rent dry-run` | План без mutating-вызовов | Второй `up` при живой VM: не создавать второй GPU; сделать autocomplete-check и **push** локальных папок; если `PULL_OUTPUT` — подтянуть Output; напомнить про `tunnel` / `stop`. `--adopt` если нашли тег без state. Нет команды `generate`. Нет зеркала каталога локального SwarmUI — только папки приложения, см. [local-folders.md](local-folders.md). -Сейчас в коде: после seed/`start swarmui` — idle-killer, Idle backend, boot snapshot, toast/MCP-сниппет, `resize-data`. `tunnel` с watchdog: EXPIRED → unshelve + reconnect FIP/SSH. UX-полировка flavors/цен ещё впереди. +Сейчас в коде: после seed/`start swarmui` — idle-killer, Idle backend, boot snapshot, toast/MCP-сниппет, `resize-data`. `tunnel` с Nova-watchdog: EXPIRED → unshelve + reconnect. Опционально `watchdog install` — локальный safety net. UX-полировка flavors/цен ещё впереди. + +## Local watchdog + +Опционально. **Idle-killer на VM** остаётся основным: ноут можно закрыть, GPU живёт до простоя. Local watchdog — если хочешь гасить GPU при «убили окно / ребут» без `stop`. + +1. `gpu-rent watchdog install` (раз на машине, в корне репо). +2. Пока крутится `up`/`tunnel`, пишется heartbeat в `.gpu-rent/local-lease.json`. +3. **Ctrl+C** → detach, GPU **не** трогаем (как раньше). +4. Процесс умер / ребут → через `LOCAL_WATCHDOG_GRACE_MINUTES` (дефолт 10) тик вызывает `stop` (диски остаются). +5. `gpu-rent stop` чистит lease сам. + +Без `install` поведение прежнее. Без открытого туннеля (`up --no-tunnel`) lease не вооружается — работает только VM idle-killer. ## `doctor` diff --git a/docs/concept.md b/docs/concept.md index db8fc64..424ba5b 100644 --- a/docs/concept.md +++ b/docs/concept.md @@ -9,7 +9,7 @@ GPU в облаке дорогой. Веса для генерации карт 1. Арендовать **прерываемый** облачный сервер Selectel с GPU (~70% дешевле обычного, без SLA). 2. Хранить ОС, бинарники SwarmUI, модели, ComfyUI и результаты на **сетевых дисках**. Они переживают удаление и остановку VM. 3. Локальный CLI создаёт (или восстанавливает) сервер и отдельно, когда нужно, пробрасывает SwarmUI на `localhost:17801`. -4. GPU гасится командой `stop` **или** idle-killer на самой VM (пустая очередь). Диски остаются. Ноут можно закрыть — compute от этого не умирает. +4. GPU гасится командой `stop`, idle-killer на VM, или (опционально) local-watchdog при unclean exit. Диски остаются. Без watchdog ноут можно закрыть — compute не умирает. Пользователь открывает браузер на `http://127.0.0.1:17801`, переключает Cursor MCP на этот URL или бьёт в HTTP API SwarmUI. Локальный SwarmUI на `7801` не трогаем. diff --git a/docs/decisions.md b/docs/decisions.md index 3bccc13..446a3fe 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -5,7 +5,7 @@ | Тема | Решение | | --- | --- | | Аудитория | Личный инструмент, репозиторий можно показать другим: MIT, без аккаунтных id и без имён локальных чекпоинтов | -| Сессия | GPU живёт до `stop` **или** простоя (idle-killer **на VM**). Ноут можно закрыть — compute не обязан умереть | +| Сессия | GPU живёт до `stop` **или** простоя (idle-killer **на VM**). Ноут можно закрыть — compute не обязан умереть. Опционально: `gpu-rent watchdog install` — локальный тик на Windows/macOS/Linux; если туннель умер без Ctrl+C/`stop`, после `LOCAL_WATCHDOG_GRACE_MINUTES` (дефолт 10) → `stop` | | Модели | Пустой `./Models` → не грузим ничего. Появились веса + метадата → на `up` выгрузить эту пару. Civitai-seed отдельно. С сервера не удаляем | | Civitai хост | Дефолт API `civitai.red` (полный каталог). `.com` — SFW-витрина, NSFW с неё часто 404. Ссылки `.com`/`.red`/`.green` в манифесте принимаем. 404 → один retry на второй хост. Один токен на оба домена | | Пул GPU | Перед `up`/`flavors` сканируем `SCAN_POOLS` (дефолт `ru-6,ru-7`). `ru-6` — мультизональный: ходим на `https://ru-6.cloud.api.selcloud.ru/compute/` тем же токеном (SDK-каталог часто знает только RC-пул). Собираем типы GPU из extra_specs и совпадения с `FLAVOR_PREFERENCE`. Автоматом `.env` не пишем — печатаем рекомендацию `OS_REGION_NAME` / `GPU_RENT_AZ` | @@ -21,6 +21,7 @@ | Льгота после boot | Killer молчит во время clone/seed/push, hold, качалки в UI SwarmUI, пока backend не Idle, и 45 мин после ACTIVE/unshelve | | `up` / `tunnel` | `up` по умолчанию после ready открывает туннель `:17801`, печатает URL и ждёт. `--no-tunnel` — только облако. Ctrl+C на туннеле GPU не гасит (`stop` отдельно). Команда `tunnel` остаётся для повторного входа | | Git update | На каждом `up` по умолчанию: `git pull` SwarmUI + репы из `extensions.yaml` + уже установленные на data (`Extensions`/`DLNodes`). `--no-update` или `UPDATE_GIT=false` — не тянуть | +| LLM (opt-in) | `none` по умолчанию. `ollama` / `llamacpp` — флаг `--ollama`/`--llamacpp`/`--llm`, `LLM_RUNTIME` в vars, или вопрос в interactive `up`/`setup`. Ollama-модели из `ollama-models.yaml`. Порты: Ollama 17811, llama.cpp 17812. Назначение: помощь с промптами, не замена SwarmUI | | Data-диск | Старт **100 GB**, рост через resize вверх (вниз Selectel не умеет) | | SSH | CLI генерирует `/.gpu-rent/id_ed25519` без passphrase и сам регистрирует keypair | | Локальные файлы | Всё в корне репозитория: `.env`, `models.yaml`, `extensions.yaml`; runtime (`state.json`, lock, SSH) в `/.gpu-rent/`. Не `%USERPROFILE%\.gpu-rent` | @@ -41,6 +42,6 @@ 1. **Туннель ≠ жизнь GPU.** `stop` и idle-killer не зависят от того, открыт ли SSH с ноутбука. 2. **Локальный SwarmUI на 7801 не трогаем.** Туннель по умолчанию на **17801** (на VM по-прежнему 7801 на loopback). -3. **`Ctrl+C` на туннеле не удаляет VM.** Иначе «закрыл ноут» невозможно. Чтобы убить GPU — `gpu-rent stop` или простой. +3. **`Ctrl+C` на туннеле не удаляет VM.** Иначе «закрыл ноут» невозможно. Чтобы убить GPU — `gpu-rent stop` или простой. При установленном **local-watchdog** Ctrl+C по-прежнему detach; убийство процесса/ребут без detach → stop после grace. 4. Idle-killer на VM **не может** быть `shutdown -h`: у Selectel остановленная изнутри VM часто продолжает тарифицировать ресурсы. Нужен вызов OpenStack: удалить **этот** compute, диски оставить. 5. Для этого на VM — OpenStack **application credential** с правом удалить/shelve сервер в проекте (не пароль владельца аккаунта). Компрометация SwarmUI в худшем случае сносит GPU-сессию, а не создаёт новые дорогие машины, если роль без `compute:create`. diff --git a/docs/llm.md b/docs/llm.md new file mode 100644 index 0000000..52a3906 --- /dev/null +++ b/docs/llm.md @@ -0,0 +1,59 @@ +# LLM рядом со SwarmUI (opt-in) + +По умолчанию поднимается только SwarmUI. Ollama или llama.cpp — по флагу, `LLM_RUNTIME` в `gpu-rent.vars` / `.env`, или через wizard. + +## Включение + +```text +gpu-rent setup # спросит none/ollama/llamacpp + пресет моделей +gpu-rent up --ollama # разово +gpu-rent up --llm llamacpp +# или в gpu-rent.vars: +LLM_RUNTIME=ollama +``` + +Без параметров `gpu-rent` / `gpu-rent up` (без `--yes`) спросит про LLM, если в vars ещё `none`. + +## Порты (только loopback + туннель) + +| Сервис | VM | localhost | +| --- | --- | --- | +| SwarmUI | 7801 | 17801 | +| Ollama | 11434 | 17811 | +| llama.cpp | 8080 | 17812 | + +```text +gpu-rent tunnel +gpu-rent open --llm # http://127.0.0.1:17811 (Ollama) +# клиент: +set OLLAMA_HOST=http://127.0.0.1:17811 +``` + +## Ollama models + +Как Civitai `models.yaml`: + +- `ollama-models.example.yaml` — в git +- `ollama-models.yaml` — локальный (gitignore) + +На `up` при `LLM_RUNTIME=ollama` CLI делает `ollama pull` по списку. Уже скачанные не трогает; лишние на диске не удаляет. + +### Пресеты setup + +| preset | tag | зачем | +| --- | --- | --- | +| **recommended** | `huihui_ai/qwen2.5-abliterate:7b` | RU/EN, ~5GB, мало отказов — помощь с промптами | +| light | `qwen2.5:3b` | быстрее, слабее | +| stock | `qwen2.5:7b` | официальный, больше цензуры | +| alt | `richardyoung/qwen2.5-7b-instruct-abliterated` | другой abliterate | +| empty | `[]` | только runtime | + +Community abliterate-модели без гарантий безопасности — для личного prompt-help / NSFW-тегов рядом со SwarmUI. + +## llama.cpp + +Ставит `llama-server` и systemd. GGUF клади вручную в `/mnt/swarm_data/llamacpp/models` на data-диске (или через SSH), затем `systemctl restart gpu-rent-llamacpp`. + +## Idle-killer + +Busy также если идёт `ollama pull`, в Ollama есть loaded model, или llama.cpp занимает слоты. diff --git a/docs/open-questions.md b/docs/open-questions.md index 3a8b1b7..84b28ee 100644 --- a/docs/open-questions.md +++ b/docs/open-questions.md @@ -7,7 +7,7 @@ - Качалка модели в UI: отдельного poll-API нет (`DoModelDownloadWS` только WS) → v1 считает busy через `waiting_gens` / `live_gens` / `loading_models` / backend≠idle; иначе пользователь жмёт `hold`. - Реальная цена 100 GB сетевого диска в выбранном сегменте и цена 1×4090 preemptible ₽/час (в OpenStack API нет). - Имя GPU-образа (без Docker) и flavor id в твоём пуле (в git не класть); какие из списка фоллбека реально есть. -- Хватает ли application credential с правами сервисного `member`, или Selectel отдаёт более узкую роль «только delete». +- Хватает ли application credential с access_rules (delete/GET server) на Selectel — при отказе CLI падает назад на unrestricted cred и пишет в лог. - Snapshot attached boot volume после Idle: время и можно ли сразу create from snapshot. Если spike покажет, что 45 минут льготы мало на первую установку ComfyUI — поднять `IDLE_GRACE_MINUTES`, не отключать killer. diff --git a/docs/reviews/2026-08-21-review-1.md b/docs/reviews/2026-08-21-review-1.md new file mode 100644 index 0000000..67cf61d --- /dev/null +++ b/docs/reviews/2026-08-21-review-1.md @@ -0,0 +1,111 @@ +# Project Review — 2026-08-21 (1) + +## Prior Reviews Summary + +> Based on the last 3 review files analysed in Phase 0. + +### Still Open (carried forward) +None. + +### Resolved Since Last Review +None. + +--- + +## Phase 1: Code Quality + +### SOLID +- `session.cmd_up` / `_bind_access` orchestrate cloud, SSH, bootstrap, provision, snapshot, notify — god-flow (`src/gpu_rent/session.py`). +- Docs name separate modules (`civitai_seed`, `models_push`, `git_seed`, `watchdog`, `reconcile`) that do not exist as packages; logic is folded into `provision.py`, `sync_files.py`, `tunnel.py`, `cli.status`. +- `Config` dataclass is a wide bag of unrelated settings (auth, paths, idle, autocomplete, civitai). + +### Performance +- `push_tree` / `pull_tree`: per-file `remote_sha256` + `put_file` each open a new SSH (`sync_files.py`, `ssh_ops.py`). +- Every `up` re-runs full `bootstrap.sh` (apt-get) via `_bind_access` even when already bootstrapped. +- `scan_pools` on every `up`: extra Keystone authorize + per-flavor HTTP extra_specs (ThreadPool 16). +- Tunnel watchdog: `connect()` / Keystone authorize every ~30s. + +### Correctness & Bugs +- `wait_ssh` aborts after 3 `AuthenticationException`s (~15s) while cloud-init may still be injecting keys (`ssh_ops.py` ~114–120). +- `cmd_stop` leaves `bootstrapped=True` (`session.py` ~397–400) → next ACTIVE path mis-labeled / wrong branch when FIP cleared. +- `phase` set to `ready_cloud` before bootstrap finishes (`session.py` ~344–346). +- Application credential created without role/access_rules restriction (`idle_killer.py` ~52–57) vs architecture “delete/shelve only”. +- `GIT_TOKEN` embedded in git remote URL and never stripped (`clone_ext.py` `with_token` / clone). +- SG rules accumulate old CIDRs; never prune (`cloud.py` `ensure_security_group`). +- Session tests out of sync: no `bootstrapped`+FIP setup for “second GPU”; `run_bootstrap` mock missing `update=` (`tests/test_session.py`). + +### Code Quality +- State machine phases in docs (`bootstrapping`, `seeding_*`, `waiting_ui`) never written by code (only `idle` / `provisioning` / `ready_*`). +- `os_client.connect` reports `app_version="0.1.0"` while package is `0.2.0`. +- Duplicated SwarmUI busy/Idle polling logic in `ready.py` and `remote/idle_killer.py`. + +--- + +## Phase 2: Logical Consistency + +### Domain & Application Layer +- Architecture table invents module names; actual layout is flatter (`cli` → `session`/`doctor`/`tunnel` → `cloud`/`provision`/`ssh_ops`). + +### Data Flow +- Config sources: `.env` (dotenv override=False) then `gpu-rent.vars` (override empty only) — OK. +- Volume IDs: `state` preferred over `BOOT_VOLUME_ID` / `DATA_VOLUME_ID` env — OK; AZ not re-validated on reuse. + +### State Management +- `bootstrapped` not cleared on `stop`. +- `bootstrapped` True only at end of `_bind_access`; failure mid-bind leaves `ready_cloud` + `bootstrapped=False`. +- Marker files on VM (`/opt/swarmui/.gpu-rent-bootstrapped`) independent of local `state.bootstrapped`. + +### Consistency +- Error handling: mostly `GpuRentError` / `CloudError`; some soft-log (`wait_backend_idle`, snapshot, idle-killer arm). +- Civitai bad token is **blocking** in doctor → blocks `up` even if user only needs default SwarmUI models. + +--- + +## Phase 3: UI/UX (CLI) + +### Usability +- `up` always prints full doctor table (noisy). +- Invalid/expired `CIVITAI_API_TOKEN` blocks entire session start. +- `destroy` has no `--no-pull` (always respects `PULL_OUTPUT` via `cmd_stop`). + +### Visual & Consistency +- N/A (CLI). Mixed RU/EN messages by design. + +### Interaction & Feedback +- Confirm on `up` without `--yes` is good; `--yes` skips. +- Tunnel Ctrl+C messaging is clear. +- `logs` only tails cloud-init + `is-active`, not `journalctl -u swarmui` as docs imply. + +### Accessibility +- N/A. + +--- + +## Full exploration notes (sections 1–10) + +See chat report for narrative. Tasks below are actionable. + +--- + +## Tasks + +- [x] 1. [Bug] Soften `wait_ssh` early auth abort (retry until timeout / distinguish “sshd up, keys not ready”) — `src/gpu_rent/ssh_ops.py` line 114 +- [x] 2. [Bug] Clear `bootstrapped=False` (and align phase) in `cmd_stop` — `src/gpu_rent/session.py` line 397 +- [x] 3. [Security] Restrict idle-killer application credential (access_rules / roles delete-only) — `src/gpu_rent/idle_killer.py` line 52 +- [x] 4. [Security] Strip `GIT_TOKEN` from git `origin` after clone/fetch — `src/gpu_rent/remote/clone_ext.py` line 30 +- [x] 5. [Security] Stop accumulating stale SSH SG CIDRs; replace or prune old /32 — `src/gpu_rent/cloud.py` line 154 +- [x] 6. [Security] Revisit default `GPU_RENT_SSH_CIDR=0.0.0.0/0` in `env.example` — `env.example` line 16 +- [x] 7. [Performance] Reuse one SSH/SFTP session in `push_tree` / `pull_tree` / hash checks — `src/gpu_rent/sync_files.py` line 15 +- [x] 8. [Performance] Skip full bootstrap (or apt) when VM already marked bootstrapped — `src/gpu_rent/session.py` line 87 +- [x] 9. [Logic] Do not set `phase=ready_cloud` before `_bind_access` succeeds — `src/gpu_rent/session.py` line 344 +- [x] 10. [Logic] Make doctor Civitai failure non-blocking when seed is optional — `src/gpu_rent/doctor.py` line 271 +- [x] 11. [CodeQuality] Fix `tests/test_session.py` mocks (`bootstrapped`+FIP; `run_bootstrap(..., update=)`) — `tests/test_session.py` line 40 +- [x] 12. [UX] `logs` should include `journalctl -u swarmui` as docs claim — `src/gpu_rent/cli.py` line 404 +- [x] 13. [Logic] Update `docs/architecture.md` module names, phases, questionary, hold `--minutes` wording — `docs/architecture.md` line 37 + +## Closure notes (2026-08-21) + +- `wait_ssh`: auth streak give-up after 180s continuous AuthenticationException (not 3 attempts). +- Idle-killer: access_rules for GET/DELETE server; fallback without rules if Keystone rejects. +- Bootstrap light path: `GPU_RENT_BOOTSTRAP_LIGHT=1` skips apt when marker present. +- Also fixed `os_client` `app_version` → package `__version__` (noted in Phase 1, not a numbered task). diff --git a/env.example b/env.example index 214ddab..1818935 100644 --- a/env.example +++ b/env.example @@ -12,8 +12,9 @@ GPU_RENT_AZ=ru-7a SSH_PRIVATE_KEY_PATH= SSH_USER=ubuntu -# Spike / WARP: open SSH. Tighten to your /32 later. -GPU_RENT_SSH_CIDR=0.0.0.0/0 +# Prefer auto public /32 (leave unset). Open to world only for spike / WARP: +# GPU_RENT_SSH_CIDR=0.0.0.0/0 +GPU_RENT_SSH_CIDR= BOOT_VOLUME_ID= DATA_VOLUME_ID= @@ -37,6 +38,10 @@ AUTOCOMPLETE_GITHUB_REF=main AUTOCOMPLETE_FILENAME=danbooru.csv SWARMUI_LOCAL_PORT=17801 +# Optional LLM beside SwarmUI: none | ollama | llamacpp (or gpu-rent setup / --ollama) +LLM_RUNTIME=none +OLLAMA_LOCAL_PORT=17811 +LLAMACPP_LOCAL_PORT=17812 # git pull SwarmUI + extensions on each up (default true). CLI: --no-update UPDATE_GIT=true @@ -49,5 +54,7 @@ DEFAULT_SPOT=true KEEP_FLOATING_IP=false IDLE_MINUTES=30 IDLE_GRACE_MINUTES=45 +# Optional local safety net (after: gpu-rent watchdog install) +# LOCAL_WATCHDOG_GRACE_MINUTES=10 PULL_OUTPUT=false NOTIFY_READY=true diff --git a/gpu-rent.ps1 b/gpu-rent.ps1 index 6f6d87e..0997229 100644 --- a/gpu-rent.ps1 +++ b/gpu-rent.ps1 @@ -118,6 +118,7 @@ function Copy-IfMissing { } Copy-IfMissing (Join-Path $Root "models.example.yaml") (Join-Path $Root "models.yaml") "models.yaml" Copy-IfMissing (Join-Path $Root "extensions.example.yaml") (Join-Path $Root "extensions.yaml") "extensions.yaml" +Copy-IfMissing (Join-Path $Root "ollama-models.example.yaml") (Join-Path $Root "ollama-models.yaml") "ollama-models.yaml" Copy-IfMissing (Join-Path $Root "gpu-rent.vars.example") (Join-Path $Root "gpu-rent.vars") "gpu-rent.vars" Import-GpuRentVars (Join-Path $Root "gpu-rent.vars") diff --git a/gpu-rent.sh b/gpu-rent.sh index a3aceb9..0f9915f 100644 --- a/gpu-rent.sh +++ b/gpu-rent.sh @@ -95,6 +95,10 @@ if [[ ! -f "$ROOT/extensions.yaml" && -f "$ROOT/extensions.example.yaml" ]]; the cp "$ROOT/extensions.example.yaml" "$ROOT/extensions.yaml" echo "gpu-rent: created extensions.yaml" fi +if [[ ! -f "$ROOT/ollama-models.yaml" && -f "$ROOT/ollama-models.example.yaml" ]]; then + cp "$ROOT/ollama-models.example.yaml" "$ROOT/ollama-models.yaml" + echo "gpu-rent: created ollama-models.yaml" +fi if [[ ! -f "$ROOT/gpu-rent.vars" && -f "$ROOT/gpu-rent.vars.example" ]]; then cp "$ROOT/gpu-rent.vars.example" "$ROOT/gpu-rent.vars" echo "gpu-rent: created gpu-rent.vars" diff --git a/gpu-rent.vars.example b/gpu-rent.vars.example index 3fddc39..3c56ce1 100644 --- a/gpu-rent.vars.example +++ b/gpu-rent.vars.example @@ -15,8 +15,13 @@ # SCAN_POOLS=ru-6,ru-7 # FLAVOR_PREFERENCE=4090-24,4090-48,a5000,a100-40 # SWARMUI_LOCAL_PORT=17801 +# LLM_RUNTIME=none +# OLLAMA_LOCAL_PORT=17811 +# LLAMACPP_LOCAL_PORT=17812 # IDLE_MINUTES=30 # IDLE_GRACE_MINUTES=45 +# LOCAL_WATCHDOG_GRACE_MINUTES=10 # NOTIFY_READY=true # PULL_OUTPUT=false +# Prefer auto /32 (leave unset). Spike/WARP only: # GPU_RENT_SSH_CIDR=0.0.0.0/0 diff --git a/ollama-models.example.yaml b/ollama-models.example.yaml new file mode 100644 index 0000000..5c30afc --- /dev/null +++ b/ollama-models.example.yaml @@ -0,0 +1,17 @@ +# Copy to ollama-models.yaml (gitignored). Used when LLM_RUNTIME=ollama. +# name = exact tag for `ollama pull`. Empty models: [] → runtime only, no pull. +# Purpose: help craft SwarmUI prompts (RU/EN, low refusal). + +models: + # Recommended: Russian+English, ~5GB, low refusal (community abliterate). + - name: huihui_ai/qwen2.5-abliterate:7b + default: true + + # Lighter / faster (weaker prompts): + # - name: qwen2.5:3b + + # Official stock (more refusals): + # - name: qwen2.5:7b + + # Alternate abliterate pack: + # - name: richardyoung/qwen2.5-7b-instruct-abliterated diff --git a/src/gpu_rent/bootstrap.py b/src/gpu_rent/bootstrap.py index fdd86f4..5d22310 100644 --- a/src/gpu_rent/bootstrap.py +++ b/src/gpu_rent/bootstrap.py @@ -16,8 +16,18 @@ def bootstrap_script() -> str: return files("gpu_rent.remote").joinpath("bootstrap.sh").read_text(encoding="utf-8") -def run_bootstrap(cfg: Config, host: str, log: Log, *, update: bool = True) -> None: - log("bootstrap SwarmUI на VM (идемпотентно, без Docker)") +def run_bootstrap( + cfg: Config, + host: str, + log: Log, + *, + update: bool = True, + light: bool = False, +) -> None: + if light: + log("bootstrap SwarmUI (light: без apt)") + else: + log("bootstrap SwarmUI на VM (идемпотентно, без Docker)") if update: log("git update: SwarmUI on") else: @@ -32,6 +42,7 @@ def run_bootstrap(cfg: Config, host: str, log: Log, *, update: bool = True) -> N env={ "SWARM_USER": cfg.ssh_user, "GPU_RENT_UPDATE_GIT": "1" if update else "0", + "GPU_RENT_BOOTSTRAP_LIGHT": "1" if light else "0", }, log=log, ) diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index 1fd2f50..3fbaa1c 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -33,22 +33,26 @@ if sys.platform == "win32": pass app = typer.Typer( - no_args_is_help=True, + invoke_without_command=True, pretty_exceptions_enable=False, add_completion=False, - help="Прерываемый GPU Selectel + SwarmUI на localhost:17801. Сначала: gpu-rent doctor. Ключи: docs/setup.md", + help="Прерываемый GPU Selectel + SwarmUI на localhost:17801. Сначала: gpu-rent setup / doctor. Ключи: docs/setup.md", ) console = Console(highlight=False, legacy_windows=False) _DEBUG = False -@app.callback() +@app.callback(invoke_without_command=True) def _root( + ctx: typer.Context, debug: bool = typer.Option(False, "--debug", help="Показать traceback"), ) -> None: global _DEBUG _DEBUG = debug + if ctx.invoked_subcommand is None: + # Без подкоманды → interactive up (как «запуск без параметров»). + ctx.invoke(up) def _die(exc: BaseException) -> None: @@ -249,6 +253,20 @@ def status() -> None: table.add_row("диск used/free", "нужен живой FIP + SSH-ключ") table.add_row("idle-killer", "нужен SSH на живую VM") + from gpu_rent.local_watchdog import watchdog_status_lines + + table.add_row("local-watchdog", "; ".join(watchdog_status_lines())) + from gpu_rent.llm_runtime import normalize_runtime + + rt = normalize_runtime(cfg.llm_runtime) + noted = (state.notes or {}).get("llm_runtime") + if noted: + rt = f"{rt} (сессия: {noted})" + table.add_row( + "LLM", + f"{rt}; ollama :{cfg.ollama_local_port} / llamacpp :{cfg.llamacpp_local_port}", + ) + if cfg.auth_ok: try: conn = connect(cfg) @@ -271,10 +289,30 @@ def status() -> None: @app.command() -def open() -> None: - """Открыть браузер на http://127.0.0.1:17801. Туннель уже должен слушать порт.""" +def open( + llm: bool = typer.Option(False, "--llm", help="Открыть LLM API URL вместо SwarmUI"), +) -> None: + """Открыть браузер на SwarmUI :17801 (или --llm на Ollama/llama.cpp).""" cfg = load_config(require_auth=False) - port = cfg.swarmui_local_port + if llm: + from gpu_rent.llm_runtime import normalize_runtime + + runtime = normalize_runtime(cfg.llm_runtime) + state = load_state() + if (state.notes or {}).get("llm_runtime"): + try: + runtime = normalize_runtime(str(state.notes["llm_runtime"])) + except ValueError: + pass + if runtime == "ollama": + port = cfg.ollama_local_port + elif runtime == "llamacpp": + port = cfg.llamacpp_local_port + else: + console.print("[red]LLM не выбран[/red] (LLM_RUNTIME / gpu-rent setup)") + raise typer.Exit(1) + else: + port = cfg.swarmui_local_port if not _port_open(port): console.print( f"[red]localhost:{port} молчит.[/red] Сначала `gpu-rent tunnel`, потом open." @@ -285,6 +323,43 @@ def open() -> None: console.print(url) +@app.command() +def setup( + llm: Optional[str] = typer.Option(None, "--llm", help="none|ollama|llamacpp"), + ollama_preset: Optional[str] = typer.Option( + None, "--ollama-preset", help="recommended|light|stock|alt|empty" + ), + watchdog: Optional[bool] = typer.Option( + None, "--watchdog/--no-watchdog", help="Поставить local-watchdog" + ), + yes: bool = typer.Option(False, "--yes", help="Без вопросов (дефолты)"), +) -> None: + """Интерактивная установка: файлы конфига, LLM, опционально watchdog.""" + try: + from gpu_rent.setup_wizard import run_setup + + def confirm(msg: str) -> bool: + if yes: + return False if watchdog is False else bool(watchdog) + return typer.confirm(msg) + + def ask(msg: str, default: str) -> str: + if yes: + return default + return typer.prompt(msg, default=default) + + run_setup( + llm=llm if llm is not None else ("none" if yes else None), + ollama_preset=ollama_preset if ollama_preset is not None else ("recommended" if yes else None), + install_watchdog=watchdog if watchdog is not None else (False if yes else None), + confirm=None if yes and watchdog is None else confirm, + ask=None if yes and llm is not None else ask, + log=lambda m: console.print(m), + ) + except Exception as exc: + _die(exc) + + @app.command() def up( no_spot: bool = typer.Option(False, "--no-spot", help="Обычный сервер, не preemptible"), @@ -306,15 +381,67 @@ def up( "--no-update", help="Не делать git pull SwarmUI и установленных extensions", ), + llm: Optional[str] = typer.Option( + None, "--llm", help="none|ollama|llamacpp (override LLM_RUNTIME)" + ), + ollama: bool = typer.Option(False, "--ollama", help="То же что --llm ollama"), + llamacpp: bool = typer.Option(False, "--llamacpp", help="То же что --llm llamacpp"), ) -> None: """Create/unshelve GPU, bootstrap SwarmUI, по умолчанию туннель на :17801.""" try: + from dataclasses import replace + + from gpu_rent.llm_runtime import ( + PRESET_HELP, + append_vars_llm_runtime, + decide_runtime, + ensure_ollama_manifest_from_example, + write_ollama_models_preset, + ) + from gpu_rent.paths import vars_path + checks = run_doctor() code = _print_checks(checks) if code != 0: raise typer.Exit(1) cfg = load_config(require_auth=True) + try: + runtime = decide_runtime( + flag=llm, + ollama_flag=ollama, + llamacpp_flag=llamacpp, + from_config=cfg.llm_runtime, + ) + except ValueError as exc: + raise GpuRentError(str(exc)) from exc + + if not yes and runtime == "none" and not llm and not ollama and not llamacpp: + choice = typer.prompt( + "Поднять LLM рядом со SwarmUI? [none/ollama/llamacpp]", + default="none", + ) + try: + runtime = decide_runtime( + flag=choice, ollama_flag=False, llamacpp_flag=False, from_config="none" + ) + except ValueError as exc: + raise GpuRentError(str(exc)) from exc + if runtime != "none" and typer.confirm("Запомнить LLM_RUNTIME в gpu-rent.vars?", default=True): + append_vars_llm_runtime(vars_path(), runtime) + if runtime == "ollama": + ensure_ollama_manifest_from_example() + console.print(PRESET_HELP) + preset = typer.prompt("Ollama preset", default="recommended") + if preset.strip().lower() not in {"keep", "example"}: + write_ollama_models_preset( + cfg.ollama_models_manifest, preset.strip().lower() + ) + + cfg = replace(cfg, llm_runtime=runtime) + if runtime != "none": + console.print(f"LLM runtime: {runtime}") + def confirm(msg: str) -> bool: return typer.confirm(msg) @@ -368,6 +495,7 @@ def stop( @app.command() def destroy( i_understand_data_loss: bool = typer.Option(False, "--i-understand-data-loss"), + no_pull: bool = typer.Option(False, "--no-pull"), ) -> None: """stop + диски.""" if not i_understand_data_loss: @@ -375,7 +503,7 @@ def destroy( raise typer.Exit(1) try: cfg = load_config(require_auth=True) - cmd_stop(cfg, destroy_disks=True, log=lambda m: console.print(m)) + cmd_stop(cfg, destroy_disks=True, no_pull=no_pull, log=lambda m: console.print(m)) except GpuRentError as exc: _die(exc) @@ -395,7 +523,7 @@ def ssh() -> None: @app.command() def logs() -> None: - """cloud-init / journalctl на VM.""" + """cloud-init / journalctl -u swarmui на VM.""" try: cfg = load_config(require_auth=True) state = load_state() @@ -404,9 +532,14 @@ def logs() -> None: out = run_ssh( cfg, state.floating_ip, - "sudo -n tail -n 80 /var/log/cloud-init-output.log 2>/dev/null; " - "systemctl is-active swarmui 2>/dev/null || true", + "echo '=== cloud-init (tail) ==='; " + "sudo -n tail -n 60 /var/log/cloud-init-output.log 2>/dev/null || true; " + "echo; echo '=== systemctl swarmui ==='; " + "systemctl is-active swarmui 2>/dev/null || true; " + "echo; echo '=== journalctl -u swarmui ==='; " + "sudo -n journalctl -u swarmui -n 80 --no-pager 2>/dev/null || true", check=False, + timeout=60, ) console.print(out) except GpuRentError as exc: @@ -538,6 +671,86 @@ def resize_data(gb: int = typer.Option(..., "--gb", help="Новый разме _die(exc) +watchdog_app = typer.Typer( + help=( + "Локальный сервис: если туннель умер без Ctrl+C / stop — " + "через grace удалить compute. Не путать с idle-killer на VM." + ), + no_args_is_help=True, +) +app.add_typer(watchdog_app, name="watchdog") + + +def _apply_project_root(project: Optional[str]) -> None: + if not project: + return + import os + from pathlib import Path + + root = Path(project).expanduser().resolve() + os.environ["GPU_RENT_ROOT"] = str(root) + try: + os.chdir(root) + except OSError: + pass + + +@watchdog_app.command("install") +def watchdog_install( + interval: int = typer.Option(5, "--interval", help="Минуты между тиками"), +) -> None: + """Поставить Task Scheduler / systemd user / launchd.""" + try: + from gpu_rent.local_watchdog import install_watchdog + + install_watchdog(interval_minutes=interval, log=lambda m: console.print(m)) + except Exception as exc: + console.print(f"[red]install fail:[/red] {exc}") + raise typer.Exit(1) from exc + + +@watchdog_app.command("uninstall") +def watchdog_uninstall() -> None: + """Снять локальный watchdog.""" + try: + from gpu_rent.local_watchdog import uninstall_watchdog + + uninstall_watchdog(log=lambda m: console.print(m)) + except Exception as exc: + console.print(f"[red]uninstall fail:[/red] {exc}") + raise typer.Exit(1) from exc + + +@watchdog_app.command("status") +def watchdog_status_cmd() -> None: + """Состояние установки и local lease.""" + from gpu_rent.local_watchdog import watchdog_status_lines + + for line in watchdog_status_lines(): + console.print(line) + + +@watchdog_app.command("tick") +def watchdog_tick( + project: Optional[str] = typer.Option( + None, + "--project", + help="Корень репо (для планировщика; выставляет GPU_RENT_ROOT)", + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Только решение, без stop"), +) -> None: + """Один тик (вызывает планировщик).""" + _apply_project_root(project) + try: + from gpu_rent.local_watchdog import run_tick + + decision = run_tick(dry_run=dry_run, log=lambda m: console.print(m)) + if decision.kind == "stop" and not dry_run: + raise typer.Exit(0) + except GpuRentError as exc: + _die(exc) + + def _port_open(port: int) -> bool: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(0.4) diff --git a/src/gpu_rent/cloud.py b/src/gpu_rent/cloud.py index 7e97542..0cd0ae8 100644 --- a/src/gpu_rent/cloud.py +++ b/src/gpu_rent/cloud.py @@ -164,21 +164,35 @@ def ensure_security_group(conn, cidr: str, log: Callable[[str], None]) -> Any: except Exception as exc: raise _wrap(exc, "security group") from exc - # Ensure TCP/22 from operator CIDR (idempotent; old /32 from WARP may be stale). + # Ensure TCP/22 from operator CIDR; drop stale SSH /32s from old VPN/WARP IPs. have_cidr = False + stale_ssh: list[Any] = [] try: for rule in conn.network.security_group_rules(security_group_id=sg.id): if ( - getattr(rule, "direction", None) == "ingress" - and getattr(rule, "protocol", None) == "tcp" - and int(getattr(rule, "port_range_min", 0) or 0) == 22 - and int(getattr(rule, "port_range_max", 0) or 0) == 22 - and (getattr(rule, "remote_ip_prefix", None) or "") == cidr + getattr(rule, "direction", None) != "ingress" + or getattr(rule, "protocol", None) != "tcp" + or int(getattr(rule, "port_range_min", 0) or 0) != 22 + or int(getattr(rule, "port_range_max", 0) or 0) != 22 ): + continue + prefix = getattr(rule, "remote_ip_prefix", None) or "" + if prefix == cidr: have_cidr = True - break + elif prefix: + stale_ssh.append(rule) except Exception: have_cidr = False + stale_ssh = [] + + for rule in stale_ssh: + old = getattr(rule, "remote_ip_prefix", None) or "?" + try: + conn.network.delete_security_group_rule(rule.id) + log(f"SG {SG_NAME}: − устаревший TCP/22 с {old}") + except Exception as exc: + log(f"SG {SG_NAME}: не удалить stale {old}: {exc}") + if not have_cidr: try: conn.network.create_security_group_rule( diff --git a/src/gpu_rent/config.py b/src/gpu_rent/config.py index f181c25..dc5f22f 100644 --- a/src/gpu_rent/config.py +++ b/src/gpu_rent/config.py @@ -16,10 +16,12 @@ from gpu_rent.paths import ( extensions_manifest_path, migrate_legacy_if_needed, models_manifest_path, + ollama_models_manifest_path, runtime_dir, vars_path, ) from gpu_rent.varsfile import apply_vars_file +from gpu_rent.llm_runtime import normalize_runtime def _as_bool(value: str | None, default: bool) -> bool: @@ -80,6 +82,11 @@ class Config: swarmui_image: str update_git: bool + llm_runtime: str + ollama_models_manifest: Path + ollama_local_port: int + llamacpp_local_port: int + default_flavor_id: str flavor_preference: tuple[str, ...] flavor_fallback: bool @@ -146,6 +153,15 @@ def load_config(*, require_auth: bool = True) -> Config: (os.environ.get("EXTENSIONS_MANIFEST") or "").strip() or str(extensions_manifest_path()) ).expanduser() + ollama_manifest = Path( + (os.environ.get("OLLAMA_MODELS_MANIFEST") or "").strip() + or str(ollama_models_manifest_path()) + ).expanduser() + + try: + llm_runtime = normalize_runtime(os.environ.get("LLM_RUNTIME")) + except ValueError: + llm_runtime = "none" def _dir(env_name: str, folder: str) -> Path: raw = (os.environ.get(env_name) or "").strip() @@ -187,6 +203,10 @@ def load_config(*, require_auth: bool = True) -> Config: swarmui_local_port=_as_int(os.environ.get("SWARMUI_LOCAL_PORT"), 17801), swarmui_image=(os.environ.get("SWARMUI_IMAGE") or "").strip(), update_git=_as_bool(os.environ.get("UPDATE_GIT"), True), + llm_runtime=llm_runtime, + ollama_models_manifest=ollama_manifest, + ollama_local_port=_as_int(os.environ.get("OLLAMA_LOCAL_PORT"), 17811), + llamacpp_local_port=_as_int(os.environ.get("LLAMACPP_LOCAL_PORT"), 17812), default_flavor_id=(os.environ.get("DEFAULT_FLAVOR_ID") or "").strip(), flavor_preference=_csv( os.environ.get("FLAVOR_PREFERENCE"), diff --git a/src/gpu_rent/doctor.py b/src/gpu_rent/doctor.py index f3c9bd2..cd25ee2 100644 --- a/src/gpu_rent/doctor.py +++ b/src/gpu_rent/doctor.py @@ -271,9 +271,10 @@ def _civitai(cfg: Config, checks: list[Check]) -> None: checks.append( Check( "Civitai", - False, True, - f"{probe.host}: {probe.detail}; fallback {alt.host}: {alt.detail}", + False, + f"{probe.host}: {probe.detail}; fallback {alt.host}: {alt.detail}. " + "seed-models недоступен, дефолт SwarmUI ок — up не блокируем", ) ) diff --git a/src/gpu_rent/idle_killer.py b/src/gpu_rent/idle_killer.py index 8dcac89..d1ffb4b 100644 --- a/src/gpu_rent/idle_killer.py +++ b/src/gpu_rent/idle_killer.py @@ -54,12 +54,45 @@ def create_application_credential(conn, cfg: Config, server_id: str, log: Log) - name=name, secret=secret, description="gpu-rent idle-killer: delete this compute", + access_rules=[ + { + "service": "compute", + "method": "DELETE", + "path": f"/v2.1/servers/{server_id}", + }, + { + "service": "compute", + "method": "DELETE", + "path": "/v2.1/servers/*", + }, + # sdk may GET server before delete / confirm status + { + "service": "compute", + "method": "GET", + "path": f"/v2.1/servers/{server_id}", + }, + { + "service": "compute", + "method": "GET", + "path": "/v2.1/servers/*", + }, + ], ) except Exception as exc: - raise CloudError( - f"не создать application credential: {exc}. " - "Нужны права identity:application_credential_create на сервисного пользователя." - ) from exc + # Selectel / older Keystone may reject access_rules — fall back unrestricted delete. + log(f"app cred с access_rules не вышло ({exc}); пробуем без правил") + try: + ac = conn.identity.create_application_credential( + user=user_id, + name=name, + secret=secret, + description="gpu-rent idle-killer: delete this compute", + ) + except Exception as exc2: + raise CloudError( + f"не создать application credential: {exc2}. " + "Нужны права identity:application_credential_create на сервисного пользователя." + ) from exc2 ac_id = getattr(ac, "id", None) or (ac.get("id") if isinstance(ac, dict) else None) ac_secret = getattr(ac, "secret", None) or secret if not ac_id: diff --git a/src/gpu_rent/llm_runtime.py b/src/gpu_rent/llm_runtime.py new file mode 100644 index 0000000..5974fbb --- /dev/null +++ b/src/gpu_rent/llm_runtime.py @@ -0,0 +1,166 @@ +"""Optional LLM runtimes (Ollama / llama.cpp) beside SwarmUI.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from gpu_rent.paths import ollama_models_example_path, ollama_models_manifest_path + +VALID_RUNTIMES = frozenset({"none", "ollama", "llamacpp"}) + +# Presets for setup / interactive up (prompt help: RU + low refusal). +OLLAMA_PRESETS: dict[str, list[str]] = { + "recommended": ["huihui_ai/qwen2.5-abliterate:7b"], + "light": ["qwen2.5:3b"], + "stock": ["qwen2.5:7b"], + "alt": ["richardyoung/qwen2.5-7b-instruct-abliterated"], + "empty": [], +} + +PRESET_HELP = ( + "recommended — Qwen2.5 7B abliterate (RU/EN, мало отказов, ~5GB)\n" + "light — qwen2.5:3b (быстрее, слабее)\n" + "stock — официальный qwen2.5:7b (больше цензуры)\n" + "alt — другой abliterate-пак 7B\n" + "empty — только runtime, без pull" +) + + +@dataclass(frozen=True) +class OllamaModelEntry: + name: str + default: bool = False + + +def normalize_runtime(value: str | None) -> str: + raw = (value or "none").strip().lower().replace("-", "").replace("_", "") + if raw in {"", "none", "off", "no", "0"}: + return "none" + if raw in {"ollama"}: + return "ollama" + if raw in {"llamacpp", "llama", "llamacppserver"}: + return "llamacpp" + raise ValueError(f"неизвестный LLM_RUNTIME={value!r}; жду none|ollama|llamacpp") + + +def decide_runtime( + *, + flag: str | None, + ollama_flag: bool, + llamacpp_flag: bool, + from_config: str, +) -> str: + """CLI flags win over config/vars.""" + if ollama_flag and llamacpp_flag: + raise ValueError("укажи только --ollama или --llamacpp, не оба") + if ollama_flag: + return "ollama" + if llamacpp_flag: + return "llamacpp" + if flag is not None and str(flag).strip() != "": + return normalize_runtime(flag) + return normalize_runtime(from_config) + + +def parse_ollama_models(path: Path) -> list[OllamaModelEntry]: + if not path.is_file(): + return [] + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(raw, dict): + return [] + items = raw.get("models") + if items is None: + return [] + if not isinstance(items, list): + raise ValueError(f"{path}: models должен быть списком") + out: list[OllamaModelEntry] = [] + for item in items: + if isinstance(item, str): + name = item.strip() + if name: + out.append(OllamaModelEntry(name=name)) + continue + if not isinstance(item, dict): + continue + name = str(item.get("name") or "").strip() + if not name: + continue + out.append(OllamaModelEntry(name=name, default=bool(item.get("default")))) + return out + + +def write_ollama_models_preset(path: Path, preset: str) -> None: + key = (preset or "recommended").strip().lower() + if key not in OLLAMA_PRESETS: + raise ValueError(f"пресет {preset!r}; варианты: {', '.join(OLLAMA_PRESETS)}") + names = OLLAMA_PRESETS[key] + lines = [ + "# Локальный манифест Ollama (не коммить). Пример: ollama-models.example.yaml", + "# name = точный тег для `ollama pull`. Пустой models: [] — без pull.", + "models:", + ] + if not names: + lines.append(" []") + else: + for i, name in enumerate(names): + lines.append(f" - name: {name}") + if i == 0: + lines.append(" default: true") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def ensure_ollama_manifest_from_example() -> Path: + dest = ollama_models_manifest_path() + if dest.is_file(): + return dest + example = ollama_models_example_path() + if example.is_file(): + dest.write_text(example.read_text(encoding="utf-8"), encoding="utf-8") + else: + write_ollama_models_preset(dest, "recommended") + return dest + + +def llm_local_port(cfg: Any) -> int | None: + runtime = normalize_runtime(getattr(cfg, "llm_runtime", "none")) + if runtime == "ollama": + return int(getattr(cfg, "ollama_local_port", 17811)) + if runtime == "llamacpp": + return int(getattr(cfg, "llamacpp_local_port", 17812)) + return None + + +def llm_remote_port(runtime: str) -> int | None: + runtime = normalize_runtime(runtime) + if runtime == "ollama": + return 11434 + if runtime == "llamacpp": + return 8080 + return None + + +def append_vars_llm_runtime(vars_file: Path, runtime: str) -> None: + runtime = normalize_runtime(runtime) + line = f"LLM_RUNTIME={runtime}\n" + if not vars_file.is_file(): + vars_file.write_text("# gpu-rent.vars — несекреты\n" + line, encoding="utf-8") + return + text = vars_file.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + out: list[str] = [] + replaced = False + for row in lines: + if row.lstrip().startswith("LLM_RUNTIME="): + out.append(line if row.endswith("\n") else line.rstrip("\n")) + replaced = True + else: + out.append(row) + if not replaced: + if out and not out[-1].endswith("\n"): + out[-1] = out[-1] + "\n" + out.append(line) + vars_file.write_text("".join(out), encoding="utf-8") diff --git a/src/gpu_rent/local_watchdog.py b/src/gpu_rent/local_watchdog.py new file mode 100644 index 0000000..8bf4a53 --- /dev/null +++ b/src/gpu_rent/local_watchdog.py @@ -0,0 +1,499 @@ +"""Local optional watchdog: stale tunnel lease → stop GPU. + +VM idle-killer remains the default safety net. This module is opt-in via +`gpu-rent watchdog install`: while a tunnel is armed, a scheduled tick stops +compute if the laptop/process died without Ctrl+C detach or `gpu-rent stop`. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +import threading +import time +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from gpu_rent.paths import ( + app_root, + local_lease_path, + local_watchdog_marker_path, + runtime_dir, +) + +Log = Callable[[str], None] + + +@dataclass +class LocalLease: + version: int = 1 + armed: bool = False + detached: bool = False + pid: int | None = None + heartbeat_at: str | None = None + app_root: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> LocalLease: + known = {k: v for k, v in data.items() if k in cls.__dataclass_fields__} + return cls(**known) + + +@dataclass(frozen=True) +class TickDecision: + kind: str # noop | stop + detail: str + + +def utc_now() -> datetime: + return datetime.now(timezone.utc).replace(microsecond=0) + + +def utc_now_iso() -> str: + return utc_now().isoformat() + + +def load_lease() -> LocalLease | None: + path = local_lease_path() + if not path.is_file(): + return None + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(raw, dict): + return None + return LocalLease.from_dict(raw) + + +def save_lease(lease: LocalLease) -> None: + runtime_dir().mkdir(parents=True, exist_ok=True) + local_lease_path().write_text( + json.dumps(lease.to_dict(), indent=2) + "\n", encoding="utf-8" + ) + + +def clear_lease() -> None: + path = local_lease_path() + if path.is_file(): + path.unlink(missing_ok=True) + + +def load_marker() -> dict[str, Any] | None: + path = local_watchdog_marker_path() + if not path.is_file(): + return None + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return raw if isinstance(raw, dict) else None + + +def watchdog_installed() -> bool: + return load_marker() is not None + + +def grace_seconds() -> int: + raw = (os.environ.get("LOCAL_WATCHDOG_GRACE_MINUTES") or "").strip() + try: + minutes = int(raw) if raw else 10 + except ValueError: + minutes = 10 + return max(1, minutes) * 60 + + +def pid_alive(pid: int | None) -> bool: + if pid is None or pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return False + return True + + +def _parse_iso(stamp: str | None) -> datetime | None: + if not stamp: + return None + try: + dt = datetime.fromisoformat(stamp) + except ValueError: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + + +def decide_local_tick( + *, + installed: bool, + has_server: bool, + lease: LocalLease | None, + now: datetime, + process_alive: bool, + grace_sec: int, +) -> TickDecision: + if not installed: + return TickDecision("noop", "watchdog not installed") + if not has_server: + return TickDecision("noop", "no compute in state") + if lease is None: + return TickDecision("noop", "no local lease (tunnel never armed)") + if lease.detached: + return TickDecision("noop", "detached after Ctrl+C") + if not lease.armed: + return TickDecision("noop", "lease not armed") + if process_alive: + return TickDecision("noop", "lease pid alive") + hb = _parse_iso(lease.heartbeat_at) + if hb is None: + return TickDecision("stop", "armed lease without heartbeat") + age = (now - hb).total_seconds() + if age < grace_sec: + return TickDecision("noop", f"grace {int(age)}s/{grace_sec}s") + return TickDecision("stop", f"stale heartbeat {int(age)}s, pid dead") + + +def arm_lease_for_tunnel() -> LocalLease: + lease = LocalLease( + armed=True, + detached=False, + pid=os.getpid(), + heartbeat_at=utc_now_iso(), + app_root=str(app_root()), + ) + save_lease(lease) + return lease + + +def touch_heartbeat() -> None: + lease = load_lease() + if lease is None or not lease.armed or lease.detached: + return + lease.heartbeat_at = utc_now_iso() + lease.pid = os.getpid() + save_lease(lease) + + +def detach_lease_keep_gpu() -> None: + """Ctrl+C on tunnel: leave GPU running; local tick must not stop.""" + lease = load_lease() + if lease is None: + lease = LocalLease() + lease.armed = False + lease.detached = True + lease.heartbeat_at = utc_now_iso() + lease.pid = None + lease.app_root = str(app_root()) + save_lease(lease) + + +_heartbeat_stop: threading.Event | None = None +_heartbeat_thread: threading.Thread | None = None + + +def start_heartbeat_thread(*, interval_seconds: float = 30.0) -> None: + global _heartbeat_stop, _heartbeat_thread + stop_heartbeat_thread() + if not watchdog_installed(): + return + arm_lease_for_tunnel() + stop = threading.Event() + _heartbeat_stop = stop + + def _loop() -> None: + while not stop.wait(interval_seconds): + try: + touch_heartbeat() + except OSError: + pass + + thread = threading.Thread(target=_loop, name="gpu-rent-lease-hb", daemon=True) + _heartbeat_thread = thread + thread.start() + + +def stop_heartbeat_thread() -> None: + global _heartbeat_stop, _heartbeat_thread + if _heartbeat_stop is not None: + _heartbeat_stop.set() + _heartbeat_stop = None + _heartbeat_thread = None + + +def _task_name(root: Path) -> str: + digest = hashlib.sha1(str(root.resolve()).encode("utf-8")).hexdigest()[:10] + return f"gpu-rent-local-watchdog-{digest}" + + +def install_watchdog( + *, + interval_minutes: int = 5, + log: Log = print, +) -> dict[str, Any]: + root = app_root().resolve() + runtime_dir().mkdir(parents=True, exist_ok=True) + interval = max(1, interval_minutes) + name = _task_name(root) + platform = sys.platform + if platform == "win32": + _install_windows(name, root, interval, log) + elif platform == "darwin": + _install_macos(name, root, interval, log) + else: + _install_linux(name, root, interval, log) + + marker = { + "version": 1, + "installed_at": utc_now_iso(), + "platform": platform, + "task_name": name, + "app_root": str(root), + "python": sys.executable, + "interval_minutes": interval, + } + local_watchdog_marker_path().write_text( + json.dumps(marker, indent=2) + "\n", encoding="utf-8" + ) + log( + f"local-watchdog установлен ({platform}): тик каждые {interval} мин. " + f"Grace {grace_seconds() // 60} мин после смерти процесса туннеля → stop. " + f"Ctrl+C на туннеле GPU не гасит." + ) + return marker + + +def uninstall_watchdog(*, log: Log = print) -> None: + marker = load_marker() + root = app_root().resolve() + name = (marker or {}).get("task_name") or _task_name(root) + platform = sys.platform + if platform == "win32": + _uninstall_windows(str(name), log) + elif platform == "darwin": + _uninstall_macos(str(name), log) + else: + _uninstall_linux(str(name), log) + path = local_watchdog_marker_path() + if path.is_file(): + path.unlink(missing_ok=True) + clear_lease() + log("local-watchdog снят") + + +def watchdog_status_lines() -> list[str]: + marker = load_marker() + lease = load_lease() + lines: list[str] = [] + if marker is None: + lines.append("не установлен (gpu-rent watchdog install)") + else: + lines.append( + f"установлен {marker.get('platform')} task={marker.get('task_name')} " + f"каждые {marker.get('interval_minutes')}м" + ) + if lease is None: + lines.append("lease: нет") + else: + alive = pid_alive(lease.pid) + lines.append( + f"lease: armed={lease.armed} detached={lease.detached} " + f"pid={lease.pid} alive={alive} hb={lease.heartbeat_at}" + ) + lines.append(f"grace: {grace_seconds() // 60} мин (LOCAL_WATCHDOG_GRACE_MINUTES)") + return lines + + +def run_tick(*, dry_run: bool = False, log: Log = print) -> TickDecision: + from gpu_rent.session import cmd_stop + from gpu_rent.state import load_state + + state = load_state() + lease = load_lease() + decision = decide_local_tick( + installed=watchdog_installed(), + has_server=bool(state.server_id), + lease=lease, + now=utc_now(), + process_alive=pid_alive(lease.pid if lease else None), + grace_sec=grace_seconds(), + ) + if decision.kind == "noop": + log(f"watchdog tick: noop ({decision.detail})") + return decision + log(f"watchdog tick: STOP — {decision.detail}") + if dry_run: + return decision + from gpu_rent.config import load_config + + cfg = load_config(require_auth=True) + cmd_stop(cfg, no_pull=True, log=log) + clear_lease() + return decision + + +def _install_windows(name: str, root: Path, interval: int, log: Log) -> None: + tr = ( + f'"{sys.executable}" -m gpu_rent watchdog tick ' + f'--project "{root}"' + ) + cmd = [ + "schtasks", + "/Create", + "/TN", + name, + "/SC", + "MINUTE", + "/MO", + str(interval), + "/TR", + tr, + "/F", + "/RL", + "LIMITED", + ] + proc = subprocess.run(cmd, capture_output=True, text=True, check=False) + if proc.returncode != 0: + err = (proc.stderr or proc.stdout or "").strip() + raise RuntimeError(f"schtasks failed: {err or proc.returncode}") + log(f"Task Scheduler: {name}") + + +def _uninstall_windows(name: str, log: Log) -> None: + subprocess.run( + ["schtasks", "/Delete", "/TN", name, "/F"], + capture_output=True, + text=True, + check=False, + ) + log(f"Task Scheduler удалён: {name}") + + +def _linux_unit_paths(name: str) -> tuple[Path, Path]: + base = Path.home() / ".config" / "systemd" / "user" + return base / f"{name}.service", base / f"{name}.timer" + + +def _install_linux(name: str, root: Path, interval: int, log: Log) -> None: + service_path, timer_path = _linux_unit_paths(name) + service_path.parent.mkdir(parents=True, exist_ok=True) + py = sys.executable + service_path.write_text( + f"""[Unit] +Description=gpu-rent local watchdog tick ({root}) + +[Service] +Type=oneshot +WorkingDirectory={root} +Environment=GPU_RENT_ROOT={root} +ExecStart={py} -m gpu_rent watchdog tick --project {root} +""", + encoding="utf-8", + ) + timer_path.write_text( + f"""[Unit] +Description=gpu-rent local watchdog every {interval} min + +[Timer] +OnBootSec=2min +OnUnitActiveSec={interval}min +AccuracySec=1min +Persistent=true + +[Install] +WantedBy=timers.target +""", + encoding="utf-8", + ) + subprocess.run(["systemctl", "--user", "daemon-reload"], check=False) + subprocess.run( + ["systemctl", "--user", "enable", "--now", f"{name}.timer"], + check=False, + ) + log(f"systemd user timer: {name}.timer") + + +def _uninstall_linux(name: str, log: Log) -> None: + subprocess.run( + ["systemctl", "--user", "disable", "--now", f"{name}.timer"], + check=False, + capture_output=True, + ) + service_path, timer_path = _linux_unit_paths(name) + service_path.unlink(missing_ok=True) + timer_path.unlink(missing_ok=True) + subprocess.run(["systemctl", "--user", "daemon-reload"], check=False) + log(f"systemd timer снят: {name}") + + +def _macos_plist_path(name: str) -> Path: + return Path.home() / "Library" / "LaunchAgents" / f"{name}.plist" + + +def _install_macos(name: str, root: Path, interval: int, log: Log) -> None: + plist = _macos_plist_path(name) + plist.parent.mkdir(parents=True, exist_ok=True) + seconds = interval * 60 + py = sys.executable + body = f""" + + + + Label + {name} + ProgramArguments + + {py} + -m + gpu_rent + watchdog + tick + --project + {root} + + WorkingDirectory + {root} + EnvironmentVariables + + GPU_RENT_ROOT + {root} + + StartInterval + {seconds} + RunAtLoad + + + +""" + plist.write_text(body, encoding="utf-8") + subprocess.run(["launchctl", "unload", str(plist)], check=False, capture_output=True) + proc = subprocess.run( + ["launchctl", "load", str(plist)], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + err = (proc.stderr or proc.stdout or "").strip() + raise RuntimeError(f"launchctl load failed: {err or proc.returncode}") + log(f"launchd: {plist}") + + +def _uninstall_macos(name: str, log: Log) -> None: + plist = _macos_plist_path(name) + subprocess.run(["launchctl", "unload", str(plist)], check=False, capture_output=True) + plist.unlink(missing_ok=True) + log(f"launchd снят: {name}") diff --git a/src/gpu_rent/os_client.py b/src/gpu_rent/os_client.py index 78b7f29..8a8aae3 100644 --- a/src/gpu_rent/os_client.py +++ b/src/gpu_rent/os_client.py @@ -5,6 +5,7 @@ from __future__ import annotations from collections.abc import Iterator from typing import Any +from gpu_rent import __version__ from gpu_rent.config import Config from gpu_rent.errors import CloudError @@ -43,7 +44,7 @@ def connect(cfg: Config): interface="public", compute_api_version=COMPUTE_MICROVERSION, app_name="gpu-rent", - app_version="0.1.0", + app_version=__version__, ) conn.authorize() try: diff --git a/src/gpu_rent/paths.py b/src/gpu_rent/paths.py index ff9b14f..019aaf0 100644 --- a/src/gpu_rent/paths.py +++ b/src/gpu_rent/paths.py @@ -2,11 +2,15 @@ from __future__ import annotations +import os import shutil from pathlib import Path def detect_app_root() -> Path: + override = (os.environ.get("GPU_RENT_ROOT") or "").strip() + if override: + return Path(override).expanduser().resolve() cwd = Path.cwd().resolve() for candidate in (cwd, *cwd.parents): if (candidate / "Models").is_dir() and (candidate / "docs").is_dir(): @@ -23,6 +27,14 @@ def runtime_dir() -> Path: return app_root() / ".gpu-rent" +def local_lease_path() -> Path: + return runtime_dir() / "local-lease.json" + + +def local_watchdog_marker_path() -> Path: + return runtime_dir() / "local-watchdog.json" + + # Back-compat alias used across the package. def home_dir() -> Path: return runtime_dir() @@ -36,6 +48,14 @@ def models_manifest_path() -> Path: return app_root() / "models.yaml" +def ollama_models_manifest_path() -> Path: + return app_root() / "ollama-models.yaml" + + +def ollama_models_example_path() -> Path: + return app_root() / "ollama-models.example.yaml" + + def extensions_manifest_path() -> Path: return app_root() / "extensions.yaml" diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index f18c230..76b185c 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -254,6 +254,61 @@ def ensure_swarmui_running(cfg: Config, host: str, log: Log, restart: bool) -> N run_ssh(cfg, host, "sudo -n systemctl enable swarmui", check=False) +def provision_llm(cfg: Config, host: str, log: Log) -> None: + from gpu_rent.llm_runtime import normalize_runtime, parse_ollama_models + from gpu_rent.ssh_ops import run_script_sudo + from gpu_rent.state import load_state, save_state + + runtime = normalize_runtime(cfg.llm_runtime) + if runtime == "none": + st = load_state() + st.notes = dict(st.notes or {}) + st.notes["llm_runtime"] = "none" + save_state(st) + return + if runtime == "ollama": + log("LLM: ставим/запускаем Ollama") + run_script_sudo( + cfg, + host, + _pkg_text("install_ollama.sh"), + remote_path="/tmp/gpu-rent-install_ollama.sh", + timeout=900, + env={"SWARM_USER": cfg.ssh_user}, + log=log, + ) + entries = parse_ollama_models(cfg.ollama_models_manifest) + names = [e.name for e in entries] + if not names: + log("ollama-models.yaml пуст — pull skip") + else: + put_text(cfg, host, "/tmp/gpu-rent-ollama-models.json", json.dumps(names, indent=2)) + log(f"Ollama: pull {len(names)} из манифеста") + run_python( + cfg, + host, + _pkg_text("ollama_pull.py"), + remote_path="/tmp/gpu-rent-ollama_pull.py", + timeout=7200, + log=log, + ) + elif runtime == "llamacpp": + log("LLM: ставим/запускаем llama.cpp server") + run_script_sudo( + cfg, + host, + _pkg_text("install_llamacpp.sh"), + remote_path="/tmp/gpu-rent-install_llamacpp.sh", + timeout=1200, + env={"SWARM_USER": cfg.ssh_user}, + log=log, + ) + st = load_state() + st.notes = dict(st.notes or {}) + st.notes["llm_runtime"] = runtime + save_state(st) + + def provision_vm( cfg: Config, host: str, @@ -282,10 +337,21 @@ def provision_vm( if cfg.pull_output: pull_tree(cfg, host, f"{DATA}/Output", cfg.local_output_dir, log) ensure_swarmui_running(cfg, host, log, restart=restart) + try: + provision_llm(cfg, host, log) + except Exception as exc: + log(f"LLM runtime: {exc}") if conn is not None and server_id: try: arm_idle_killer(cfg, host, conn, server_id, log) except GpuRentError as exc: log(f"idle-killer: {exc}") log("SwarmUI слушает 127.0.0.1:7801 — gpu-rent tunnel") + from gpu_rent.llm_runtime import normalize_runtime + + rt = normalize_runtime(cfg.llm_runtime) + if rt == "ollama": + log(f"Ollama API → localhost:{cfg.ollama_local_port} (туннель)") + elif rt == "llamacpp": + log(f"llama.cpp → localhost:{cfg.llamacpp_local_port} (туннель)") log("Hold killer: gpu-rent hold | Стоп GPU: gpu-rent stop") diff --git a/src/gpu_rent/remote/bootstrap.sh b/src/gpu_rent/remote/bootstrap.sh index 4e940ab..b3dd111 100644 --- a/src/gpu_rent/remote/bootstrap.sh +++ b/src/gpu_rent/remote/bootstrap.sh @@ -77,8 +77,12 @@ ensure_bind() { } log "пакеты (без upgrade ядра)" -apt-get update -qq -apt-get install -y -qq git python3 python3-venv python3-pip python3-full curl ca-certificates +if [[ "${GPU_RENT_BOOTSTRAP_LIGHT:-0}" == "1" && -f "$MARKER_BOOT" ]]; then + log "light bootstrap — пропускаем apt-get" +else + apt-get update -qq + apt-get install -y -qq git python3 python3-venv python3-pip python3-full curl ca-certificates +fi ensure_data_mount diff --git a/src/gpu_rent/remote/clone_ext.py b/src/gpu_rent/remote/clone_ext.py index 6dac1f4..be2ad00 100644 --- a/src/gpu_rent/remote/clone_ext.py +++ b/src/gpu_rent/remote/clone_ext.py @@ -27,6 +27,19 @@ def strip_auth(url: str) -> str: return urlunsplit((parts.scheme, host, parts.path, parts.query, parts.fragment)) +def scrub_origin(dest: Path, clean_url: str) -> None: + """Remove embedded tokens from git remote origin after clone/fetch.""" + try: + origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"]) + except subprocess.CalledProcessError: + return + wanted = strip_auth(clean_url) if clean_url else strip_auth(origin) + if origin == wanted: + return + run(["git", "-C", str(dest), "remote", "set-url", "origin", wanted]) + print(f"scrubbed token from origin {dest}") + + def with_token(url: str, token: str) -> str: if not token: return url @@ -71,16 +84,23 @@ def fetch_and_checkout(dest: Path, ref: str) -> None: print(f"updated {dest} ({ref})") -def update_tracking_branch(dest: Path) -> None: +def update_tracking_branch(dest: Path, token: str = "") -> None: branch = out(["git", "-C", str(dest), "rev-parse", "--abbrev-ref", "HEAD"]) if not branch or branch == "HEAD": print(f"skip detached {dest}") return - run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"]) + origin = out(["git", "-C", str(dest), "remote", "get-url", "origin"]) + clean = strip_auth(origin) + if token: + run(["git", "-C", str(dest), "remote", "set-url", "origin", with_token(clean, token)]) try: - run(["git", "-C", str(dest), "reset", "--hard", f"origin/{branch}"]) - except subprocess.CalledProcessError: - run(["git", "-C", str(dest), "pull", "--ff-only"]) + run(["git", "-C", str(dest), "fetch", "--recurse-submodules", "--tags", "origin"]) + try: + run(["git", "-C", str(dest), "reset", "--hard", f"origin/{branch}"]) + except subprocess.CalledProcessError: + run(["git", "-C", str(dest), "pull", "--ff-only"]) + finally: + scrub_origin(dest, clean) print(f"updated installed {dest} ({branch})") @@ -95,28 +115,50 @@ def clone_one(job: dict, token: str, update: bool) -> None: if strip_auth(origin) != strip_auth(url): print(f"FAIL origin mismatch {dest}: {origin} != {url}", file=sys.stderr) raise SystemExit(2) + if token and strip_auth(origin) == strip_auth(url): + run(["git", "-C", str(dest), "remote", "set-url", "origin", authed]) if not update: + scrub_origin(dest, url) print(f"skip update {dest}") return - fetch_and_checkout(dest, ref) + try: + fetch_and_checkout(dest, ref) + finally: + scrub_origin(dest, url) return if dest.exists(): print(f"FAIL {dest} exists but is not a git repo", file=sys.stderr) raise SystemExit(2) - if is_sha(ref): - run(["git", "clone", "--recurse-submodules", authed, str(dest)]) - run(["git", "-C", str(dest), "fetch", "origin", ref]) - run(["git", "-C", str(dest), "checkout", ref]) - else: - try: - run(["git", "clone", "--recurse-submodules", "--depth", "1", "--branch", ref, authed, str(dest)]) - except subprocess.CalledProcessError: + try: + if is_sha(ref): run(["git", "clone", "--recurse-submodules", authed, str(dest)]) + run(["git", "-C", str(dest), "fetch", "origin", ref]) run(["git", "-C", str(dest), "checkout", ref]) + else: + try: + run( + [ + "git", + "clone", + "--recurse-submodules", + "--depth", + "1", + "--branch", + ref, + authed, + str(dest), + ] + ) + except subprocess.CalledProcessError: + run(["git", "clone", "--recurse-submodules", authed, str(dest)]) + run(["git", "-C", str(dest), "checkout", ref]) + finally: + if dest.is_dir() and (dest / ".git").is_dir(): + scrub_origin(dest, url) print(f"cloned {dest}") -def update_installed_extras(known: set[str], update: bool) -> None: +def update_installed_extras(known: set[str], update: bool, token: str = "") -> None: if not update: return for root in EXTRA_ROOTS: @@ -129,7 +171,7 @@ def update_installed_extras(known: set[str], update: bool) -> None: if key in known: continue try: - update_tracking_branch(child) + update_tracking_branch(child, token=token) except Exception as exc: print(f"FAIL installed {child}: {exc}", file=sys.stderr) raise @@ -151,7 +193,7 @@ def main() -> int: failed += 1 print(f"FAIL {job.get('dest')}: {exc}", file=sys.stderr) try: - update_installed_extras(known, update) + update_installed_extras(known, update, token=token) except Exception: failed += 1 if TOKEN_PATH.is_file(): diff --git a/src/gpu_rent/remote/idle_killer.py b/src/gpu_rent/remote/idle_killer.py index 0fef0d2..6a65ad0 100644 --- a/src/gpu_rent/remote/idle_killer.py +++ b/src/gpu_rent/remote/idle_killer.py @@ -88,6 +88,46 @@ def swarm_busy(swarm_url: str, timeout: float = 8.0) -> tuple[bool, str]: return False, f"idle backend={bstat}" +def llm_busy(timeout: float = 3.0) -> tuple[bool, str]: + """Ollama pull / loaded models or llama.cpp with a model count as busy.""" + if (DATA / ".gpu-rent-ollama-pulling").is_file(): + return True, "ollama pulling" + ctx = ssl.create_default_context() + # Ollama: any running model + try: + req = urllib.request.Request("http://127.0.0.1:11434/api/ps", method="GET") + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + data = json.loads(resp.read().decode("utf-8")) + models = data.get("models") or [] + if models: + names = ",".join(str(m.get("name") or "?") for m in models[:3]) + return True, f"ollama running {names}" + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, json.JSONDecodeError, OSError): + pass + # llama.cpp OpenAI models endpoint — if server up and lists a model, treat lightly: + # only busy if /health ok AND we recently had activity is hard; use loaded via props. + try: + req = urllib.request.Request("http://127.0.0.1:8080/health", method="GET") + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + if getattr(resp, "status", 200) == 200: + # Server alive with a model is OK for idle unless slots busy — skip kill only + # when props show n_slots_in_use if available. + try: + req2 = urllib.request.Request("http://127.0.0.1:8080/props", method="GET") + with urllib.request.urlopen(req2, timeout=timeout, context=ctx) as resp2: + props = json.loads(resp2.read().decode("utf-8")) + in_use = int(props.get("total_slots") or 0) - int( + props.get("available_slots") or props.get("total_slots") or 0 + ) + if in_use > 0: + return True, f"llamacpp slots_in_use={in_use}" + except Exception: + pass + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError): + pass + return False, "llm idle" + + def keystone_token(creds: dict) -> tuple[str, str]: """Return (token, compute_url).""" auth = { @@ -189,6 +229,12 @@ def main() -> int: log(f"busy: {detail}") return 0 + llm_is_busy, llm_detail = llm_busy() + if llm_is_busy: + write_ts(IDLE_SINCE, None) + log(f"busy: {llm_detail}") + return 0 + idle_minutes = float(creds.get("idle_minutes") or 30) since = read_ts(IDLE_SINCE) if since is None: diff --git a/src/gpu_rent/remote/install_llamacpp.sh b/src/gpu_rent/remote/install_llamacpp.sh new file mode 100644 index 0000000..fc0a298 --- /dev/null +++ b/src/gpu_rent/remote/install_llamacpp.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Install llama-server (CUDA) for OpenAI-compatible API on loopback :8080. +set -euo pipefail + +SWARM_USER="${SWARM_USER:-ubuntu}" +DATA_ROOT="/mnt/swarm_data" +LLAMA_ROOT="${DATA_ROOT}/llamacpp" +MODELS_DIR="${LLAMA_ROOT}/models" +BIN_DIR="${LLAMA_ROOT}/bin" +UNIT="gpu-rent-llamacpp" + +log() { echo "[gpu-rent-llamacpp] $*"; } + +if [[ "$(id -u)" -ne 0 ]]; then + echo "нужен root" >&2 + exit 1 +fi + +mkdir -p "$MODELS_DIR" "$BIN_DIR" +chown -R "${SWARM_USER}:${SWARM_USER}" "$LLAMA_ROOT" + +SERVER_BIN="${BIN_DIR}/llama-server" +if [[ ! -x "$SERVER_BIN" ]]; then + log "скачиваю llama-server (cuda) release…" + # Pin a known-good release asset pattern; fallback to CPU if CUDA asset missing. + TMP="$(mktemp -d)" + cd "$TMP" + API="https://api.github.com/repos/ggerganov/llama.cpp/releases/latest" + URL="$(curl -fsSL "$API" | python3 -c ' +import json,sys,re +data=json.load(sys.stdin) +assets=data.get("assets") or [] +prefer=[] +for a in assets: + n=(a.get("name") or "").lower() + u=a.get("browser_download_url") or "" + if not u.endswith(".zip") and not u.endswith(".tar.gz"): + continue + if "cuda" in n or "cu12" in n or "cu11" in n: + prefer.append(u) + elif "ubuntu" in n or "linux" in n: + prefer.append(u) +print(prefer[0] if prefer else "") +')" + if [[ -z "$URL" ]]; then + log "не нашёл бинарь в latest release — поставь llama-server вручную в ${SERVER_BIN}" + exit 1 + fi + log "asset $URL" + curl -fL "$URL" -o pkg.bin + if file pkg.bin | grep -qi zip; then + apt-get install -y -qq unzip >/dev/null 2>&1 || true + unzip -qo pkg.bin -d out + else + mkdir -p out + tar -xaf pkg.bin -C out 2>/dev/null || tar -xzf pkg.bin -C out + fi + FOUND="$(find out -type f -name 'llama-server' | head -n1 || true)" + if [[ -z "$FOUND" ]]; then + FOUND="$(find out -type f -name 'server' | head -n1 || true)" + fi + if [[ -z "$FOUND" ]]; then + log "в архиве нет llama-server" + exit 1 + fi + install -m 755 "$FOUND" "$SERVER_BIN" + chown "${SWARM_USER}:${SWARM_USER}" "$SERVER_BIN" + rm -rf "$TMP" +fi + +# Pick first GGUF if present; otherwise unit starts but API may idle without model. +MODEL_ARG="" +FIRST_GGUF="$(find "$MODELS_DIR" -type f \( -name '*.gguf' -o -name '*.GGUF' \) | head -n1 || true)" +if [[ -n "$FIRST_GGUF" ]]; then + MODEL_ARG="-m ${FIRST_GGUF}" + log "модель ${FIRST_GGUF}" +else + log "нет GGUF в ${MODELS_DIR} — положи файл вручную и systemctl restart ${UNIT}" +fi + +cat >/etc/systemd/system/${UNIT}.service <&2 + exit 1 +fi + +mkdir -p "$OLLAMA_HOME" +chown -R "${SWARM_USER}:${SWARM_USER}" "$OLLAMA_HOME" + +if ! command -v ollama >/dev/null 2>&1; then + log "ставлю ollama" + curl -fsSL https://ollama.com/install.sh | sh +else + log "ollama уже в PATH: $(command -v ollama)" +fi + +# Stop stock unit if present — we run our own bind to loopback + data dir. +systemctl stop ollama 2>/dev/null || true +systemctl disable ollama 2>/dev/null || true + +cat >/etc/systemd/system/${UNIT}.service </dev/null +log "ok — OLLAMA_HOST=127.0.0.1:11434 models=${OLLAMA_HOME}" diff --git a/src/gpu_rent/remote/ollama_pull.py b/src/gpu_rent/remote/ollama_pull.py new file mode 100644 index 0000000..c03a133 --- /dev/null +++ b/src/gpu_rent/remote/ollama_pull.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Pull Ollama models from a JSON list. Stdlib only. Runs on the VM.""" +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +JOBS = Path("/tmp/gpu-rent-ollama-models.json") +MARKER = Path("/mnt/swarm_data/.gpu-rent-ollama-pulling") + + +def listed() -> set[str]: + try: + out = subprocess.check_output(["ollama", "list"], text=True, stderr=subprocess.DEVNULL) + except (subprocess.CalledProcessError, FileNotFoundError): + return set() + names: set[str] = set() + for i, line in enumerate(out.splitlines()): + if i == 0 and line.lower().startswith("name"): + continue + parts = line.split() + if parts: + names.add(parts[0]) + # also bare name without tag + names.add(parts[0].split(":")[0]) + return names + + +def main() -> int: + if not JOBS.is_file(): + print("no jobs file", file=sys.stderr) + return 1 + models = json.loads(JOBS.read_text(encoding="utf-8")) + if not isinstance(models, list) or not models: + print("ollama pull: пустой список — skip") + return 0 + have = listed() + MARKER.parent.mkdir(parents=True, exist_ok=True) + MARKER.write_text("1\n", encoding="utf-8") + failed = 0 + try: + for i, name in enumerate(models, 1): + name = str(name).strip() + if not name: + continue + bare = name.split(":")[0] + if name in have or bare in have: + # Prefer exact tag match when possible + exact = any(h == name or h.startswith(name + ":") or name.startswith(h) for h in have) + if name in have or exact: + print(f"[{i}/{len(models)}] уже есть {name}") + continue + print(f"[{i}/{len(models)}] ollama pull {name}") + try: + subprocess.check_call(["ollama", "pull", name]) + except subprocess.CalledProcessError as exc: + failed += 1 + print(f"FAIL pull {name}: {exc}", file=sys.stderr) + finally: + MARKER.unlink(missing_ok=True) + if failed: + return 1 + print("ollama pull ok") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/gpu_rent/session.py b/src/gpu_rent/session.py index 9020527..170b431 100644 --- a/src/gpu_rent/session.py +++ b/src/gpu_rent/session.py @@ -84,12 +84,25 @@ def _bind_access( save_state(state) wait_ssh(cfg, ip) log(f"SSH {cfg.ssh_user}@{ip}") + state.phase = "bootstrapping" + save_state(state) if update: active = run_ssh(cfg, ip, "systemctl is-active swarmui 2>/dev/null || true", check=False).strip() if active == "active": log("systemctl stop swarmui перед git update") run_ssh(cfg, ip, "sudo -n systemctl stop swarmui", timeout=120, check=False) - run_bootstrap(cfg, ip, log, update=update) + # Skip apt-heavy bootstrap when the VM already finished first-boot. + marker = run_ssh( + cfg, + ip, + "test -f /opt/swarmui/.gpu-rent-bootstrapped && echo yes || echo no", + check=False, + ).strip() + if marker == "yes" and state.bootstrapped: + log("bootstrap уже на VM — лёгкий проход (без apt)") + run_bootstrap(cfg, ip, log, update=update, light=True) + else: + run_bootstrap(cfg, ip, log, update=update, light=False) provision_vm( cfg, ip, @@ -135,6 +148,8 @@ def adopt_server(cfg: Config, log: Log = _log_default, *, update: bool = True) - save_state(state) log(f"подхватили {server.id} статус {server_status(server)}") if server_status(server) == "ACTIVE": + state.phase = "bootstrapping" + save_state(state) _bind_access(conn, server, state, cfg, log, update=update) return state @@ -165,7 +180,7 @@ def cmd_up( if status == "ACTIVE": if state.bootstrapped and state.floating_ip: log("сервер уже ACTIVE — второй GPU не создаём") - state.phase = "ready_cloud" + state.phase = "bootstrapping" save_state(state) _bind_access(conn, existing, state, cfg, log, update=do_update) return state @@ -187,7 +202,7 @@ def cmd_up( outcome = probe_ssh(cfg, fip, attempts=2) if fip else "down" if outcome == "ok": log("сервер ACTIVE, SSH ок — продолжаем bootstrap") - state.phase = "ready_cloud" + state.phase = "bootstrapping" save_state(state) _bind_access(conn, existing, state, cfg, log, update=do_update) return state @@ -213,7 +228,7 @@ def cmd_up( if existing is not None and status in {"EXPIRED", "SHELVED", "SHELVED_OFFLOADED"}: existing = unshelve(conn, existing, log) state.server_id = existing.id - state.phase = "ready_cloud" + state.phase = "bootstrapping" state.unshelved_at = utc_now() save_state(state) _bind_access(conn, existing, state, cfg, log, update=do_update) @@ -341,7 +356,7 @@ def cmd_up( state.server_name = getattr(server, "name", None) state.created_at = utc_now() state.unshelved_at = None - state.phase = "ready_cloud" + state.phase = "bootstrapping" save_state(state) _bind_access(conn, server, state, cfg, log, update=do_update) return state @@ -396,7 +411,14 @@ def cmd_stop( state.server_id = None state.server_name = None + state.bootstrapped = False state.phase = "idle" save_state(state) + try: + from gpu_rent.local_watchdog import clear_lease + + clear_lease() + except Exception: + pass log("фаза idle" + ("" if destroy_disks else " (диски на месте)")) return state diff --git a/src/gpu_rent/setup_wizard.py b/src/gpu_rent/setup_wizard.py new file mode 100644 index 0000000..b69fd7c --- /dev/null +++ b/src/gpu_rent/setup_wizard.py @@ -0,0 +1,100 @@ +"""Interactive setup wizard: files + LLM_RUNTIME + optional watchdog.""" + +from __future__ import annotations + +import shutil +from collections.abc import Callable +from pathlib import Path + +from gpu_rent.llm_runtime import ( + PRESET_HELP, + append_vars_llm_runtime, + ensure_ollama_manifest_from_example, + normalize_runtime, + write_ollama_models_preset, +) +from gpu_rent.paths import ( + app_root, + env_path, + extensions_manifest_path, + models_manifest_path, + ollama_models_example_path, + ollama_models_manifest_path, + vars_example_path, + vars_path, +) + +Log = Callable[[str], None] + + +def _copy_if_missing(src: Path, dst: Path, label: str, log: Log) -> None: + if dst.is_file(): + log(f"есть {label}") + return + if src.is_file(): + shutil.copy2(src, dst) + log(f"создал {label} из example") + else: + log(f"нет example для {label}: {src}") + + +def run_setup( + *, + llm: str | None = None, + ollama_preset: str | None = None, + install_watchdog: bool | None = None, + confirm: Callable[[str], bool] | None = None, + ask: Callable[[str, str], str] | None = None, + log: Log = print, +) -> None: + root = app_root() + log(f"setup в {root}") + + _copy_if_missing(root / "env.example", env_path(), ".env", log) + _copy_if_missing(root / "models.example.yaml", models_manifest_path(), "models.yaml", log) + _copy_if_missing( + root / "extensions.example.yaml", extensions_manifest_path(), "extensions.yaml", log + ) + _copy_if_missing(vars_example_path(), vars_path(), "gpu-rent.vars", log) + _copy_if_missing( + ollama_models_example_path(), ollama_models_manifest_path(), "ollama-models.yaml", log + ) + + runtime = llm + if runtime is None: + if ask: + runtime = ask( + "LLM runtime [none/ollama/llamacpp]", + "none", + ) + else: + runtime = "none" + runtime = normalize_runtime(runtime) + append_vars_llm_runtime(vars_path(), runtime) + log(f"LLM_RUNTIME={runtime} → gpu-rent.vars") + + if runtime == "ollama": + preset = ollama_preset + if preset is None and ask: + log(PRESET_HELP) + preset = ask("Ollama preset [recommended/light/stock/alt/empty]", "recommended") + if preset is None: + preset = "recommended" + if preset.strip().lower() in {"keep", "example", ""}: + ensure_ollama_manifest_from_example() + log("ollama-models.yaml из example") + else: + write_ollama_models_preset(ollama_models_manifest_path(), preset) + log(f"ollama-models.yaml пресет={preset}") + + do_wd = install_watchdog + if do_wd is None and confirm: + do_wd = confirm("Установить local-watchdog (аварийный stop без Ctrl+C)?") + if do_wd: + from gpu_rent.local_watchdog import install_watchdog as _install + + _install(log=log) + elif do_wd is False: + log("local-watchdog: skip") + + log("готово. Заполни .env (OS_*), потом: gpu-rent doctor && gpu-rent up") diff --git a/src/gpu_rent/ssh_ops.py b/src/gpu_rent/ssh_ops.py index 6a8f609..41b2199 100644 --- a/src/gpu_rent/ssh_ops.py +++ b/src/gpu_rent/ssh_ops.py @@ -71,18 +71,26 @@ def probe_ssh(cfg: Config, host: str, attempts: int = 3) -> str: def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None: - """Wait until sshd accepts our key. Paramiko banner noise is muted.""" + """Wait until sshd accepts our key. Paramiko banner noise is muted. + + AuthenticationException is normal while cloud-init injects keys: keep + retrying. Only give up early after AUTH_GIVE_UP seconds of *continuous* + auth rejection (sshd up, key still wrong) so recreate-with-user_data can run. + """ import logging logging.getLogger("paramiko").setLevel(logging.CRITICAL) logging.getLogger("paramiko.transport").setLevel(logging.CRITICAL) + # sshd may answer before authorized_keys is ready (boot-from-volume / user_data). + auth_give_up = 180.0 + wait_tcp(host, 22, timeout=min(timeout, 300)) deadline = time.time() + timeout key = str(cfg.ssh_private_key_path) last = None attempt = 0 - auth_fails = 0 + auth_streak_started: float | None = None while time.time() < deadline: attempt += 1 client = paramiko.SSHClient() @@ -103,19 +111,26 @@ def wait_ssh(cfg: Config, host: str, timeout: float = 900.0) -> None: except Exception as exc: last = exc name = type(exc).__name__ - if "Authentication" in name or "Authentication" in str(exc): - auth_fails += 1 + is_auth = "Authentication" in name or "Authentication" in str(exc) + if is_auth: + if auth_streak_started is None: + auth_streak_started = time.time() + else: + auth_streak_started = None try: client.close() except Exception: pass if attempt == 1 or attempt % 6 == 0: print(f"жду SSH {cfg.ssh_user}@{host}… ({name})", flush=True) - # Key never injected (boot-from-volume): don't burn full timeout. - if auth_fails >= 3: + if ( + is_auth + and auth_streak_started is not None + and (time.time() - auth_streak_started) >= auth_give_up + ): raise CloudError( - f"SSH {cfg.ssh_user}@{host}: ключ отклонён (AuthenticationException). " - "Nova keypair не попал в authorized_keys при boot-from-volume. " + f"SSH {cfg.ssh_user}@{host}: ключ отклонён {int(auth_give_up)}с подряд. " + "Nova keypair / user_data не попал в authorized_keys. " "up пересоздаст compute с user_data (Base64), диски оставит." ) from exc time.sleep(5) @@ -270,41 +285,67 @@ def _connect(cfg: Config, host: str) -> paramiko.SSHClient: return client +def open_ssh(cfg: Config, host: str) -> paramiko.SSHClient: + """Public alias for a connected SSH client (caller must close).""" + return _connect(cfg, host) + + def put_text(cfg: Config, host: str, remote_path: str, text: str, mode: int = 0o644) -> None: client = _connect(cfg, host) try: - sftp = client.open_sftp() + put_text_on(client, remote_path, text, mode=mode) + finally: + client.close() + + +def put_text_on( + client: paramiko.SSHClient, remote_path: str, text: str, mode: int = 0o644 +) -> None: + sftp = client.open_sftp() + try: _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/")) with sftp.file(remote_path, "w") as fh: fh.write(text) sftp.chmod(remote_path, mode) - sftp.close() finally: - client.close() + sftp.close() def put_file(cfg: Config, host: str, local: Path, remote_path: str) -> None: client = _connect(cfg, host) try: - sftp = client.open_sftp() - _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/")) - sftp.put(str(local), remote_path) - sftp.close() + put_file_on(client, local, remote_path) finally: client.close() +def put_file_on(client: paramiko.SSHClient, local: Path, remote_path: str) -> None: + sftp = client.open_sftp() + try: + _sftp_mkdirs(sftp, str(Path(remote_path).parent).replace("\\", "/")) + sftp.put(str(local), remote_path) + finally: + sftp.close() + + def get_file(cfg: Config, host: str, remote_path: str, local: Path) -> None: local.parent.mkdir(parents=True, exist_ok=True) client = _connect(cfg, host) try: - sftp = client.open_sftp() - sftp.get(remote_path, str(local)) - sftp.close() + get_file_on(client, remote_path, local) finally: client.close() +def get_file_on(client: paramiko.SSHClient, remote_path: str, local: Path) -> None: + local.parent.mkdir(parents=True, exist_ok=True) + sftp = client.open_sftp() + try: + sftp.get(remote_path, str(local)) + finally: + sftp.close() + + def remote_exists(cfg: Config, host: str, remote_path: str) -> bool: client = _connect(cfg, host) try: @@ -321,15 +362,35 @@ def remote_exists(cfg: Config, host: str, remote_path: str) -> bool: def remote_sha256(cfg: Config, host: str, remote_path: str) -> str | None: - out = run_ssh( - cfg, - host, + client = _connect(cfg, host) + try: + return remote_sha256_on(client, remote_path) + finally: + client.close() + + +def remote_sha256_on(client: paramiko.SSHClient, remote_path: str) -> str | None: + _stdin, stdout, stderr = client.exec_command( f"sha256sum {shlex.quote(remote_path)} 2>/dev/null | awk '{{print $1}}'", - check=False, - ).strip() + timeout=60, + ) + del stderr + out = stdout.read().decode("utf-8", errors="replace").strip() return out or None +def run_ssh_on( + client: paramiko.SSHClient, command: str, timeout: int = 60, check: bool = True +) -> str: + _stdin, stdout, stderr = client.exec_command(command, timeout=timeout) + out = stdout.read().decode("utf-8", errors="replace") + err = stderr.read().decode("utf-8", errors="replace") + code = stdout.channel.recv_exit_status() + if check and code != 0: + raise CloudError(f"SSH `{command}` exit {code}: {err or out}") + return out + + def _sftp_mkdirs(sftp, remote_dir: str) -> None: if not remote_dir or remote_dir == "/": return diff --git a/src/gpu_rent/sync_files.py b/src/gpu_rent/sync_files.py index b16cf3b..85a2af8 100644 --- a/src/gpu_rent/sync_files.py +++ b/src/gpu_rent/sync_files.py @@ -7,7 +7,13 @@ from pathlib import Path from gpu_rent.config import Config from gpu_rent.payload import has_payload, iter_payload_files, model_push_set, sha256_file -from gpu_rent.ssh_ops import get_file, put_file, remote_sha256, run_ssh +from gpu_rent.ssh_ops import ( + get_file_on, + open_ssh, + put_file_on, + remote_sha256_on, + run_ssh_on, +) Log = Callable[[str], None] @@ -26,19 +32,22 @@ def push_tree( return 0 files = model_push_set(local_root) if models else iter_payload_files(local_root) sent = 0 - for path in files: - rel = path.relative_to(local_root).as_posix() - remote = f"{remote_root.rstrip('/')}/{rel}" - local_hash = sha256_file(path) - remote_hash = remote_sha256(cfg, host, remote) - if remote_hash and remote_hash.lower() == local_hash.lower(): - continue - if models and not _is_weight_name(path.name): - # sidecar: warn if we somehow got here without weight — still send - pass - log(f"push {rel}") - put_file(cfg, host, path, remote) - sent += 1 + client = open_ssh(cfg, host) + try: + for path in files: + rel = path.relative_to(local_root).as_posix() + remote = f"{remote_root.rstrip('/')}/{rel}" + local_hash = sha256_file(path) + remote_hash = remote_sha256_on(client, remote) + if remote_hash and remote_hash.lower() == local_hash.lower(): + continue + if models and not _is_weight_name(path.name): + pass + log(f"push {rel}") + put_file_on(client, path, remote) + sent += 1 + finally: + client.close() if sent == 0: log(f"push {local_root.name}: всё уже на VM") else: @@ -48,29 +57,34 @@ def push_tree( def _is_weight_name(name: str) -> bool: lower = name.lower() - return lower.endswith((".safetensors", ".ckpt", ".pt", ".pth", ".bin", ".gguf", ".sft", ".onnx")) + return lower.endswith( + (".safetensors", ".ckpt", ".pt", ".pth", ".bin", ".gguf", ".sft", ".onnx") + ) def pull_tree(cfg: Config, host: str, remote_root: str, local_root: Path, log: Log) -> int: - listing = run_ssh( - cfg, - host, - f"find {remote_root} -type f 2>/dev/null | sed 's|^{remote_root}/||'", - check=False, - timeout=120, - ) - names = [line.strip() for line in listing.splitlines() if line.strip()] - pulled = 0 - for rel in names: - if rel.endswith("/.gitkeep") or rel.rsplit("/", 1)[-1] in {".gitkeep", "README.md"}: - continue - remote = f"{remote_root.rstrip('/')}/{rel}" - local = local_root / rel - remote_hash = remote_sha256(cfg, host, remote) - if local.is_file() and remote_hash and sha256_file(local).lower() == remote_hash.lower(): - continue - log(f"pull {rel}") - get_file(cfg, host, remote, local) - pulled += 1 + client = open_ssh(cfg, host) + try: + listing = run_ssh_on( + client, + f"find {remote_root} -type f 2>/dev/null | sed 's|^{remote_root}/||'", + check=False, + timeout=120, + ) + names = [line.strip() for line in listing.splitlines() if line.strip()] + pulled = 0 + for rel in names: + if rel.endswith("/.gitkeep") or rel.rsplit("/", 1)[-1] in {".gitkeep", "README.md"}: + continue + remote = f"{remote_root.rstrip('/')}/{rel}" + local = local_root / rel + remote_hash = remote_sha256_on(client, remote) + if local.is_file() and remote_hash and sha256_file(local).lower() == remote_hash.lower(): + continue + log(f"pull {rel}") + get_file_on(client, remote, local) + pulled += 1 + finally: + client.close() log(f"pull Output: {pulled} файл(ов)" if pulled else "pull Output: нечего забирать") return pulled diff --git a/src/gpu_rent/tunnel.py b/src/gpu_rent/tunnel.py index 91a12b3..4e258fc 100644 --- a/src/gpu_rent/tunnel.py +++ b/src/gpu_rent/tunnel.py @@ -15,6 +15,7 @@ from gpu_rent.cloud import ( ) from gpu_rent.config import Config from gpu_rent.errors import CloudError, GpuRentError +from gpu_rent.llm_runtime import llm_local_port, llm_remote_port, normalize_runtime from gpu_rent.os_client import connect from gpu_rent.ssh_ops import wait_ssh from gpu_rent.state import load_state, save_state, utc_now @@ -67,27 +68,55 @@ def decide_watch(status: str | None, tunnel_alive: bool) -> WatchDecision: return WatchDecision("reconnect", "туннель мёртв, сервер ACTIVE") if st == "ACTIVE": return WatchDecision("ok", "ACTIVE") - # transitional: BUILD, REBOOT, … return WatchDecision("ok", f"ждём {st}") -def _start_forwarder(cfg: Config, host: str, local_port: int): +def tunnel_forwards(cfg: Config) -> list[tuple[int, int]]: + """List of (local_port, remote_port). SwarmUI always; LLM if configured.""" + pairs = [(cfg.swarmui_local_port, 7801)] + runtime = normalize_runtime(cfg.llm_runtime) + # Prefer state notes if provision recorded a different runtime this session. + state = load_state() + noted = (state.notes or {}).get("llm_runtime") + if noted: + try: + runtime = normalize_runtime(str(noted)) + except ValueError: + pass + remote = llm_remote_port(runtime) + local = llm_local_port(cfg) if runtime != "none" else None + # llm_local_port uses cfg.llm_runtime — override by mutating check: + if runtime == "ollama": + local = cfg.ollama_local_port + remote = 11434 + elif runtime == "llamacpp": + local = cfg.llamacpp_local_port + remote = 8080 + if local and remote: + pairs.append((local, remote)) + return pairs + + +def _start_forwarder(cfg: Config, host: str, forwards: list[tuple[int, int]] | None = None): SSHTunnelForwarder = _ssh_tunnel_forwarder() + pairs = forwards or tunnel_forwards(cfg) + local_binds = [("127.0.0.1", loc) for loc, _ in pairs] + remote_binds = [("127.0.0.1", rem) for _, rem in pairs] server = SSHTunnelForwarder( (host, 22), ssh_username=cfg.ssh_user, ssh_pkey=str(cfg.ssh_private_key_path), - remote_bind_address=("127.0.0.1", 7801), - local_bind_address=("127.0.0.1", local_port), + remote_bind_addresses=remote_binds, + local_bind_addresses=local_binds, set_keepalive=30, ) try: server.start() except Exception as exc: + ports = ",".join(str(p[0]) for p in pairs) raise CloudError( - f"не открыть туннель на {local_port}: {exc}. Порт занят локальным SwarmUI? " - "17801 должен быть свободен." + f"не открыть туннель на {ports}: {exc}. Порт занят?" ) from exc return server @@ -139,7 +168,7 @@ def _poll_nova(cfg: Config, log: Log) -> tuple[str | None, str]: return server_status(server), server.id except GpuRentError as exc: log(f"watch: OpenStack временно недоступен ({exc})") - return "ACTIVE", "auth-soft-fail" # don't tear down on transient auth blip + return "ACTIVE", "auth-soft-fail" def run_tunnel( @@ -156,24 +185,49 @@ def run_tunnel( except ImportError as exc: raise CloudError("Нет sshtunnel. Переустанови пакет: pip install -e .") from exc - local_port = cfg.swarmui_local_port + forwards = tunnel_forwards(cfg) current_host = host - log(f"туннель 127.0.0.1:{local_port} -> {current_host}:7801") + for loc, rem in forwards: + log(f"туннель 127.0.0.1:{loc} -> {current_host}:{rem}") log("Ctrl+C закрывает туннель, GPU оставляет. Стоп GPU: gpu-rent stop") log("watchdog: EXPIRED → unshelve + reconnect") - server = _start_forwarder(cfg, current_host, local_port) - url = f"http://127.0.0.1:{local_port}" - log(f"UI {url}") - log(f"API {url}/API/") - log(f"MCP {url}/mcp") - if open_browser: - webbrowser.open(url) - + server = _start_forwarder(cfg, current_host, forwards) + swarm_url = f"http://127.0.0.1:{cfg.swarmui_local_port}" + log(f"UI {swarm_url}") + log(f"API {swarm_url}/API/") + log(f"MCP {swarm_url}/mcp") + runtime = normalize_runtime(cfg.llm_runtime) state = load_state() + if (state.notes or {}).get("llm_runtime"): + try: + runtime = normalize_runtime(str(state.notes["llm_runtime"])) + except ValueError: + pass + if runtime == "ollama": + log(f"Ollama API http://127.0.0.1:{cfg.ollama_local_port} (OLLAMA_HOST=…)") + elif runtime == "llamacpp": + log(f"llama.cpp http://127.0.0.1:{cfg.llamacpp_local_port}") + if open_browser: + webbrowser.open(swarm_url) + state.phase = "ready_tunneled" save_state(state) + from gpu_rent.local_watchdog import ( + detach_lease_keep_gpu, + start_heartbeat_thread, + stop_heartbeat_thread, + watchdog_installed, + ) + + if watchdog_installed(): + start_heartbeat_thread() + log( + "local-watchdog: heartbeat активен — аварийное закрытие " + "(не Ctrl+C) → stop после grace" + ) + try: if wait is not None: wait() @@ -183,7 +237,6 @@ def run_tunnel( while True: time.sleep(1) if not server.is_active: - # fall through to poll immediately next_poll = 0 if time.time() < next_poll: continue @@ -201,7 +254,7 @@ def run_tunnel( log(f"reconnect: {decision.detail}") _stop_forwarder(server) try: - server = _start_forwarder(cfg, current_host, local_port) + server = _start_forwarder(cfg, current_host, forwards) log(f"туннель снова на {current_host}") except CloudError as exc: log(f"reconnect не вышел: {exc}") @@ -212,12 +265,14 @@ def run_tunnel( _stop_forwarder(server) try: current_host = _recover_unshelve(cfg, log) - server = _start_forwarder(cfg, current_host, local_port) - log(f"туннель после unshelve → {current_host}:7801") + server = _start_forwarder(cfg, current_host, forwards) + log(f"туннель после unshelve → {current_host}") except (CloudError, GpuRentError) as exc: log(f"unshelve/reconnect fail: {exc}") return except KeyboardInterrupt: + detach_lease_keep_gpu() log("туннель закрыт. GPU жив.") finally: + stop_heartbeat_thread() _stop_forwarder(server) diff --git a/src/gpu_rent/ux.py b/src/gpu_rent/ux.py index eaf15f6..11e4777 100644 --- a/src/gpu_rent/ux.py +++ b/src/gpu_rent/ux.py @@ -39,7 +39,9 @@ def cost_and_risk_lines(cfg: Config, *, spot: bool, flavor_name: str) -> list[st f"{cfg.idle_minutes} мин пустой очереди → delete compute. Отложить: gpu-rent hold", "preemptible: хостер может усыпить (~24 ч окно) → EXPIRED; tunnel сам unshelve, " "или gpu-rent up", - "Ctrl+C на tunnel GPU не гасит — только gpu-rent stop или idle-killer", + "Ctrl+C на tunnel GPU не гасит — только gpu-rent stop или idle-killer. " + "Опционально: gpu-rent watchdog install — аварийное закрытие окна/ребут " + "после grace тоже stop (Ctrl+C по-прежнему detach)", ] diff --git a/tests/test_cli.py b/tests/test_cli.py index b893a4d..6c0c094 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -14,7 +14,7 @@ def test_help(): def test_version(): result = runner.invoke(app, ["version"]) assert result.exit_code == 0 - assert "0.1.0" in result.stdout + assert "0.2.0" in result.stdout def test_up_nyi_after_missing_env(monkeypatch, tmp_path): diff --git a/tests/test_llm_runtime.py b/tests/test_llm_runtime.py new file mode 100644 index 0000000..9e84200 --- /dev/null +++ b/tests/test_llm_runtime.py @@ -0,0 +1,59 @@ +from pathlib import Path + +import pytest + +from gpu_rent.llm_runtime import ( + decide_runtime, + normalize_runtime, + parse_ollama_models, + write_ollama_models_preset, +) + + +def test_normalize_runtime(): + assert normalize_runtime(None) == "none" + assert normalize_runtime("OLLAMA") == "ollama" + assert normalize_runtime("llama-cpp") == "llamacpp" + with pytest.raises(ValueError): + normalize_runtime("foo") + + +def test_decide_runtime_flags_win(): + assert ( + decide_runtime(flag=None, ollama_flag=True, llamacpp_flag=False, from_config="none") + == "ollama" + ) + assert ( + decide_runtime(flag="llamacpp", ollama_flag=False, llamacpp_flag=False, from_config="ollama") + == "llamacpp" + ) + with pytest.raises(ValueError): + decide_runtime(flag=None, ollama_flag=True, llamacpp_flag=True, from_config="none") + + +def test_parse_ollama_models(tmp_path: Path): + path = tmp_path / "m.yaml" + path.write_text( + "models:\n - name: huihui_ai/qwen2.5-abliterate:7b\n default: true\n - qwen2.5:3b\n", + encoding="utf-8", + ) + entries = parse_ollama_models(path) + assert [e.name for e in entries] == [ + "huihui_ai/qwen2.5-abliterate:7b", + "qwen2.5:3b", + ] + assert entries[0].default is True + + +def test_parse_empty_manifest(tmp_path: Path): + path = tmp_path / "empty.yaml" + path.write_text("models: []\n", encoding="utf-8") + assert parse_ollama_models(path) == [] + assert parse_ollama_models(tmp_path / "missing.yaml") == [] + + +def test_write_preset(tmp_path: Path): + path = tmp_path / "out.yaml" + write_ollama_models_preset(path, "recommended") + entries = parse_ollama_models(path) + assert entries[0].name == "huihui_ai/qwen2.5-abliterate:7b" diff --git a/tests/test_local_watchdog.py b/tests/test_local_watchdog.py new file mode 100644 index 0000000..58dec32 --- /dev/null +++ b/tests/test_local_watchdog.py @@ -0,0 +1,87 @@ +from datetime import datetime, timedelta, timezone + +from gpu_rent.local_watchdog import LocalLease, decide_local_tick + + +def _now() -> datetime: + return datetime(2026, 8, 21, 12, 0, tzinfo=timezone.utc) + + +def test_noop_when_not_installed(): + d = decide_local_tick( + installed=False, + has_server=True, + lease=LocalLease(armed=True, heartbeat_at=_now().isoformat()), + now=_now(), + process_alive=False, + grace_sec=600, + ) + assert d.kind == "noop" + + +def test_noop_when_detached(): + d = decide_local_tick( + installed=True, + has_server=True, + lease=LocalLease(armed=False, detached=True, heartbeat_at=_now().isoformat()), + now=_now(), + process_alive=False, + grace_sec=600, + ) + assert d.kind == "noop" + assert "detached" in d.detail + + +def test_noop_while_pid_alive(): + d = decide_local_tick( + installed=True, + has_server=True, + lease=LocalLease( + armed=True, + pid=1, + heartbeat_at=(_now() - timedelta(hours=1)).isoformat(), + ), + now=_now(), + process_alive=True, + grace_sec=600, + ) + assert d.kind == "noop" + + +def test_noop_inside_grace(): + hb = _now() - timedelta(minutes=5) + d = decide_local_tick( + installed=True, + has_server=True, + lease=LocalLease(armed=True, pid=999, heartbeat_at=hb.isoformat()), + now=_now(), + process_alive=False, + grace_sec=600, + ) + assert d.kind == "noop" + assert "grace" in d.detail + + +def test_stop_when_stale(): + hb = _now() - timedelta(minutes=20) + d = decide_local_tick( + installed=True, + has_server=True, + lease=LocalLease(armed=True, pid=999, heartbeat_at=hb.isoformat()), + now=_now(), + process_alive=False, + grace_sec=600, + ) + assert d.kind == "stop" + + +def test_noop_without_lease(): + d = decide_local_tick( + installed=True, + has_server=True, + lease=None, + now=_now(), + process_alive=False, + grace_sec=600, + ) + assert d.kind == "noop" diff --git a/tests/test_seed.py b/tests/test_seed.py index cd2d0f6..8cf49a7 100644 --- a/tests/test_seed.py +++ b/tests/test_seed.py @@ -57,3 +57,35 @@ def test_git_token_injection(): assert strip_auth(with_token(url, "secret")) == url assert is_sha("a" * 40) assert not is_sha("main") + + +def test_scrub_origin(tmp_path: Path, monkeypatch): + from gpu_rent.remote import clone_ext + + dest = tmp_path / "repo" + dest.mkdir() + (dest / ".git").mkdir() + calls: list[list[str]] = [] + + def fake_out(argv, cwd=None): + if "get-url" in argv: + return "https://x-access-token:secret@github.com/org/Ext.git" + raise AssertionError(argv) + + def fake_run(argv, cwd=None): + calls.append(argv) + + monkeypatch.setattr(clone_ext, "out", fake_out) + monkeypatch.setattr(clone_ext, "run", fake_run) + clone_ext.scrub_origin(dest, "https://github.com/org/Ext.git") + assert calls == [ + [ + "git", + "-C", + str(dest), + "remote", + "set-url", + "origin", + "https://github.com/org/Ext.git", + ] + ] \ No newline at end of file diff --git a/tests/test_session.py b/tests/test_session.py index b6824ca..4b15720 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -30,6 +30,32 @@ def _cfg(monkeypatch): return load_config(require_auth=True) +def _mock_bind(monkeypatch): + monkeypatch.setattr( + "gpu_rent.session.ensure_floating_ip", + lambda conn, server, existing_id, existing_addr, log: ("203.0.113.9", "fip1"), + ) + monkeypatch.setattr("gpu_rent.session.wait_ssh", lambda cfg, host, timeout=900.0: None) + monkeypatch.setattr( + "gpu_rent.session.run_ssh", + lambda cfg, host, command, **kw: ( + "yes" if "gpu-rent-bootstrapped" in command else "inactive" + ), + ) + monkeypatch.setattr( + "gpu_rent.session.run_bootstrap", + lambda cfg, host, log, update=True, light=False: None, + ) + monkeypatch.setattr("gpu_rent.session.provision_vm", lambda cfg, host, log, **kw: None) + monkeypatch.setattr("gpu_rent.session.wait_backend_idle", lambda cfg, host, log, **kw: None) + monkeypatch.setattr( + "gpu_rent.session.ensure_boot_snapshot", + lambda conn, boot_volume_id, cfg, log: None, + ) + monkeypatch.setattr("gpu_rent.session.notify_ready", lambda cfg, log: None) + monkeypatch.setattr("gpu_rent.session.print_mcp_snippet", lambda cfg, log: None) + + def test_cmd_up_refuses_zero_gpu_quota(monkeypatch): monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 0}) @@ -42,24 +68,19 @@ def test_cmd_up_does_not_create_second_gpu(monkeypatch): monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) monkeypatch.setattr("gpu_rent.session.compute_quotas", lambda conn: {"gpu": 1}) monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server()) - monkeypatch.setattr( - "gpu_rent.session.ensure_floating_ip", - lambda conn, server, existing_id, existing_addr, log: ("203.0.113.9", "fip1"), - ) - monkeypatch.setattr("gpu_rent.session.wait_ssh", lambda cfg, host, timeout=420.0: None) - monkeypatch.setattr("gpu_rent.session.run_bootstrap", lambda cfg, host, log: None) - monkeypatch.setattr("gpu_rent.session.provision_vm", lambda cfg, host, log, **kw: None) - monkeypatch.setattr("gpu_rent.session.wait_backend_idle", lambda cfg, host, log, **kw: None) - monkeypatch.setattr( - "gpu_rent.session.ensure_boot_snapshot", - lambda conn, boot_volume_id, cfg, log: None, - ) - monkeypatch.setattr("gpu_rent.session.notify_ready", lambda cfg, log: None) - monkeypatch.setattr("gpu_rent.session.print_mcp_snippet", lambda cfg, log: None) + _mock_bind(monkeypatch) monkeypatch.setattr( "gpu_rent.session.create_gpu_server", lambda *a, **k: created.append("created") or Server(), ) + save_state( + SessionState( + server_id="s1", + floating_ip="203.0.113.9", + bootstrapped=True, + phase="ready_cloud", + ) + ) state = cmd_up(_cfg(monkeypatch), yes=True) assert created == [] assert state.server_id == "s1" @@ -80,20 +101,7 @@ def test_cmd_up_unshelves_expired(monkeypatch): "gpu_rent.session.unshelve", lambda conn, server, log: unshelved.append(server.id) or Server(status="ACTIVE"), ) - monkeypatch.setattr( - "gpu_rent.session.ensure_floating_ip", - lambda conn, server, existing_id, existing_addr, log: ("203.0.113.9", "fip1"), - ) - monkeypatch.setattr("gpu_rent.session.wait_ssh", lambda cfg, host, timeout=420.0: None) - monkeypatch.setattr("gpu_rent.session.run_bootstrap", lambda cfg, host, log: None) - monkeypatch.setattr("gpu_rent.session.provision_vm", lambda cfg, host, log, **kw: None) - monkeypatch.setattr("gpu_rent.session.wait_backend_idle", lambda cfg, host, log, **kw: None) - monkeypatch.setattr( - "gpu_rent.session.ensure_boot_snapshot", - lambda conn, boot_volume_id, cfg, log: None, - ) - monkeypatch.setattr("gpu_rent.session.notify_ready", lambda cfg, log: None) - monkeypatch.setattr("gpu_rent.session.print_mcp_snippet", lambda cfg, log: None) + _mock_bind(monkeypatch) monkeypatch.setattr( "gpu_rent.session.create_gpu_server", lambda *a, **k: created.append("created"), @@ -108,8 +116,6 @@ def test_cmd_up_unshelves_expired(monkeypatch): def test_cmd_stop_deletes_compute_keeps_disks(monkeypatch): deleted = [] - monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: object()) - monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server()) monkeypatch.setattr( "gpu_rent.session.delete_server", lambda conn, server, log: deleted.append(server.id), @@ -119,7 +125,13 @@ def test_cmd_stop_deletes_compute_keeps_disks(monkeypatch): lambda conn, fip_id, address, log: deleted.append("fip"), ) save_state( - SessionState(server_id="s1", boot_volume_id="b1", data_volume_id="d1", floating_ip="1.1.1.1") + SessionState( + server_id="s1", + boot_volume_id="b1", + data_volume_id="d1", + floating_ip="1.1.1.1", + bootstrapped=True, + ) ) class Conn: @@ -129,10 +141,12 @@ def test_cmd_stop_deletes_compute_keeps_disks(monkeypatch): return Server(server_id=sid) monkeypatch.setattr("gpu_rent.session.connect", lambda cfg: Conn()) + monkeypatch.setattr("gpu_rent.session.pick_existing_server", lambda conn: Server()) state = cmd_stop(_cfg(monkeypatch)) assert "s1" in deleted assert state.phase == "idle" assert state.server_id is None + assert state.bootstrapped is False assert state.boot_volume_id == "b1" assert state.data_volume_id == "d1" diff --git a/tests/test_tunnel_watch.py b/tests/test_tunnel_watch.py index 6446f95..1dcbf00 100644 --- a/tests/test_tunnel_watch.py +++ b/tests/test_tunnel_watch.py @@ -1,4 +1,4 @@ -from gpu_rent.tunnel import decide_watch +from gpu_rent.tunnel import decide_watch, tunnel_forwards def test_decide_ok_active(): @@ -25,3 +25,25 @@ def test_decide_exit_error(): def test_decide_exit_missing(): d = decide_watch(None, tunnel_alive=False) assert d.kind == "exit" + + +def test_tunnel_forwards_swarm_only(monkeypatch): + class Cfg: + swarmui_local_port = 17801 + llm_runtime = "none" + ollama_local_port = 17811 + llamacpp_local_port = 17812 + + monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})()) + assert tunnel_forwards(Cfg()) == [(17801, 7801)] + + +def test_tunnel_forwards_ollama(monkeypatch): + class Cfg: + swarmui_local_port = 17801 + llm_runtime = "ollama" + ollama_local_port = 17811 + llamacpp_local_port = 17812 + + monkeypatch.setattr("gpu_rent.tunnel.load_state", lambda: type("S", (), {"notes": {}})()) + assert tunnel_forwards(Cfg()) == [(17801, 7801), (17811, 11434)]