From 07bb521b7040d341d91344b63577f67e5055a476 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Sun, 23 Aug 2026 22:56:07 +0300 Subject: [PATCH] Add Assistent reference books and remove training dataset output. Introduce books/ with civitai-krea2 and HF fictext builders, sha-diff seed to VM, and drop train.jsonl from the civitai scrape pipeline. Co-authored-by: Cursor --- .gitignore | 7 +- books-dataset.bat | 4 + books-dataset.ps1 | 2 + books-dataset.sh | 4 + books.example/civitai-krea2/book.yaml | 14 + books/.gitkeep | 0 books/README.md | 27 ++ datasets/README.md | 20 +- docs/architecture.md | 5 +- docs/cli.md | 7 +- docs/decisions.md | 2 +- docs/extensions.md | 2 +- docs/knowledge-books.md | 47 +++ docs/llm.md | 2 +- docs/local-folders.md | 3 +- docs/roadmap.md | 2 +- pyproject.toml | 1 + src/gpu_rent/books.py | 458 ++++++++++++++++++++++++ src/gpu_rent/civitai_dataset.py | 38 +- src/gpu_rent/cli.py | 14 + src/gpu_rent/debug_assistent_session.py | 10 + src/gpu_rent/paths.py | 8 + src/gpu_rent/provision.py | 74 +++- tests/test_assistent_personas_seed.py | 4 +- tests/test_books.py | 129 +++++++ tests/test_civitai_dataset.py | 14 +- tmp_smoke.sh | 2 +- 27 files changed, 836 insertions(+), 64 deletions(-) create mode 100644 books-dataset.bat create mode 100644 books-dataset.ps1 create mode 100644 books-dataset.sh create mode 100644 books.example/civitai-krea2/book.yaml create mode 100644 books/.gitkeep create mode 100644 books/README.md create mode 100644 docs/knowledge-books.md create mode 100644 src/gpu_rent/books.py create mode 100644 tests/test_books.py diff --git a/.gitignore b/.gitignore index 06a1a3d..976638f 100644 --- a/.gitignore +++ b/.gitignore @@ -29,13 +29,18 @@ Output/** !Output/**/.gitkeep !Output/README.md -# Civitai Krea2 scrape (meta only — train.jsonl / search.jsonl) +# Civitai Krea2 scrape + Assistent books (gitignore data) datasets/** !datasets/.gitkeep !datasets/README.md !datasets/**/.gitkeep !datasets/**/README.md +books/** +!books/.gitkeep +!books/README.md +!books.example/** + # Python .venv/ __pycache__/ diff --git a/books-dataset.bat b/books-dataset.bat new file mode 100644 index 0000000..42b2bf0 --- /dev/null +++ b/books-dataset.bat @@ -0,0 +1,4 @@ +@echo off +setlocal +cd /d "%~dp0" +python -m gpu_rent.books %* diff --git a/books-dataset.ps1 b/books-dataset.ps1 new file mode 100644 index 0000000..cfe3575 --- /dev/null +++ b/books-dataset.ps1 @@ -0,0 +1,2 @@ +Set-Location $PSScriptRoot +python -m gpu_rent.books @args diff --git a/books-dataset.sh b/books-dataset.sh new file mode 100644 index 0000000..dc5b347 --- /dev/null +++ b/books-dataset.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")" +exec python -m gpu_rent.books "$@" diff --git a/books.example/civitai-krea2/book.yaml b/books.example/civitai-krea2/book.yaml new file mode 100644 index 0000000..00f68ca --- /dev/null +++ b/books.example/civitai-krea2/book.yaml @@ -0,0 +1,14 @@ +kind: book +id: civitai-krea2 +title: Civitai Krea2 +description: >- + Примеры промптов, негативов, steps/cfg и LoRA для Krea 2 с Civitai. + Ищи сюда, когда нужен рабочий prompt или параметры генерации. +content_kind: prompt-examples +language: en +tags: + - civitai + - krea2 + - prompts +source: civitai-dataset +license: civitai-terms diff --git a/books/.gitkeep b/books/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/books/README.md b/books/README.md new file mode 100644 index 0000000..e2949ae --- /dev/null +++ b/books/README.md @@ -0,0 +1,27 @@ +# Assistent reference books (local) + +FTS `search.jsonl` indexes for Swarm Assistent Knowledge Hub. **Not** in Typer main CLI — use: + +```text +books-dataset.bat # python -m gpu_rent.books +python -m gpu_rent.books list +python -m gpu_rent.books build civitai +python -m gpu_rent.books download fictext --variant both +gpu-rent seed-books # push changed books → VM +``` + +## Layout + +| Path | Role | +| --- | --- | +| `civitai-krea2/` | From `civitai-dataset split` → `book.yaml` + `search.jsonl` + `meta.json` | +| `ru-fictext-rplus/` | HF `ru-fictext-nsfw-data-r+.parquet` → chunked search rows | +| `ru-fictext-full/` | HF full parquet | + +On `up` / `seed-personas` / `seed-books`: changed books (by `meta.content_sha`) → `/mnt/swarm_data/Assistent/books//`. + +Civitai scrape: manual on laptop; `up` pushes when local book changed. + +Fictext: `pip install 'gpu-rent[books]'` (pyarrow) for build. + +See [docs/local-folders.md](../docs/local-folders.md), [datasets/README.md](../datasets/README.md). diff --git a/datasets/README.md b/datasets/README.md index 6dfaff3..10d0add 100644 --- a/datasets/README.md +++ b/datasets/README.md @@ -1,13 +1,15 @@ # Civitai Krea2 datasets (local) -Scraped generation metadata for Assistent training + cheap FTS search. **No image files.** +Scraped generation metadata for Assistent **books** (FTS search). **No image files.** UI training (`train.jsonl`) removed. -Not part of `gpu-rent` Typer CLI and **not** `models.yaml` / `gpu-rent push`. Separate entrypoint: +Not part of `gpu-rent` Typer CLI and **not** `models.yaml` / `gpu-rent push`. Separate entrypoints: ```text civitai-dataset.bat # Windows (or .\civitai-dataset.ps1 / ./civitai-dataset.sh) -# default = all → discover + scrape + split +# default = all → discover + scrape + split (+ books/civitai-krea2) python -m gpu_rent.civitai_dataset all + +books-dataset.bat # python -m gpu_rent.books download fictext … ``` Needs `CIVITAI_API_TOKEN` and preferably `CIVITAI_API_HOST=civitai.red` in `.env`. @@ -18,7 +20,7 @@ Needs `CIVITAI_API_TOKEN` and preferably `CIVITAI_API_HOST=civitai.red` in `.env | --- | --- | --- | | `discover` | `--top-checkpoints` (20), `--top-loras` (30) | Popular Krea2 ckpt/LoRAs (+ versions from `models.yaml`) → `catalog/models.json` | | `scrape` | `--target` (2000), `--per-version`, `--min-score` | Gallery metadata only; resume by id; NSFW→rating; skip minor tags | -| `split` | — | `by_kind/` / `by_rating/` / `train.jsonl` / `search.jsonl` | +| `split` | — | `by_kind/` / `by_rating/` / `search.jsonl` + `books/civitai-krea2/` | | `all` | same as discover+scrape | discover → scrape → split (launcher default) | Global: `--out` = app root (writes under `datasets/civitai`). @@ -31,11 +33,9 @@ Global: `--out` = app root (writes under `datasets/civitai`). | `civitai/catalog/images.jsonl` | Canonical rows (resume by id) | | `civitai/by_kind/` | checkpoint vs lora | | `civitai/by_rating/` | pg / pg13 / r / x | -| `civitai/train.jsonl` | Alpaca instruction/output for Assistent Обучение | -| `civitai/search.jsonl` | Compact FTS index → pushed to VM as `Assistent/civitai-examples.jsonl` on `up` / `seed-personas` | +| `civitai/search.jsonl` | Compact Civitai FTS source → `books/civitai-krea2` on split | +| `../books/civitai-krea2/` | Assistent book (`book.yaml`, `search.jsonl`, `meta.json`) | -Push of `search.jsonl` is skipped if the file is missing. Weights stay in [models.md](../docs/models.md) (Civitai seed / `Models/` / `capture`). +On `up` / `seed-personas` / `seed-books`: changed books push → `/mnt/swarm_data/Assistent/books/` (sha diff). Legacy `civitai-examples.jsonl` only if `books/` empty. -Git ignores the data; this README stays. - -See also: [docs/cli.md](../docs/cli.md) (civitai-dataset block), [docs/extensions.md](../docs/extensions.md), [docs/local-folders.md](../docs/local-folders.md). +See [books/README.md](../books/README.md), [docs/cli.md](../docs/cli.md), [docs/local-folders.md](../docs/local-folders.md). diff --git a/docs/architecture.md b/docs/architecture.md index 359dc6c..92a4097 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,8 +44,9 @@ | `inventory` / `ux` / `placement` / `flavor_presets` | Flavors, квоты, AZ; preview / ServerPlan | | `session` | `cmd_up` / `cmd_stop` / adopt | | `bootstrap` + `remote/bootstrap.sh` | Идемпотентный first-boot; light без apt | -| `provision` | extensions, autocomplete, civitai seed, LLM, push, idle-killer arm, start SwarmUI; seed `search.jsonl` → `Assistent/civitai-examples.jsonl` | -| `civitai` / `civitai_dataset` | Civitai HTTP API; локальный scrape Krea2 → train/search jsonl (отдельный CLI) | +| `provision` | extensions, autocomplete, books seed, LLM, push, idle-killer arm, start SwarmUI; `seed_books()` → `Assistent/books/` (sha diff) | +| `civitai` / `civitai_dataset` | Civitai HTTP API; локальный scrape Krea2 → search jsonl + book `books/civitai-krea2` | +| `books` | HF fictext + civitai book build; `meta.content_sha`; отдельный CLI + `seed-books` | | `huggingface` | HF probe / метадата / URL для seed и capture | | `capture` | Инвентарь VM → merge ссылок в локальные манифесты | | `doctor` | Preflight без mutating compute | diff --git a/docs/cli.md b/docs/cli.md index bd5b613..55a533a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -79,7 +79,8 @@ gpu-rent up --yes --ollama | `gpu-rent ssh` | Оболочка на VM | | `gpu-rent seed-models` | Докачать новые строки Civitai-манифеста на живой диск | | `gpu-rent seed-extensions` | Доклонировать/обновить git-репы; restart swarmui | -| `gpu-rent seed-personas` | Пуш `assistent-extensions/` → `Assistent/extensions/` (+ `_base`); также `search.jsonl` → FTS, если есть | +| `gpu-rent seed-personas` | Пуш `assistent-extensions/` → `Assistent/extensions/` (+ `_base`); также `seed-books` если есть `books/` | +| `gpu-rent seed-books` | Push изменённых `books/*/` → `Assistent/books/` (`--force` — все) | | `gpu-rent push` / `push-models` | Локальные деревья → VM | | `gpu-rent pull-output` | VM `Output/` → `./Output` | | `gpu-rent capture` / `capture all` | Инвентарь VM → merge **ссылок** в `models.yaml` + `extensions.yaml` (веса не качать) | @@ -102,10 +103,10 @@ gpu-rent up --yes --ollama | --- | --- | | `discover` | Топ Krea2 checkpoints/LoRAs → `catalog/models.json` | | `scrape` | Галереи (метаданные, **без** картинок); `--target`, `--per-version`, `--min-score` | -| `split` | `by_kind/` / `by_rating/` / `train.jsonl` / `search.jsonl` | +| `split` | `by_kind/` / `by_rating/` / `search.jsonl` + `books/civitai-krea2/` | | `all` | discover + scrape + split | -Нужен `CIVITAI_API_TOKEN` (лучше `CIVITAI_API_HOST=civitai.red`). На `up` / `seed-personas` файл `datasets/civitai/search.jsonl` пушится на VM как `Assistent/civitai-examples.jsonl` (если есть). +Нужен `CIVITAI_API_TOKEN` (лучше `CIVITAI_API_HOST=civitai.red`). Книги: `books-dataset.bat` / `python -m gpu_rent.books download fictext --variant both` (pyarrow: `pip install 'gpu-rent[books]'`). На `up` / `seed-books` изменённые `books/*/` → `Assistent/books/`. --- diff --git a/docs/decisions.md b/docs/decisions.md index 68d133d..7aece05 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -62,4 +62,4 @@ | LLM runtime | Только `none` \| `ollama`. **llama.cpp снят** из продукта; leftover unit `gpu-rent-llamacpp` на `up` стопается | | Debug API | Принятая поверхность: localhost `:17821` на `up`/`tunnel` / `gpu-rent debug`; Assistent diagnose, session, chat-eval | | Assistent | First-class поверх SwarmUI + Ollama (чат, Cards, personas), не только «prompt-help» | -| Civitai dataset | Локальный scrape метаданных Krea2 (без картинок) → `train.jsonl` / `search.jsonl`; на `up`/`seed-personas` `search.jsonl` → `Assistent/civitai-examples.jsonl` (FTS) | +| Civitai dataset | Локальный scrape Krea2 (без картинок) → `search.jsonl` + book `books/civitai-krea2/`; на `up`/`seed-books` push при изменении `meta.content_sha` → `Assistent/books/` (FTS). Legacy `civitai-examples.jsonl` если books нет | diff --git a/docs/extensions.md b/docs/extensions.md index 68275a9..762708e 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -70,7 +70,7 @@ Bundled в swarm-assistent: `neutral` (Нормальный), `aggressive`, `dre Формат пака: `assistent-pack.yaml` (`kind: persona`, `id: …`) + полки JSON в корне (или `personas//` для мульти-пака). На `up` также пишется `_base/assistant.json` (`default_persona: neutral`, опционально `num_ctx` с GPU tier). Seed **не** удаляет overlay-личности, созданные в UI. Legacy `assistent-personas/` / `personas.json` больше не сидятся. -Если есть `datasets/civitai/search.jsonl`, на том же шаге пушится FTS-индекс → `/mnt/swarm_data/Assistent/civitai-examples.jsonl` ([datasets/README](../datasets/README.md)). Диагностика Assistent: Debug API `/assistent/*` ([cli.md](cli.md)). +Если есть `books/` (или legacy `datasets/civitai/search.jsonl`), на том же шаге `seed-books` пушит FTS → `/mnt/swarm_data/Assistent/books/` ([books/README](../books/README.md), [knowledge-books.md](knowledge-books.md)). Диагностика Assistent: Debug API `/assistent/*` ([cli.md](cli.md)). В git репозитория gpu-rent не коммитить рабочий список с лишними приватными URL сверх примера. diff --git a/docs/knowledge-books.md b/docs/knowledge-books.md new file mode 100644 index 0000000..13a31a5 --- /dev/null +++ b/docs/knowledge-books.md @@ -0,0 +1,47 @@ +# Assistent Knowledge Hub (swarm-assistent) + +gpu-rent builds and seeds **books** locally. Runtime lives in [swarm-assistent](https://gitea.hsrv.site/mrleo1nid/swarm-assistent.git). + +## VM layout + +```text +/mnt/swarm_data/Assistent/books// + book.yaml # title, description, content_kind + search.jsonl # FTS rows (unified contract) + meta.json # row_count, content_sha, built_at + .gpu-rent-meta.json # same sha for gpu-rent skip/push +``` + +## Unified search row + +```json +{ + "id": "civitai-krea2:42", + "book": "civitai-krea2", + "title": "", + "tags": ["woman"], + "text": "excerpt ≤500 chars", + "body": "longer chunk", + "rating": "pg13", + "meta": {} +} +``` + +## swarm-assistent tasks (not in gpu-rent) + +1. **`Knowledge/IKnowledgeSource.cs`** + adapters: Book, MemorySeed, MemoryPersonal, Tag, Session +2. **`AssistentKnowledgeHub`**: ListCatalog, Search, Upsert, Forget, RunPipeline, BuildSystemLayer +3. **Remove training UI**: Обучение tab, AssistentTrainingJobs, AssistentListTrainSamples, taste APIs +4. **HTTP** `AssistentChat` → `knowledge: {catalog, hops, results}`; legacy `civitai_results` shim +5. **Persona UI** «Знания»: attach toggles; bundled personas default attach civitai-krea2 + ru-fictext-rplus +6. **Skill `knowledge`**: list / search / upsert / forget + +## Default attach (bundled only) + +- `neutral`, `aggressive`, `dreamer`: civitai-krea2 + ru-fictext-rplus +- UI-created personas: no auto-attach +- Pack `leonid`: from `assistent-pack.yaml` `knowledge.attach` + +## Index on VM + +On book push or sha change: reindex `search.jsonl` → sqlite FTS `books_fts`. Progress in UI on first chat. diff --git a/docs/llm.md b/docs/llm.md index 8f22eff..6b68ad7 100644 --- a/docs/llm.md +++ b/docs/llm.md @@ -113,7 +113,7 @@ Unit `gpu-rent-ollama` читает `/mnt/swarm_data/.gpu-rent-gpu.json`: \*Flash на Ampere+ (compute ≥ 8.0). `NUM_PARALLEL=2`, `MAX_LOADED_MODELS=2` (chat VL + memory embed). Memory models use `use: memory` and a CPU Modelfile (`num_gpu 0`) so embed does not steal VRAM from the chat model. Ollama default `num_ctx` is 4096; we set `OLLAMA_CONTEXT_LENGTH` so Assistent + vision fits. Mid/high keep-alive is **5m** because Assistent parks the chat model before Generate (`AssistentParkLlm`) and warms after. Env: `/mnt/swarm_data/.gpu-rent-ollama.env`. -Personas: bundled `neutral` / `aggressive` / `dreamer` in swarm-assistent; extra packs via `extensions.yaml` `assistent:` or laptop `assistent-extensions/` → `Assistent/extensions/` on `up` / `seed-personas` / `push` (+ `_base/assistant.json` with `default_persona` / `num_ctx`). See [extensions.md](extensions.md). Optional FTS examples: [datasets/README](../datasets/README.md) (`search.jsonl` → `Assistent/civitai-examples.jsonl`). Journal: `gpu-rent logs -u ollama`. +Personas: bundled `neutral` / `aggressive` / `dreamer` in swarm-assistent; extra packs via `extensions.yaml` `assistent:` or laptop `assistent-extensions/` → `Assistent/extensions/` on `up` / `seed-personas` / `push` (+ `_base/assistant.json` with `default_persona` / `num_ctx`). See [extensions.md](extensions.md). Reference books: [books/README](../books/README.md) → `Assistent/books/` on `seed-books` ([knowledge-books.md](knowledge-books.md)). Journal: `gpu-rent logs -u ollama`. `ollama-models.yaml` entries: diff --git a/docs/local-folders.md b/docs/local-folders.md index d0f3cb2..d485f71 100644 --- a/docs/local-folders.md +++ b/docs/local-folders.md @@ -26,7 +26,8 @@ | Локально | На VM | Когда | | --- | --- | --- | | `assistent-extensions//` | `/mnt/swarm_data/Assistent/extensions//` | `up` / `seed-personas` / `push` (без `.git`); см. [extensions.md](extensions.md) | -| `datasets/civitai/search.jsonl` | `Assistent/civitai-examples.jsonl` | `up` / `seed-personas`, если файл есть; см. [datasets/README](../datasets/README.md) | +| `datasets/civitai/search.jsonl` | (local build) → `books/civitai-krea2/` | civitai-dataset split | +| `books//` | `Assistent/books//` | `up` / `seed-personas` / `seed-books` if `meta.content_sha` changed; see [books/README](../books/README.md) | ## Push (Models / Wildcards / CustomWorkflows / packs) diff --git a/docs/roadmap.md b/docs/roadmap.md index 491f169..bef9776 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -90,7 +90,7 @@ ## Уже в коде после v1 - Debug API на `127.0.0.1:17821` (`gpu-rent debug` / sidecar на `up`/`tunnel`); Assistent session / chat-eval -- `civitai-dataset` → `datasets/civitai/search.jsonl` → FTS push на `up` / `seed-personas` +- `civitai-dataset` → `books/civitai-krea2/`; `gpu-rent books download fictext`; `seed-books` на `up` / `seed-personas` (sha diff) - `seed-personas`, `capture wanted`, `logs --unit swarm|ollama|killer|cloud-init` - Balance notify (`SELECTEL_API_TOKEN` + watchdog) diff --git a/pyproject.toml b/pyproject.toml index 858d994..a63d2a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ [project.optional-dependencies] dev = ["pytest>=8.0"] +books = ["pyarrow>=15.0"] [project.scripts] gpu-rent = "gpu_rent.cli:app" diff --git a/src/gpu_rent/books.py b/src/gpu_rent/books.py new file mode 100644 index 0000000..9271046 --- /dev/null +++ b/src/gpu_rent/books.py @@ -0,0 +1,458 @@ +"""Assistent reference books: build locally, seed to VM on change (FTS search.jsonl).""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections.abc import Callable, Iterable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx +import yaml + +from gpu_rent.civitai_dataset import looks_minor, read_jsonl, write_jsonl +from gpu_rent.config import load_config +from gpu_rent.errors import GpuRentError +from gpu_rent.paths import app_root + +Log = Callable[[str], None] + +BOOK_CIVITAI = "civitai-krea2" +BOOK_FICTEXT_RPLUS = "ru-fictext-rplus" +BOOK_FICTEXT_FULL = "ru-fictext-full" + +VM_BOOKS_PREFIX = "/mnt/swarm_data/Assistent/books" +SIDEcar_NAME = ".gpu-rent-meta.json" + +HF_REPO = "krplt/ru-fictext-nsfw" +HF_FILES = { + BOOK_FICTEXT_RPLUS: "ru-fictext-nsfw-data-r+.parquet", + BOOK_FICTEXT_FULL: "ru-fictext-nsfw-data.parquet", +} + +CHUNK_CHARS = 500 +CHUNK_OVERLAP = 80 +EXCERPT_MAX = 500 + +BOOK_SPECS: dict[str, dict[str, Any]] = { + BOOK_CIVITAI: { + "kind": "book", + "id": BOOK_CIVITAI, + "title": "Civitai Krea2", + "description": ( + "Примеры промптов, негативов, steps/cfg и LoRA для Krea 2 с Civitai. " + "Ищи сюда, когда нужен рабочий prompt или параметры генерации." + ), + "content_kind": "prompt-examples", + "language": "en", + "tags": ["civitai", "krea2", "prompts"], + "source": "civitai-dataset", + "license": "civitai-terms", + }, + BOOK_FICTEXT_RPLUS: { + "kind": "book", + "id": BOOK_FICTEXT_RPLUS, + "title": "RU ficbook R+", + "description": ( + "Отрывки русской фанфикшн (mature/NSFW) для стиля, лексики и тона сцен. " + "Не цитируй дословно длинно — бери формулировки и ритм." + ), + "content_kind": "prose-style", + "language": "ru", + "tags": ["fanfiction", "nsfw", "russian"], + "source": f"huggingface.co/datasets/{HF_REPO}", + "license": "cc-by-4.0", + }, + BOOK_FICTEXT_FULL: { + "kind": "book", + "id": BOOK_FICTEXT_FULL, + "title": "RU ficbook (full)", + "description": ( + "Отрывки русской фанфикшн (NSFW + safe) для стиля и лексики. " + "Шире чем R+; для нейтральных сцен предпочитай safe-теги." + ), + "content_kind": "prose-style", + "language": "ru", + "tags": ["fanfiction", "russian"], + "source": f"huggingface.co/datasets/{HF_REPO}", + "license": "cc-by-4.0", + }, +} + + +def _log(msg: str) -> None: + print(msg, flush=True) + + +def books_root(root: Path | None = None) -> Path: + return (root or app_root()) / "books" + + +def book_dir(book_id: str, root: Path | None = None) -> Path: + return books_root(root) / book_id + + +def write_book_yaml(book_id: str, root: Path | None = None) -> Path: + spec = BOOK_SPECS.get(book_id) + if not spec: + raise GpuRentError(f"Unknown book id: {book_id}") + path = book_dir(book_id, root) / "book.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(spec, allow_unicode=True, sort_keys=False), encoding="utf-8") + return path + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def write_meta( + book_id: str, + *, + row_count: int, + root: Path | None = None, + source_sha: str | None = None, + extra: dict[str, Any] | None = None, +) -> Path: + bdir = book_dir(book_id, root) + search = bdir / "search.jsonl" + if not search.is_file(): + raise GpuRentError(f"Нет {search} для meta") + payload: dict[str, Any] = { + "book_id": book_id, + "row_count": row_count, + "content_sha": sha256_file(search), + "built_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"), + "search_bytes": search.stat().st_size, + } + if source_sha: + payload["source_sha"] = source_sha + if extra: + payload.update(extra) + path = bdir / "meta.json" + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return path + + +def read_meta(book_id: str, root: Path | None = None) -> dict[str, Any] | None: + path = book_dir(book_id, root) / "meta.json" + if not path.is_file(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + return data if isinstance(data, dict) else None + + +def iter_installed_books(root: Path | None = None) -> list[str]: + root_p = books_root(root) + if not root_p.is_dir(): + return [] + out: list[str] = [] + for child in sorted(root_p.iterdir()): + if not child.is_dir(): + continue + if (child / "search.jsonl").is_file(): + out.append(child.name) + return out + + +def civitai_search_to_book_row(row: dict[str, Any], book_id: str = BOOK_CIVITAI) -> dict[str, Any]: + prompt = str(row.get("prompt") or "").strip() + neg = str(row.get("negative") or row.get("negativePrompt") or "").strip() + parts = [prompt] if prompt else [] + if neg: + parts.append(f"Negative: {neg}") + params = row.get("params") if isinstance(row.get("params"), dict) else {} + for key, label in (("steps", "steps"), ("cfg", "cfg"), ("cfgScale", "cfg"), ("sampler", "sampler")): + if params.get(key) is not None: + parts.append(f"{label}: {params[key]}") + text = "\n".join(parts)[:EXCERPT_MAX] + meta = { + "prompt": prompt, + "negative": neg, + "params": params, + "loras": list(row.get("loras") or []), + "modelVersionId": row.get("modelVersionId"), + "kind": row.get("kind"), + "score": row.get("score"), + } + rid = row.get("id") + return { + "id": f"{book_id}:{rid}", + "book": book_id, + "title": "", + "tags": list(row.get("tags") or []), + "text": text, + "body": prompt[:2000] if prompt else text, + "rating": row.get("rating") or "pg", + "meta": meta, + } + + +def _chunk_text(text: str, *, chunk_chars: int = CHUNK_CHARS, overlap: int = CHUNK_OVERLAP) -> list[str]: + text = re.sub(r"\r\n?", "\n", text.strip()) + if not text: + return [] + paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()] + chunks: list[str] = [] + for para in paras: + if len(para) <= chunk_chars: + chunks.append(para) + continue + start = 0 + while start < len(para): + end = min(len(para), start + chunk_chars) + chunks.append(para[start:end]) + if end >= len(para): + break + start = max(0, end - overlap) + return chunks + + +def fictext_row_to_book_rows( + row: dict[str, Any], + book_id: str, + *, + row_index: int, +) -> list[dict[str, Any]]: + title = str(row.get("title") or "").strip() + tags_raw = row.get("tags") + tags: list[str] = [] + if isinstance(tags_raw, list): + tags = [str(t).strip() for t in tags_raw if str(t).strip()] + elif isinstance(tags_raw, str): + tags = [t.strip() for t in re.split(r"[,;]", tags_raw) if t.strip()] + text = str(row.get("text") or "").strip() + if not text or looks_minor(text, tags): + return [] + out: list[dict[str, Any]] = [] + for ci, chunk in enumerate(_chunk_text(text)): + excerpt = chunk[:EXCERPT_MAX] + out.append( + { + "id": f"{book_id}:{row_index}:{ci}", + "book": book_id, + "title": title, + "tags": tags[:20], + "text": excerpt, + "body": chunk[:2000], + "rating": "r" if book_id == BOOK_FICTEXT_RPLUS else "pg13", + "meta": {"source_row": row_index, "chunk": ci}, + } + ) + return out + + +def build_civitai_book(*, out_root: Path | None = None, log: Log = _log) -> dict[str, int]: + """Write books/civitai-krea2 from datasets/civitai/search.jsonl.""" + root = out_root or app_root() + src = root / "datasets" / "civitai" / "search.jsonl" + if not src.is_file(): + # fallback: build from catalog images via civitai_dataset split output path + raise GpuRentError(f"Нет {src} — сначала civitai-dataset split") + rows = read_jsonl(src) + book_rows = [civitai_search_to_book_row(r) for r in rows] + bdir = book_dir(BOOK_CIVITAI, root) + write_book_yaml(BOOK_CIVITAI, root) + n = write_jsonl(bdir / "search.jsonl", book_rows) + write_meta(BOOK_CIVITAI, row_count=n, root=root, extra={"source": str(src)}) + log(f"books/{BOOK_CIVITAI}: {n} rows → {bdir}") + return {"rows": n} + + +def _hf_download_url(filename: str) -> str: + return f"https://huggingface.co/datasets/{HF_REPO}/resolve/main/{filename}" + + +def _require_pyarrow(): + try: + import pyarrow.parquet as pq # noqa: F401 + + return pq + except ImportError as exc: + raise GpuRentError( + "Нужен pyarrow для fictext: pip install 'gpu-rent[books]' или pip install pyarrow" + ) from exc + + +def download_fictext_parquet( + book_id: str, + *, + out_root: Path | None = None, + log: Log = _log, + token: str | None = None, +) -> Path: + if book_id not in HF_FILES: + raise GpuRentError(f"Unknown fictext book: {book_id}") + filename = HF_FILES[book_id] + bdir = book_dir(book_id, out_root) + bdir.mkdir(parents=True, exist_ok=True) + dest = bdir / filename + url = _hf_download_url(filename) + headers: dict[str, str] = {} + if token: + headers["Authorization"] = f"Bearer {token}" + log(f"download: {url}") + with httpx.Client(follow_redirects=True, timeout=600.0) as client: + with client.stream("GET", url, headers=headers) as resp: + resp.raise_for_status() + with dest.open("wb") as fh: + for chunk in resp.iter_bytes(1024 * 1024): + fh.write(chunk) + log(f"saved {dest} ({dest.stat().st_size // 1024} KB)") + return dest + + +def build_fictext_book( + book_id: str, + *, + out_root: Path | None = None, + log: Log = _log, + max_rows: int | None = None, +) -> dict[str, int]: + if book_id not in HF_FILES: + raise GpuRentError(f"Unknown fictext book: {book_id}") + pq = _require_pyarrow() + bdir = book_dir(book_id, out_root) + parquet = bdir / HF_FILES[book_id] + if not parquet.is_file(): + raise GpuRentError(f"Нет {parquet} — сначала books download fictext") + write_book_yaml(book_id, out_root) + table = pq.read_table(parquet) + data = table.to_pydict() + titles = data.get("title") or [] + tags_col = data.get("tags") or [] + texts = data.get("text") or [] + n_src = len(texts) + book_rows: list[dict] = [] + for i in range(n_src): + if max_rows is not None and i >= max_rows: + break + row = { + "title": titles[i] if i < len(titles) else "", + "tags": tags_col[i] if i < len(tags_col) else [], + "text": texts[i] if i < len(texts) else "", + } + book_rows.extend(fictext_row_to_book_rows(row, book_id, row_index=i)) + n = write_jsonl(bdir / "search.jsonl", book_rows) + write_meta( + book_id, + row_count=n, + root=out_root, + source_sha=sha256_file(parquet), + extra={"source_parquet": parquet.name, "source_stories": n_src}, + ) + log(f"books/{book_id}: {n} chunks from {n_src} stories → {bdir}") + return {"rows": n, "stories": n_src} + + +def cmd_download_fictext( + *, + variant: str = "both", + out_root: Path | None = None, + log: Log = _log, +) -> None: + cfg = load_config() + token = (getattr(cfg, "hf_token", None) or "").strip() or None + ids = [] + if variant in {"rplus", "both"}: + ids.append(BOOK_FICTEXT_RPLUS) + if variant in {"full", "both"}: + ids.append(BOOK_FICTEXT_FULL) + for book_id in ids: + download_fictext_parquet(book_id, out_root=out_root, log=log, token=token) + build_fictext_book(book_id, out_root=out_root, log=log) + + +def remote_book_dir(book_id: str) -> str: + return f"{VM_BOOKS_PREFIX}/{book_id}" + + +def seed_book_files(local_book: Path, remote_book: str) -> list[tuple[Path, str]]: + """Local book dir files to push.""" + pairs: list[tuple[Path, str]] = [] + for name in ("book.yaml", "search.jsonl", "meta.json"): + local = local_book / name + if local.is_file(): + pairs.append((local, f"{remote_book}/{name}")) + meta = local_book / "meta.json" + if meta.is_file(): + pairs.append((meta, f"{remote_book}/{SIDEcar_NAME}")) + return pairs + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog="python -m gpu_rent.books", + description="Build Assistent reference books (search.jsonl + book.yaml)", + ) + p.add_argument("--out", type=Path, default=None, help="App root") + sub = p.add_subparsers(dest="cmd") + + d = sub.add_parser("download", help="Download HF fictext parquet(s)") + d.add_argument( + "target", + choices=["fictext"], + nargs="?", + default="fictext", + ) + d.add_argument("--variant", choices=["rplus", "full", "both"], default="both") + + b = sub.add_parser("build", help="Build book search.jsonl") + b.add_argument("target", choices=["civitai", "fictext", "all"]) + b.add_argument("--variant", choices=["rplus", "full", "both"], default="both") + + sub.add_parser("list", help="List local books with meta") + return p + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + parser = build_parser() + if not argv: + argv = ["list"] + args = parser.parse_args(argv) + out = args.out + try: + if args.cmd == "download": + if args.target == "fictext": + cmd_download_fictext(variant=args.variant, out_root=out) + elif args.cmd == "build": + if args.target in {"civitai", "all"}: + build_civitai_book(out_root=out) + if args.target in {"fictext", "all"}: + for book_id in ( + [BOOK_FICTEXT_RPLUS, BOOK_FICTEXT_FULL] + if args.variant == "both" + else [BOOK_FICTEXT_RPLUS if args.variant == "rplus" else BOOK_FICTEXT_FULL] + ): + build_fictext_book(book_id, out_root=out) + elif args.cmd == "list": + for bid in iter_installed_books(out): + meta = read_meta(bid, out) or {} + log( + f"{bid}: rows={meta.get('row_count', '?')} " + f"sha={str(meta.get('content_sha', ''))[:12]}" + ) + else: + parser.print_help() + return 0 + except GpuRentError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/gpu_rent/civitai_dataset.py b/src/gpu_rent/civitai_dataset.py index 1f1e37b..f9b6e95 100644 --- a/src/gpu_rent/civitai_dataset.py +++ b/src/gpu_rent/civitai_dataset.py @@ -1,4 +1,4 @@ -"""Local Civitai Krea2 gallery scrape → train.jsonl + search.jsonl (no image files).""" +"""Local Civitai Krea2 gallery scrape → search.jsonl + Assistent book (no image files).""" from __future__ import annotations @@ -522,28 +522,6 @@ def cmd_scrape( return have -def train_row(row: dict[str, Any]) -> dict[str, Any]: - tags = row.get("tags") or [] - tag_s = ", ".join(str(t) for t in tags[:12]) - rating = row.get("rating") or "pg" - instruction = f"Write a Krea 2 prompt.\nTags: {tag_s}\nRating: {rating}" - params = row.get("params") if isinstance(row.get("params"), dict) else {} - parts = [str(row.get("prompt") or "").strip()] - neg = str(row.get("negativePrompt") or "").strip() - if neg: - parts.append(f"Negative: {neg}") - for key, label in ( - ("steps", "steps"), - ("cfgScale", "cfg"), - ("sampler", "sampler"), - ("seed", "seed"), - ("size", "size"), - ): - if params.get(key) is not None: - parts.append(f"{label}: {params[key]}") - return {"instruction": instruction, "output": "\n".join(parts)} - - def search_row(row: dict[str, Any]) -> dict[str, Any]: resources = row.get("resources") if isinstance(row.get("resources"), list) else [] loras: list[dict[str, Any]] = [] @@ -600,14 +578,12 @@ def cmd_split(*, out_root: Path | None = None, log: Log = _log) -> dict[str, int by_kind: dict[str, list] = {"checkpoint": [], "lora": []} by_rating: dict[str, list] = {} - train: list[dict] = [] search: list[dict] = [] for row in rows: kind = str(row.get("kind") or "checkpoint") by_kind.setdefault(kind, []).append(row) rating = str(row.get("rating") or "pg") by_rating.setdefault(rating, []).append(row) - train.append(train_row(row)) search.append(search_row(row)) counts: dict[str, int] = {} @@ -618,13 +594,19 @@ def cmd_split(*, out_root: Path | None = None, log: Log = _log) -> dict[str, int safe = re.sub(r"[^a-z0-9]+", "", rating.lower()) or "pg" path = root / "by_rating" / f"{safe}.jsonl" counts[f"rating:{safe}"] = write_jsonl(path, items) - counts["train"] = write_jsonl(root / "train.jsonl", train) counts["search"] = write_jsonl(root / "search.jsonl", search) log( "split: " + ", ".join(f"{k}={v}" for k, v in sorted(counts.items())) + f" → {root}" ) + try: + from gpu_rent.books import build_civitai_book + + book_counts = build_civitai_book(out_root=out_root or app_root(), log=log) + counts["book:civitai-krea2"] = book_counts.get("rows", 0) + except GpuRentError as exc: + log(f"book build warn: {exc}") return counts @@ -657,7 +639,7 @@ def cmd_all( def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( prog="python -m gpu_rent.civitai_dataset", - description="Scrape Civitai Krea2 galleries → train.jsonl + search.jsonl", + description="Scrape Civitai Krea2 galleries → search.jsonl + books/civitai-krea2", ) p.add_argument( "--out", @@ -676,7 +658,7 @@ def build_parser() -> argparse.ArgumentParser: s.add_argument("--per-version", type=int, default=DEFAULT_PER_VERSION) s.add_argument("--min-score", type=int, default=DEFAULT_MIN_SCORE) - sub.add_parser("split", help="Write by_kind / by_rating / train / search") + sub.add_parser("split", help="Write by_kind / by_rating / search + civitai book") a = sub.add_parser("all", help="discover + scrape + split (default)") a.add_argument("--top-checkpoints", type=int, default=DEFAULT_TOP_CHECKPOINTS) diff --git a/src/gpu_rent/cli.py b/src/gpu_rent/cli.py index dd36576..f4d345b 100644 --- a/src/gpu_rent/cli.py +++ b/src/gpu_rent/cli.py @@ -1007,6 +1007,20 @@ def seed_extensions_cmd() -> None: _die(exc) +@app.command("seed-books") +def seed_books_cmd( + force: bool = typer.Option(False, "--force", help="Push all books even if sha unchanged"), +) -> None: + """Push books/*/ → VM Assistent/books/ (changed only unless --force).""" + try: + from gpu_rent.provision import seed_books + + cfg, host = _live() + seed_books(cfg, host, log, force=force) + except GpuRentError as exc: + _die(exc) + + @app.command("seed-personas") def seed_personas_cmd() -> None: """Push assistent-extensions/ → VM Assistent/extensions (+ _base).""" diff --git a/src/gpu_rent/debug_assistent_session.py b/src/gpu_rent/debug_assistent_session.py index 78e8018..effd3d4 100644 --- a/src/gpu_rent/debug_assistent_session.py +++ b/src/gpu_rent/debug_assistent_session.py @@ -525,6 +525,16 @@ def build_chat_trace( } if isinstance(response, dict) and response.get("civitai_results"): out["civitai_results"] = response.get("civitai_results") + if isinstance(response, dict) and isinstance(response.get("knowledge"), dict): + out["knowledge"] = response.get("knowledge") + elif isinstance(response, dict) and response.get("civitai_results"): + # Shim until swarm-assistent ships unified knowledge block + out["knowledge"] = { + "catalog": [], + "hops": [], + "results": response.get("civitai_results"), + "legacy": "civitai_results", + } return out diff --git a/src/gpu_rent/paths.py b/src/gpu_rent/paths.py index 76f30ea..ad0ca54 100644 --- a/src/gpu_rent/paths.py +++ b/src/gpu_rent/paths.py @@ -72,6 +72,14 @@ def assistent_personas_manifest_path() -> Path: return app_root() / "assistent-personas.yaml" +def books_dir() -> Path: + return app_root() / "books" + + +def books_example_dir() -> Path: + return app_root() / "books.example" + + def assistent_personas_example_path() -> Path: """Deprecated yaml example — prefer assistent_extensions_example_dir().""" return app_root() / "assistent-personas.example.yaml" diff --git a/src/gpu_rent/provision.py b/src/gpu_rent/provision.py index 158fe1b..51ad10d 100644 --- a/src/gpu_rent/provision.py +++ b/src/gpu_rent/provision.py @@ -1141,11 +1141,81 @@ def seed_assistent_personas(cfg: Config, host: str, log: Log) -> None: f"assistent-extensions → packs/{pushed} " f"(default={default_id}{ctx_note})" ) - seed_civitai_examples(cfg, host, log) + seed_books(cfg, host, log) + + +def seed_books(cfg: Config, host: str, log: Log, *, force: bool = False) -> None: + """Push changed books/*/ → Assistent/books/ (content_sha diff).""" + import json + + from gpu_rent.books import ( + SIDEcar_NAME, + VM_BOOKS_PREFIX, + iter_installed_books, + read_meta, + remote_book_dir, + seed_book_files, + ) + from gpu_rent.paths import app_root + from gpu_rent.ssh_ops import put_file, put_text + + root = Path(getattr(cfg, "app_root", None) or app_root()) + book_ids = iter_installed_books(root) + if not book_ids: + seed_civitai_examples(cfg, host, log) + return + + run_ssh(cfg, host, f"mkdir -p {shlex.quote(VM_BOOKS_PREFIX)}", check=False) + pushed = 0 + for book_id in book_ids: + meta = read_meta(book_id, root) + local_sha = str((meta or {}).get("content_sha") or "").strip() + if not local_sha: + log(f"books/{book_id}: нет meta.content_sha — skip") + continue + remote_dir = remote_book_dir(book_id) + sidecar_remote = f"{remote_dir}/{SIDEcar_NAME}" + remote_raw = run_ssh( + cfg, + host, + f"test -f {shlex.quote(sidecar_remote)} && cat {shlex.quote(sidecar_remote)} || true", + check=False, + timeout=20, + ).strip() + remote_sha = "" + if remote_raw: + try: + remote_meta = json.loads(remote_raw) + if isinstance(remote_meta, dict): + remote_sha = str(remote_meta.get("content_sha") or "") + except json.JSONDecodeError: + remote_sha = "" + + if not force and remote_sha == local_sha: + log(f"books/{book_id}: unchanged") + continue + + run_ssh(cfg, host, f"mkdir -p {shlex.quote(remote_dir)}", check=False) + local_dir = root / "books" / book_id + for local_path, remote_path in seed_book_files(local_dir, remote_dir): + put_file(cfg, host, local_path, remote_path) + sidecar_payload = json.dumps( + {"content_sha": local_sha, "book_id": book_id, **(meta or {})}, + ensure_ascii=False, + ) + put_text(cfg, host, sidecar_remote, sidecar_payload + "\n") + size_kb = max(1, (local_dir / "search.jsonl").stat().st_size // 1024) + log(f"books/{book_id} -> {remote_dir} ({size_kb} KB, sha={local_sha[:12]})") + pushed += 1 + + if pushed: + log(f"books: pushed {pushed}/{len(book_ids)}") + elif book_ids: + log(f"books: all {len(book_ids)} unchanged") def seed_civitai_examples(cfg: Config, host: str, log: Log) -> None: - """Push datasets/civitai/search.jsonl → Assistent/civitai-examples.jsonl (FTS, no embed).""" + """Legacy FTS path when books/ not built yet.""" from gpu_rent.paths import app_root from gpu_rent.ssh_ops import put_file diff --git a/tests/test_assistent_personas_seed.py b/tests/test_assistent_personas_seed.py index 8b75996..f114dbf 100644 --- a/tests/test_assistent_personas_seed.py +++ b/tests/test_assistent_personas_seed.py @@ -56,7 +56,7 @@ def test_seed_assistent_personas_pushes_packs(tmp_path: Path, monkeypatch): "gpu_rent.paths.assistent_personas_manifest_path", lambda: tmp_path / "no.yaml" ) monkeypatch.setattr( - "gpu_rent.provision.seed_civitai_examples", lambda *a, **k: None + "gpu_rent.provision.seed_books", lambda *a, **k: None ) cfg = MagicMock() @@ -98,7 +98,7 @@ def test_seed_skips_empty_pack_dir(tmp_path: Path, monkeypatch): "gpu_rent.paths.assistent_personas_manifest_path", lambda: tmp_path / "no.yaml" ) monkeypatch.setattr( - "gpu_rent.provision.seed_civitai_examples", lambda *a, **k: None + "gpu_rent.provision.seed_books", lambda *a, **k: None ) cfg = MagicMock() diff --git a/tests/test_books.py b/tests/test_books.py new file mode 100644 index 0000000..c9cc248 --- /dev/null +++ b/tests/test_books.py @@ -0,0 +1,129 @@ +"""Tests for Assistent books build + seed helpers.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock + +from gpu_rent.books import ( + BOOK_CIVITAI, + build_civitai_book, + civitai_search_to_book_row, + fictext_row_to_book_rows, + read_meta, + sha256_file, + write_meta, +) +from gpu_rent.provision import seed_books + + +def test_civitai_search_to_book_row(): + row = { + "id": 1, + "prompt": "woman, redhead", + "negative": "blur", + "tags": ["woman"], + "rating": "pg13", + "params": {"steps": 8, "cfg": 1}, + "loras": [{"versionId": 9, "weight": 0.7}], + "modelVersionId": 100, + "kind": "checkpoint", + "score": 10, + } + out = civitai_search_to_book_row(row) + assert out["book"] == BOOK_CIVITAI + assert "woman" in out["text"] + assert out["meta"]["loras"][0]["versionId"] == 9 + + +def test_fictext_chunks_skip_minor(): + rows = fictext_row_to_book_rows( + {"title": "T", "tags": ["loli"], "text": "loli content here"}, + "ru-fictext-rplus", + row_index=0, + ) + assert rows == [] + + +def test_fictext_chunks_ok(): + text = "First paragraph.\n\nSecond paragraph with more words." + rows = fictext_row_to_book_rows( + {"title": "Story", "tags": ["romance"], "text": text}, + "ru-fictext-rplus", + row_index=3, + ) + assert len(rows) >= 1 + assert rows[0]["title"] == "Story" + assert "romance" in rows[0]["tags"] + + +def test_build_civitai_book(tmp_path: Path): + ds = tmp_path / "datasets" / "civitai" + ds.mkdir(parents=True) + search_row = { + "id": 5, + "prompt": "test prompt", + "negative": "", + "tags": ["a"], + "rating": "pg", + "params": {}, + "loras": [], + } + (ds / "search.jsonl").write_text(json.dumps(search_row) + "\n", encoding="utf-8") + counts = build_civitai_book(out_root=tmp_path, log=lambda m: None) + assert counts["rows"] == 1 + meta = read_meta(BOOK_CIVITAI, tmp_path) + assert meta and meta.get("row_count") == 1 + book_search = tmp_path / "books" / BOOK_CIVITAI / "search.jsonl" + assert book_search.is_file() + assert sha256_file(book_search) == meta["content_sha"] + + +def test_seed_books_skips_unchanged(tmp_path: Path, monkeypatch): + book = tmp_path / "books" / BOOK_CIVITAI + book.mkdir(parents=True) + (book / "search.jsonl").write_text('{"id":"x"}\n', encoding="utf-8") + write_meta(BOOK_CIVITAI, row_count=1, root=tmp_path) + meta = read_meta(BOOK_CIVITAI, tmp_path) + sidecar = json.dumps({"content_sha": meta["content_sha"]}) + + monkeypatch.setattr("gpu_rent.paths.app_root", lambda: tmp_path) + monkeypatch.setattr( + "gpu_rent.provision.run_ssh", + lambda *a, **k: sidecar if ".gpu-rent-meta" in str(a[2]) else "", + ) + put_calls: list[str] = [] + monkeypatch.setattr( + "gpu_rent.ssh_ops.put_file", + lambda *a, **k: put_calls.append(str(a[3])), + ) + monkeypatch.setattr("gpu_rent.ssh_ops.put_text", lambda *a, **k: None) + monkeypatch.setattr("gpu_rent.provision.seed_civitai_examples", lambda *a, **k: None) + + logs: list[str] = [] + seed_books(MagicMock(app_root=str(tmp_path)), "host", logs.append) + assert put_calls == [] + assert any("unchanged" in m for m in logs) + + +def test_seed_books_pushes_when_sha_diff(tmp_path: Path, monkeypatch): + book = tmp_path / "books" / BOOK_CIVITAI + book.mkdir(parents=True) + (book / "book.yaml").write_text("kind: book\n", encoding="utf-8") + (book / "search.jsonl").write_text('{"id":"x"}\n', encoding="utf-8") + write_meta(BOOK_CIVITAI, row_count=1, root=tmp_path) + + monkeypatch.setattr("gpu_rent.paths.app_root", lambda: tmp_path) + monkeypatch.setattr("gpu_rent.provision.run_ssh", lambda *a, **k: "") + put_calls: list[str] = [] + monkeypatch.setattr( + "gpu_rent.ssh_ops.put_file", + lambda *a, **k: put_calls.append(str(a[3])), + ) + monkeypatch.setattr("gpu_rent.ssh_ops.put_text", lambda *a, **k: None) + + logs: list[str] = [] + seed_books(MagicMock(app_root=str(tmp_path)), "host", logs.append) + assert any("search.jsonl" in p for p in put_calls) + assert any("books/civitai-krea2" in m for m in logs) diff --git a/tests/test_civitai_dataset.py b/tests/test_civitai_dataset.py index 1b2ed55..20b0888 100644 --- a/tests/test_civitai_dataset.py +++ b/tests/test_civitai_dataset.py @@ -18,7 +18,6 @@ from gpu_rent.civitai_dataset import ( reaction_score, search_row, tags_from_image, - train_row, ) from gpu_rent.errors import CloudError @@ -108,7 +107,7 @@ def test_normalize_skips_low_score(): ) -def test_train_and_search_rows(): +def test_search_row(): row = { "id": 7, "rating": "r", @@ -121,11 +120,6 @@ def test_train_and_search_rows(): "params": {"steps": 8, "cfgScale": 1, "sampler": "euler"}, "resources": [{"type": "lora", "modelVersionId": 1, "weight": 0.5}], } - tr = train_row(row) - assert "Tags: woman, cinematic" in tr["instruction"] - assert "Rating: r" in tr["instruction"] - assert "A woman" in tr["output"] - assert "cfg: 1" in tr["output"] sr = search_row(row) assert sr["id"] == 7 assert sr["loras"] == [{"versionId": 1, "weight": 0.5}] @@ -165,16 +159,16 @@ def test_cmd_split_writes_artifacts(tmp_path: Path): for r in rows: fh.write(json.dumps(r) + "\n") counts = cmd_split(out_root=tmp_path, log=lambda m: None) - assert counts["train"] == 2 assert counts["search"] == 2 assert counts["kind:checkpoint"] == 1 assert counts["kind:lora"] == 1 assert counts["rating:pg"] == 1 assert counts["rating:x"] == 1 + assert counts.get("book:civitai-krea2") == 2 search = (tmp_path / "datasets" / "civitai" / "search.jsonl").read_text(encoding="utf-8") assert '"id": 1' in search - train = (tmp_path / "datasets" / "civitai" / "train.jsonl").read_text(encoding="utf-8") - assert "Write a Krea 2 prompt" in train + book = tmp_path / "books" / "civitai-krea2" / "search.jsonl" + assert book.is_file() def _mock_response(payload: dict, status: int = 200) -> httpx.Response: diff --git a/tmp_smoke.sh b/tmp_smoke.sh index 5babf28..4b2f960 100644 --- a/tmp_smoke.sh +++ b/tmp_smoke.sh @@ -2,7 +2,7 @@ sudo -n systemctl restart swarmui sleep 40 S=$(curl -s -X POST http://127.0.0.1:7801/API/GetNewSession -H 'Content-Type: application/json' -d '{}' | sed -n 's/.*"session_id": *"\([^"]*\)".*/\1/p') -for api in AssistentListChats AssistentListTrainSamples AssistentListMemory; do +for api in AssistentListChats AssistentListMemory; do echo "--- $api ---" curl -s -X POST "http://127.0.0.1:7801/API/$api" -H 'Content-Type: application/json' -d "{\"session_id\":\"$S\"}" | head -c 300 echo