Add Assistent reference books and remove training dataset output.

Introduce books/ with civitai-krea2 and HF fictext builders, sha-diff seed to VM, and drop train.jsonl from the civitai scrape pipeline.

Co-authored-by: Cursor <cursoragent@cursor.com>

tests/test_assistent_personas_seed.py

@@ -56,7 +56,7 @@ def test_seed_assistent_personas_pushes_packs(tmp_path: Path, monkeypatch):
         "gpu_rent.paths.assistent_personas_manifest_path", lambda: tmp_path / "no.yaml"
     )
     monkeypatch.setattr(
-        "gpu_rent.provision.seed_civitai_examples", lambda *a, **k: None
+        "gpu_rent.provision.seed_books", lambda *a, **k: None
     )
 
     cfg = MagicMock()
@@ -98,7 +98,7 @@ def test_seed_skips_empty_pack_dir(tmp_path: Path, monkeypatch):
         "gpu_rent.paths.assistent_personas_manifest_path", lambda: tmp_path / "no.yaml"
     )
     monkeypatch.setattr(
-        "gpu_rent.provision.seed_civitai_examples", lambda *a, **k: None
+        "gpu_rent.provision.seed_books", lambda *a, **k: None
     )
 
     cfg = MagicMock()

tests/test_books.py

@@ -0,0 +1,129 @@
+"""Tests for Assistent books build + seed helpers."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from unittest.mock import MagicMock
+
+from gpu_rent.books import (
+    BOOK_CIVITAI,
+    build_civitai_book,
+    civitai_search_to_book_row,
+    fictext_row_to_book_rows,
+    read_meta,
+    sha256_file,
+    write_meta,
+)
+from gpu_rent.provision import seed_books
+
+
+def test_civitai_search_to_book_row():
+    row = {
+        "id": 1,
+        "prompt": "woman, redhead",
+        "negative": "blur",
+        "tags": ["woman"],
+        "rating": "pg13",
+        "params": {"steps": 8, "cfg": 1},
+        "loras": [{"versionId": 9, "weight": 0.7}],
+        "modelVersionId": 100,
+        "kind": "checkpoint",
+        "score": 10,
+    }
+    out = civitai_search_to_book_row(row)
+    assert out["book"] == BOOK_CIVITAI
+    assert "woman" in out["text"]
+    assert out["meta"]["loras"][0]["versionId"] == 9
+
+
+def test_fictext_chunks_skip_minor():
+    rows = fictext_row_to_book_rows(
+        {"title": "T", "tags": ["loli"], "text": "loli content here"},
+        "ru-fictext-rplus",
+        row_index=0,
+    )
+    assert rows == []
+
+
+def test_fictext_chunks_ok():
+    text = "First paragraph.\n\nSecond paragraph with more words."
+    rows = fictext_row_to_book_rows(
+        {"title": "Story", "tags": ["romance"], "text": text},
+        "ru-fictext-rplus",
+        row_index=3,
+    )
+    assert len(rows) >= 1
+    assert rows[0]["title"] == "Story"
+    assert "romance" in rows[0]["tags"]
+
+
+def test_build_civitai_book(tmp_path: Path):
+    ds = tmp_path / "datasets" / "civitai"
+    ds.mkdir(parents=True)
+    search_row = {
+        "id": 5,
+        "prompt": "test prompt",
+        "negative": "",
+        "tags": ["a"],
+        "rating": "pg",
+        "params": {},
+        "loras": [],
+    }
+    (ds / "search.jsonl").write_text(json.dumps(search_row) + "\n", encoding="utf-8")
+    counts = build_civitai_book(out_root=tmp_path, log=lambda m: None)
+    assert counts["rows"] == 1
+    meta = read_meta(BOOK_CIVITAI, tmp_path)
+    assert meta and meta.get("row_count") == 1
+    book_search = tmp_path / "books" / BOOK_CIVITAI / "search.jsonl"
+    assert book_search.is_file()
+    assert sha256_file(book_search) == meta["content_sha"]
+
+
+def test_seed_books_skips_unchanged(tmp_path: Path, monkeypatch):
+    book = tmp_path / "books" / BOOK_CIVITAI
+    book.mkdir(parents=True)
+    (book / "search.jsonl").write_text('{"id":"x"}\n', encoding="utf-8")
+    write_meta(BOOK_CIVITAI, row_count=1, root=tmp_path)
+    meta = read_meta(BOOK_CIVITAI, tmp_path)
+    sidecar = json.dumps({"content_sha": meta["content_sha"]})
+
+    monkeypatch.setattr("gpu_rent.paths.app_root", lambda: tmp_path)
+    monkeypatch.setattr(
+        "gpu_rent.provision.run_ssh",
+        lambda *a, **k: sidecar if ".gpu-rent-meta" in str(a[2]) else "",
+    )
+    put_calls: list[str] = []
+    monkeypatch.setattr(
+        "gpu_rent.ssh_ops.put_file",
+        lambda *a, **k: put_calls.append(str(a[3])),
+    )
+    monkeypatch.setattr("gpu_rent.ssh_ops.put_text", lambda *a, **k: None)
+    monkeypatch.setattr("gpu_rent.provision.seed_civitai_examples", lambda *a, **k: None)
+
+    logs: list[str] = []
+    seed_books(MagicMock(app_root=str(tmp_path)), "host", logs.append)
+    assert put_calls == []
+    assert any("unchanged" in m for m in logs)
+
+
+def test_seed_books_pushes_when_sha_diff(tmp_path: Path, monkeypatch):
+    book = tmp_path / "books" / BOOK_CIVITAI
+    book.mkdir(parents=True)
+    (book / "book.yaml").write_text("kind: book\n", encoding="utf-8")
+    (book / "search.jsonl").write_text('{"id":"x"}\n', encoding="utf-8")
+    write_meta(BOOK_CIVITAI, row_count=1, root=tmp_path)
+
+    monkeypatch.setattr("gpu_rent.paths.app_root", lambda: tmp_path)
+    monkeypatch.setattr("gpu_rent.provision.run_ssh", lambda *a, **k: "")
+    put_calls: list[str] = []
+    monkeypatch.setattr(
+        "gpu_rent.ssh_ops.put_file",
+        lambda *a, **k: put_calls.append(str(a[3])),
+    )
+    monkeypatch.setattr("gpu_rent.ssh_ops.put_text", lambda *a, **k: None)
+
+    logs: list[str] = []
+    seed_books(MagicMock(app_root=str(tmp_path)), "host", logs.append)
+    assert any("search.jsonl" in p for p in put_calls)
+    assert any("books/civitai-krea2" in m for m in logs)

tests/test_civitai_dataset.py

@@ -18,7 +18,6 @@ from gpu_rent.civitai_dataset import (
     reaction_score,
     search_row,
     tags_from_image,
-    train_row,
 )
 from gpu_rent.errors import CloudError
 
@@ -108,7 +107,7 @@ def test_normalize_skips_low_score():
     )
 
 
-def test_train_and_search_rows():
+def test_search_row():
     row = {
         "id": 7,
         "rating": "r",
@@ -121,11 +120,6 @@ def test_train_and_search_rows():
         "params": {"steps": 8, "cfgScale": 1, "sampler": "euler"},
         "resources": [{"type": "lora", "modelVersionId": 1, "weight": 0.5}],
     }
-    tr = train_row(row)
-    assert "Tags: woman, cinematic" in tr["instruction"]
-    assert "Rating: r" in tr["instruction"]
-    assert "A woman" in tr["output"]
-    assert "cfg: 1" in tr["output"]
     sr = search_row(row)
     assert sr["id"] == 7
     assert sr["loras"] == [{"versionId": 1, "weight": 0.5}]
@@ -165,16 +159,16 @@ def test_cmd_split_writes_artifacts(tmp_path: Path):
         for r in rows:
             fh.write(json.dumps(r) + "\n")
     counts = cmd_split(out_root=tmp_path, log=lambda m: None)
-    assert counts["train"] == 2
     assert counts["search"] == 2
     assert counts["kind:checkpoint"] == 1
     assert counts["kind:lora"] == 1
     assert counts["rating:pg"] == 1
     assert counts["rating:x"] == 1
+    assert counts.get("book:civitai-krea2") == 2
     search = (tmp_path / "datasets" / "civitai" / "search.jsonl").read_text(encoding="utf-8")
     assert '"id": 1' in search
-    train = (tmp_path / "datasets" / "civitai" / "train.jsonl").read_text(encoding="utf-8")
-    assert "Write a Krea 2 prompt" in train
+    book = tmp_path / "books" / "civitai-krea2" / "search.jsonl"
+    assert book.is_file()
 
 
 def _mock_response(payload: dict, status: int = 200) -> httpx.Response: