Add Assistent reference books and remove training dataset output.

Introduce books/ with civitai-krea2 and HF fictext builders, sha-diff seed to VM, and drop train.jsonl from the civitai scrape pipeline.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 22:56:07 +03:00
co-authored by Cursor
parent 84ed0bb47a
commit 07bb521b70
27 changed files with 836 additions and 64 deletions
+4 -10
View File
@@ -18,7 +18,6 @@ from gpu_rent.civitai_dataset import (
reaction_score,
search_row,
tags_from_image,
train_row,
)
from gpu_rent.errors import CloudError
@@ -108,7 +107,7 @@ def test_normalize_skips_low_score():
)
def test_train_and_search_rows():
def test_search_row():
row = {
"id": 7,
"rating": "r",
@@ -121,11 +120,6 @@ def test_train_and_search_rows():
"params": {"steps": 8, "cfgScale": 1, "sampler": "euler"},
"resources": [{"type": "lora", "modelVersionId": 1, "weight": 0.5}],
}
tr = train_row(row)
assert "Tags: woman, cinematic" in tr["instruction"]
assert "Rating: r" in tr["instruction"]
assert "A woman" in tr["output"]
assert "cfg: 1" in tr["output"]
sr = search_row(row)
assert sr["id"] == 7
assert sr["loras"] == [{"versionId": 1, "weight": 0.5}]
@@ -165,16 +159,16 @@ def test_cmd_split_writes_artifacts(tmp_path: Path):
for r in rows:
fh.write(json.dumps(r) + "\n")
counts = cmd_split(out_root=tmp_path, log=lambda m: None)
assert counts["train"] == 2
assert counts["search"] == 2
assert counts["kind:checkpoint"] == 1
assert counts["kind:lora"] == 1
assert counts["rating:pg"] == 1
assert counts["rating:x"] == 1
assert counts.get("book:civitai-krea2") == 2
search = (tmp_path / "datasets" / "civitai" / "search.jsonl").read_text(encoding="utf-8")
assert '"id": 1' in search
train = (tmp_path / "datasets" / "civitai" / "train.jsonl").read_text(encoding="utf-8")
assert "Write a Krea 2 prompt" in train
book = tmp_path / "books" / "civitai-krea2" / "search.jsonl"
assert book.is_file()
def _mock_response(payload: dict, status: int = 200) -> httpx.Response: