Add Assistent reference books and remove training dataset output.
Introduce books/ with civitai-krea2 and HF fictext builders, sha-diff seed to VM, and drop train.jsonl from the civitai scrape pipeline. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -18,7 +18,6 @@ from gpu_rent.civitai_dataset import (
|
||||
reaction_score,
|
||||
search_row,
|
||||
tags_from_image,
|
||||
train_row,
|
||||
)
|
||||
from gpu_rent.errors import CloudError
|
||||
|
||||
@@ -108,7 +107,7 @@ def test_normalize_skips_low_score():
|
||||
)
|
||||
|
||||
|
||||
def test_train_and_search_rows():
|
||||
def test_search_row():
|
||||
row = {
|
||||
"id": 7,
|
||||
"rating": "r",
|
||||
@@ -121,11 +120,6 @@ def test_train_and_search_rows():
|
||||
"params": {"steps": 8, "cfgScale": 1, "sampler": "euler"},
|
||||
"resources": [{"type": "lora", "modelVersionId": 1, "weight": 0.5}],
|
||||
}
|
||||
tr = train_row(row)
|
||||
assert "Tags: woman, cinematic" in tr["instruction"]
|
||||
assert "Rating: r" in tr["instruction"]
|
||||
assert "A woman" in tr["output"]
|
||||
assert "cfg: 1" in tr["output"]
|
||||
sr = search_row(row)
|
||||
assert sr["id"] == 7
|
||||
assert sr["loras"] == [{"versionId": 1, "weight": 0.5}]
|
||||
@@ -165,16 +159,16 @@ def test_cmd_split_writes_artifacts(tmp_path: Path):
|
||||
for r in rows:
|
||||
fh.write(json.dumps(r) + "\n")
|
||||
counts = cmd_split(out_root=tmp_path, log=lambda m: None)
|
||||
assert counts["train"] == 2
|
||||
assert counts["search"] == 2
|
||||
assert counts["kind:checkpoint"] == 1
|
||||
assert counts["kind:lora"] == 1
|
||||
assert counts["rating:pg"] == 1
|
||||
assert counts["rating:x"] == 1
|
||||
assert counts.get("book:civitai-krea2") == 2
|
||||
search = (tmp_path / "datasets" / "civitai" / "search.jsonl").read_text(encoding="utf-8")
|
||||
assert '"id": 1' in search
|
||||
train = (tmp_path / "datasets" / "civitai" / "train.jsonl").read_text(encoding="utf-8")
|
||||
assert "Write a Krea 2 prompt" in train
|
||||
book = tmp_path / "books" / "civitai-krea2" / "search.jsonl"
|
||||
assert book.is_file()
|
||||
|
||||
|
||||
def _mock_response(payload: dict, status: int = 200) -> httpx.Response:
|
||||
|
||||
Reference in New Issue
Block a user