Add Assistent reference books and remove training dataset output.
Introduce books/ with civitai-krea2 and HF fictext builders, sha-diff seed to VM, and drop train.jsonl from the civitai scrape pipeline. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
Cursor
@@ -0,0 +1,458 @@
|
||||
"""Assistent reference books: build locally, seed to VM on change (FTS search.jsonl)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Iterable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
|
||||
from gpu_rent.civitai_dataset import looks_minor, read_jsonl, write_jsonl
|
||||
from gpu_rent.config import load_config
|
||||
from gpu_rent.errors import GpuRentError
|
||||
from gpu_rent.paths import app_root
|
||||
|
||||
Log = Callable[[str], None]
|
||||
|
||||
BOOK_CIVITAI = "civitai-krea2"
|
||||
BOOK_FICTEXT_RPLUS = "ru-fictext-rplus"
|
||||
BOOK_FICTEXT_FULL = "ru-fictext-full"
|
||||
|
||||
VM_BOOKS_PREFIX = "/mnt/swarm_data/Assistent/books"
|
||||
SIDEcar_NAME = ".gpu-rent-meta.json"
|
||||
|
||||
HF_REPO = "krplt/ru-fictext-nsfw"
|
||||
HF_FILES = {
|
||||
BOOK_FICTEXT_RPLUS: "ru-fictext-nsfw-data-r+.parquet",
|
||||
BOOK_FICTEXT_FULL: "ru-fictext-nsfw-data.parquet",
|
||||
}
|
||||
|
||||
CHUNK_CHARS = 500
|
||||
CHUNK_OVERLAP = 80
|
||||
EXCERPT_MAX = 500
|
||||
|
||||
BOOK_SPECS: dict[str, dict[str, Any]] = {
|
||||
BOOK_CIVITAI: {
|
||||
"kind": "book",
|
||||
"id": BOOK_CIVITAI,
|
||||
"title": "Civitai Krea2",
|
||||
"description": (
|
||||
"Примеры промптов, негативов, steps/cfg и LoRA для Krea 2 с Civitai. "
|
||||
"Ищи сюда, когда нужен рабочий prompt или параметры генерации."
|
||||
),
|
||||
"content_kind": "prompt-examples",
|
||||
"language": "en",
|
||||
"tags": ["civitai", "krea2", "prompts"],
|
||||
"source": "civitai-dataset",
|
||||
"license": "civitai-terms",
|
||||
},
|
||||
BOOK_FICTEXT_RPLUS: {
|
||||
"kind": "book",
|
||||
"id": BOOK_FICTEXT_RPLUS,
|
||||
"title": "RU ficbook R+",
|
||||
"description": (
|
||||
"Отрывки русской фанфикшн (mature/NSFW) для стиля, лексики и тона сцен. "
|
||||
"Не цитируй дословно длинно — бери формулировки и ритм."
|
||||
),
|
||||
"content_kind": "prose-style",
|
||||
"language": "ru",
|
||||
"tags": ["fanfiction", "nsfw", "russian"],
|
||||
"source": f"huggingface.co/datasets/{HF_REPO}",
|
||||
"license": "cc-by-4.0",
|
||||
},
|
||||
BOOK_FICTEXT_FULL: {
|
||||
"kind": "book",
|
||||
"id": BOOK_FICTEXT_FULL,
|
||||
"title": "RU ficbook (full)",
|
||||
"description": (
|
||||
"Отрывки русской фанфикшн (NSFW + safe) для стиля и лексики. "
|
||||
"Шире чем R+; для нейтральных сцен предпочитай safe-теги."
|
||||
),
|
||||
"content_kind": "prose-style",
|
||||
"language": "ru",
|
||||
"tags": ["fanfiction", "russian"],
|
||||
"source": f"huggingface.co/datasets/{HF_REPO}",
|
||||
"license": "cc-by-4.0",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
print(msg, flush=True)
|
||||
|
||||
|
||||
def books_root(root: Path | None = None) -> Path:
|
||||
return (root or app_root()) / "books"
|
||||
|
||||
|
||||
def book_dir(book_id: str, root: Path | None = None) -> Path:
|
||||
return books_root(root) / book_id
|
||||
|
||||
|
||||
def write_book_yaml(book_id: str, root: Path | None = None) -> Path:
|
||||
spec = BOOK_SPECS.get(book_id)
|
||||
if not spec:
|
||||
raise GpuRentError(f"Unknown book id: {book_id}")
|
||||
path = book_dir(book_id, root) / "book.yaml"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(yaml.safe_dump(spec, allow_unicode=True, sort_keys=False), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def write_meta(
|
||||
book_id: str,
|
||||
*,
|
||||
row_count: int,
|
||||
root: Path | None = None,
|
||||
source_sha: str | None = None,
|
||||
extra: dict[str, Any] | None = None,
|
||||
) -> Path:
|
||||
bdir = book_dir(book_id, root)
|
||||
search = bdir / "search.jsonl"
|
||||
if not search.is_file():
|
||||
raise GpuRentError(f"Нет {search} для meta")
|
||||
payload: dict[str, Any] = {
|
||||
"book_id": book_id,
|
||||
"row_count": row_count,
|
||||
"content_sha": sha256_file(search),
|
||||
"built_at": datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
||||
"search_bytes": search.stat().st_size,
|
||||
}
|
||||
if source_sha:
|
||||
payload["source_sha"] = source_sha
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
path = bdir / "meta.json"
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def read_meta(book_id: str, root: Path | None = None) -> dict[str, Any] | None:
|
||||
path = book_dir(book_id, root) / "meta.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
def iter_installed_books(root: Path | None = None) -> list[str]:
|
||||
root_p = books_root(root)
|
||||
if not root_p.is_dir():
|
||||
return []
|
||||
out: list[str] = []
|
||||
for child in sorted(root_p.iterdir()):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
if (child / "search.jsonl").is_file():
|
||||
out.append(child.name)
|
||||
return out
|
||||
|
||||
|
||||
def civitai_search_to_book_row(row: dict[str, Any], book_id: str = BOOK_CIVITAI) -> dict[str, Any]:
|
||||
prompt = str(row.get("prompt") or "").strip()
|
||||
neg = str(row.get("negative") or row.get("negativePrompt") or "").strip()
|
||||
parts = [prompt] if prompt else []
|
||||
if neg:
|
||||
parts.append(f"Negative: {neg}")
|
||||
params = row.get("params") if isinstance(row.get("params"), dict) else {}
|
||||
for key, label in (("steps", "steps"), ("cfg", "cfg"), ("cfgScale", "cfg"), ("sampler", "sampler")):
|
||||
if params.get(key) is not None:
|
||||
parts.append(f"{label}: {params[key]}")
|
||||
text = "\n".join(parts)[:EXCERPT_MAX]
|
||||
meta = {
|
||||
"prompt": prompt,
|
||||
"negative": neg,
|
||||
"params": params,
|
||||
"loras": list(row.get("loras") or []),
|
||||
"modelVersionId": row.get("modelVersionId"),
|
||||
"kind": row.get("kind"),
|
||||
"score": row.get("score"),
|
||||
}
|
||||
rid = row.get("id")
|
||||
return {
|
||||
"id": f"{book_id}:{rid}",
|
||||
"book": book_id,
|
||||
"title": "",
|
||||
"tags": list(row.get("tags") or []),
|
||||
"text": text,
|
||||
"body": prompt[:2000] if prompt else text,
|
||||
"rating": row.get("rating") or "pg",
|
||||
"meta": meta,
|
||||
}
|
||||
|
||||
|
||||
def _chunk_text(text: str, *, chunk_chars: int = CHUNK_CHARS, overlap: int = CHUNK_OVERLAP) -> list[str]:
|
||||
text = re.sub(r"\r\n?", "\n", text.strip())
|
||||
if not text:
|
||||
return []
|
||||
paras = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()]
|
||||
chunks: list[str] = []
|
||||
for para in paras:
|
||||
if len(para) <= chunk_chars:
|
||||
chunks.append(para)
|
||||
continue
|
||||
start = 0
|
||||
while start < len(para):
|
||||
end = min(len(para), start + chunk_chars)
|
||||
chunks.append(para[start:end])
|
||||
if end >= len(para):
|
||||
break
|
||||
start = max(0, end - overlap)
|
||||
return chunks
|
||||
|
||||
|
||||
def fictext_row_to_book_rows(
|
||||
row: dict[str, Any],
|
||||
book_id: str,
|
||||
*,
|
||||
row_index: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
title = str(row.get("title") or "").strip()
|
||||
tags_raw = row.get("tags")
|
||||
tags: list[str] = []
|
||||
if isinstance(tags_raw, list):
|
||||
tags = [str(t).strip() for t in tags_raw if str(t).strip()]
|
||||
elif isinstance(tags_raw, str):
|
||||
tags = [t.strip() for t in re.split(r"[,;]", tags_raw) if t.strip()]
|
||||
text = str(row.get("text") or "").strip()
|
||||
if not text or looks_minor(text, tags):
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for ci, chunk in enumerate(_chunk_text(text)):
|
||||
excerpt = chunk[:EXCERPT_MAX]
|
||||
out.append(
|
||||
{
|
||||
"id": f"{book_id}:{row_index}:{ci}",
|
||||
"book": book_id,
|
||||
"title": title,
|
||||
"tags": tags[:20],
|
||||
"text": excerpt,
|
||||
"body": chunk[:2000],
|
||||
"rating": "r" if book_id == BOOK_FICTEXT_RPLUS else "pg13",
|
||||
"meta": {"source_row": row_index, "chunk": ci},
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def build_civitai_book(*, out_root: Path | None = None, log: Log = _log) -> dict[str, int]:
|
||||
"""Write books/civitai-krea2 from datasets/civitai/search.jsonl."""
|
||||
root = out_root or app_root()
|
||||
src = root / "datasets" / "civitai" / "search.jsonl"
|
||||
if not src.is_file():
|
||||
# fallback: build from catalog images via civitai_dataset split output path
|
||||
raise GpuRentError(f"Нет {src} — сначала civitai-dataset split")
|
||||
rows = read_jsonl(src)
|
||||
book_rows = [civitai_search_to_book_row(r) for r in rows]
|
||||
bdir = book_dir(BOOK_CIVITAI, root)
|
||||
write_book_yaml(BOOK_CIVITAI, root)
|
||||
n = write_jsonl(bdir / "search.jsonl", book_rows)
|
||||
write_meta(BOOK_CIVITAI, row_count=n, root=root, extra={"source": str(src)})
|
||||
log(f"books/{BOOK_CIVITAI}: {n} rows → {bdir}")
|
||||
return {"rows": n}
|
||||
|
||||
|
||||
def _hf_download_url(filename: str) -> str:
|
||||
return f"https://huggingface.co/datasets/{HF_REPO}/resolve/main/{filename}"
|
||||
|
||||
|
||||
def _require_pyarrow():
|
||||
try:
|
||||
import pyarrow.parquet as pq # noqa: F401
|
||||
|
||||
return pq
|
||||
except ImportError as exc:
|
||||
raise GpuRentError(
|
||||
"Нужен pyarrow для fictext: pip install 'gpu-rent[books]' или pip install pyarrow"
|
||||
) from exc
|
||||
|
||||
|
||||
def download_fictext_parquet(
|
||||
book_id: str,
|
||||
*,
|
||||
out_root: Path | None = None,
|
||||
log: Log = _log,
|
||||
token: str | None = None,
|
||||
) -> Path:
|
||||
if book_id not in HF_FILES:
|
||||
raise GpuRentError(f"Unknown fictext book: {book_id}")
|
||||
filename = HF_FILES[book_id]
|
||||
bdir = book_dir(book_id, out_root)
|
||||
bdir.mkdir(parents=True, exist_ok=True)
|
||||
dest = bdir / filename
|
||||
url = _hf_download_url(filename)
|
||||
headers: dict[str, str] = {}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
log(f"download: {url}")
|
||||
with httpx.Client(follow_redirects=True, timeout=600.0) as client:
|
||||
with client.stream("GET", url, headers=headers) as resp:
|
||||
resp.raise_for_status()
|
||||
with dest.open("wb") as fh:
|
||||
for chunk in resp.iter_bytes(1024 * 1024):
|
||||
fh.write(chunk)
|
||||
log(f"saved {dest} ({dest.stat().st_size // 1024} KB)")
|
||||
return dest
|
||||
|
||||
|
||||
def build_fictext_book(
|
||||
book_id: str,
|
||||
*,
|
||||
out_root: Path | None = None,
|
||||
log: Log = _log,
|
||||
max_rows: int | None = None,
|
||||
) -> dict[str, int]:
|
||||
if book_id not in HF_FILES:
|
||||
raise GpuRentError(f"Unknown fictext book: {book_id}")
|
||||
pq = _require_pyarrow()
|
||||
bdir = book_dir(book_id, out_root)
|
||||
parquet = bdir / HF_FILES[book_id]
|
||||
if not parquet.is_file():
|
||||
raise GpuRentError(f"Нет {parquet} — сначала books download fictext")
|
||||
write_book_yaml(book_id, out_root)
|
||||
table = pq.read_table(parquet)
|
||||
data = table.to_pydict()
|
||||
titles = data.get("title") or []
|
||||
tags_col = data.get("tags") or []
|
||||
texts = data.get("text") or []
|
||||
n_src = len(texts)
|
||||
book_rows: list[dict] = []
|
||||
for i in range(n_src):
|
||||
if max_rows is not None and i >= max_rows:
|
||||
break
|
||||
row = {
|
||||
"title": titles[i] if i < len(titles) else "",
|
||||
"tags": tags_col[i] if i < len(tags_col) else [],
|
||||
"text": texts[i] if i < len(texts) else "",
|
||||
}
|
||||
book_rows.extend(fictext_row_to_book_rows(row, book_id, row_index=i))
|
||||
n = write_jsonl(bdir / "search.jsonl", book_rows)
|
||||
write_meta(
|
||||
book_id,
|
||||
row_count=n,
|
||||
root=out_root,
|
||||
source_sha=sha256_file(parquet),
|
||||
extra={"source_parquet": parquet.name, "source_stories": n_src},
|
||||
)
|
||||
log(f"books/{book_id}: {n} chunks from {n_src} stories → {bdir}")
|
||||
return {"rows": n, "stories": n_src}
|
||||
|
||||
|
||||
def cmd_download_fictext(
|
||||
*,
|
||||
variant: str = "both",
|
||||
out_root: Path | None = None,
|
||||
log: Log = _log,
|
||||
) -> None:
|
||||
cfg = load_config()
|
||||
token = (getattr(cfg, "hf_token", None) or "").strip() or None
|
||||
ids = []
|
||||
if variant in {"rplus", "both"}:
|
||||
ids.append(BOOK_FICTEXT_RPLUS)
|
||||
if variant in {"full", "both"}:
|
||||
ids.append(BOOK_FICTEXT_FULL)
|
||||
for book_id in ids:
|
||||
download_fictext_parquet(book_id, out_root=out_root, log=log, token=token)
|
||||
build_fictext_book(book_id, out_root=out_root, log=log)
|
||||
|
||||
|
||||
def remote_book_dir(book_id: str) -> str:
|
||||
return f"{VM_BOOKS_PREFIX}/{book_id}"
|
||||
|
||||
|
||||
def seed_book_files(local_book: Path, remote_book: str) -> list[tuple[Path, str]]:
|
||||
"""Local book dir files to push."""
|
||||
pairs: list[tuple[Path, str]] = []
|
||||
for name in ("book.yaml", "search.jsonl", "meta.json"):
|
||||
local = local_book / name
|
||||
if local.is_file():
|
||||
pairs.append((local, f"{remote_book}/{name}"))
|
||||
meta = local_book / "meta.json"
|
||||
if meta.is_file():
|
||||
pairs.append((meta, f"{remote_book}/{SIDEcar_NAME}"))
|
||||
return pairs
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="python -m gpu_rent.books",
|
||||
description="Build Assistent reference books (search.jsonl + book.yaml)",
|
||||
)
|
||||
p.add_argument("--out", type=Path, default=None, help="App root")
|
||||
sub = p.add_subparsers(dest="cmd")
|
||||
|
||||
d = sub.add_parser("download", help="Download HF fictext parquet(s)")
|
||||
d.add_argument(
|
||||
"target",
|
||||
choices=["fictext"],
|
||||
nargs="?",
|
||||
default="fictext",
|
||||
)
|
||||
d.add_argument("--variant", choices=["rplus", "full", "both"], default="both")
|
||||
|
||||
b = sub.add_parser("build", help="Build book search.jsonl")
|
||||
b.add_argument("target", choices=["civitai", "fictext", "all"])
|
||||
b.add_argument("--variant", choices=["rplus", "full", "both"], default="both")
|
||||
|
||||
sub.add_parser("list", help="List local books with meta")
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
argv = list(sys.argv[1:] if argv is None else argv)
|
||||
parser = build_parser()
|
||||
if not argv:
|
||||
argv = ["list"]
|
||||
args = parser.parse_args(argv)
|
||||
out = args.out
|
||||
try:
|
||||
if args.cmd == "download":
|
||||
if args.target == "fictext":
|
||||
cmd_download_fictext(variant=args.variant, out_root=out)
|
||||
elif args.cmd == "build":
|
||||
if args.target in {"civitai", "all"}:
|
||||
build_civitai_book(out_root=out)
|
||||
if args.target in {"fictext", "all"}:
|
||||
for book_id in (
|
||||
[BOOK_FICTEXT_RPLUS, BOOK_FICTEXT_FULL]
|
||||
if args.variant == "both"
|
||||
else [BOOK_FICTEXT_RPLUS if args.variant == "rplus" else BOOK_FICTEXT_FULL]
|
||||
):
|
||||
build_fictext_book(book_id, out_root=out)
|
||||
elif args.cmd == "list":
|
||||
for bid in iter_installed_books(out):
|
||||
meta = read_meta(bid, out) or {}
|
||||
log(
|
||||
f"{bid}: rows={meta.get('row_count', '?')} "
|
||||
f"sha={str(meta.get('content_sha', ''))[:12]}"
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
return 0
|
||||
except GpuRentError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Local Civitai Krea2 gallery scrape → train.jsonl + search.jsonl (no image files)."""
|
||||
"""Local Civitai Krea2 gallery scrape → search.jsonl + Assistent book (no image files)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -522,28 +522,6 @@ def cmd_scrape(
|
||||
return have
|
||||
|
||||
|
||||
def train_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
tags = row.get("tags") or []
|
||||
tag_s = ", ".join(str(t) for t in tags[:12])
|
||||
rating = row.get("rating") or "pg"
|
||||
instruction = f"Write a Krea 2 prompt.\nTags: {tag_s}\nRating: {rating}"
|
||||
params = row.get("params") if isinstance(row.get("params"), dict) else {}
|
||||
parts = [str(row.get("prompt") or "").strip()]
|
||||
neg = str(row.get("negativePrompt") or "").strip()
|
||||
if neg:
|
||||
parts.append(f"Negative: {neg}")
|
||||
for key, label in (
|
||||
("steps", "steps"),
|
||||
("cfgScale", "cfg"),
|
||||
("sampler", "sampler"),
|
||||
("seed", "seed"),
|
||||
("size", "size"),
|
||||
):
|
||||
if params.get(key) is not None:
|
||||
parts.append(f"{label}: {params[key]}")
|
||||
return {"instruction": instruction, "output": "\n".join(parts)}
|
||||
|
||||
|
||||
def search_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
resources = row.get("resources") if isinstance(row.get("resources"), list) else []
|
||||
loras: list[dict[str, Any]] = []
|
||||
@@ -600,14 +578,12 @@ def cmd_split(*, out_root: Path | None = None, log: Log = _log) -> dict[str, int
|
||||
|
||||
by_kind: dict[str, list] = {"checkpoint": [], "lora": []}
|
||||
by_rating: dict[str, list] = {}
|
||||
train: list[dict] = []
|
||||
search: list[dict] = []
|
||||
for row in rows:
|
||||
kind = str(row.get("kind") or "checkpoint")
|
||||
by_kind.setdefault(kind, []).append(row)
|
||||
rating = str(row.get("rating") or "pg")
|
||||
by_rating.setdefault(rating, []).append(row)
|
||||
train.append(train_row(row))
|
||||
search.append(search_row(row))
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
@@ -618,13 +594,19 @@ def cmd_split(*, out_root: Path | None = None, log: Log = _log) -> dict[str, int
|
||||
safe = re.sub(r"[^a-z0-9]+", "", rating.lower()) or "pg"
|
||||
path = root / "by_rating" / f"{safe}.jsonl"
|
||||
counts[f"rating:{safe}"] = write_jsonl(path, items)
|
||||
counts["train"] = write_jsonl(root / "train.jsonl", train)
|
||||
counts["search"] = write_jsonl(root / "search.jsonl", search)
|
||||
log(
|
||||
"split: "
|
||||
+ ", ".join(f"{k}={v}" for k, v in sorted(counts.items()))
|
||||
+ f" → {root}"
|
||||
)
|
||||
try:
|
||||
from gpu_rent.books import build_civitai_book
|
||||
|
||||
book_counts = build_civitai_book(out_root=out_root or app_root(), log=log)
|
||||
counts["book:civitai-krea2"] = book_counts.get("rows", 0)
|
||||
except GpuRentError as exc:
|
||||
log(f"book build warn: {exc}")
|
||||
return counts
|
||||
|
||||
|
||||
@@ -657,7 +639,7 @@ def cmd_all(
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(
|
||||
prog="python -m gpu_rent.civitai_dataset",
|
||||
description="Scrape Civitai Krea2 galleries → train.jsonl + search.jsonl",
|
||||
description="Scrape Civitai Krea2 galleries → search.jsonl + books/civitai-krea2",
|
||||
)
|
||||
p.add_argument(
|
||||
"--out",
|
||||
@@ -676,7 +658,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
s.add_argument("--per-version", type=int, default=DEFAULT_PER_VERSION)
|
||||
s.add_argument("--min-score", type=int, default=DEFAULT_MIN_SCORE)
|
||||
|
||||
sub.add_parser("split", help="Write by_kind / by_rating / train / search")
|
||||
sub.add_parser("split", help="Write by_kind / by_rating / search + civitai book")
|
||||
|
||||
a = sub.add_parser("all", help="discover + scrape + split (default)")
|
||||
a.add_argument("--top-checkpoints", type=int, default=DEFAULT_TOP_CHECKPOINTS)
|
||||
|
||||
@@ -1007,6 +1007,20 @@ def seed_extensions_cmd() -> None:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@app.command("seed-books")
|
||||
def seed_books_cmd(
|
||||
force: bool = typer.Option(False, "--force", help="Push all books even if sha unchanged"),
|
||||
) -> None:
|
||||
"""Push books/*/ → VM Assistent/books/ (changed only unless --force)."""
|
||||
try:
|
||||
from gpu_rent.provision import seed_books
|
||||
|
||||
cfg, host = _live()
|
||||
seed_books(cfg, host, log, force=force)
|
||||
except GpuRentError as exc:
|
||||
_die(exc)
|
||||
|
||||
|
||||
@app.command("seed-personas")
|
||||
def seed_personas_cmd() -> None:
|
||||
"""Push assistent-extensions/ → VM Assistent/extensions (+ _base)."""
|
||||
|
||||
@@ -525,6 +525,16 @@ def build_chat_trace(
|
||||
}
|
||||
if isinstance(response, dict) and response.get("civitai_results"):
|
||||
out["civitai_results"] = response.get("civitai_results")
|
||||
if isinstance(response, dict) and isinstance(response.get("knowledge"), dict):
|
||||
out["knowledge"] = response.get("knowledge")
|
||||
elif isinstance(response, dict) and response.get("civitai_results"):
|
||||
# Shim until swarm-assistent ships unified knowledge block
|
||||
out["knowledge"] = {
|
||||
"catalog": [],
|
||||
"hops": [],
|
||||
"results": response.get("civitai_results"),
|
||||
"legacy": "civitai_results",
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -72,6 +72,14 @@ def assistent_personas_manifest_path() -> Path:
|
||||
return app_root() / "assistent-personas.yaml"
|
||||
|
||||
|
||||
def books_dir() -> Path:
|
||||
return app_root() / "books"
|
||||
|
||||
|
||||
def books_example_dir() -> Path:
|
||||
return app_root() / "books.example"
|
||||
|
||||
|
||||
def assistent_personas_example_path() -> Path:
|
||||
"""Deprecated yaml example — prefer assistent_extensions_example_dir()."""
|
||||
return app_root() / "assistent-personas.example.yaml"
|
||||
|
||||
@@ -1141,11 +1141,81 @@ def seed_assistent_personas(cfg: Config, host: str, log: Log) -> None:
|
||||
f"assistent-extensions → packs/{pushed} "
|
||||
f"(default={default_id}{ctx_note})"
|
||||
)
|
||||
seed_civitai_examples(cfg, host, log)
|
||||
seed_books(cfg, host, log)
|
||||
|
||||
|
||||
def seed_books(cfg: Config, host: str, log: Log, *, force: bool = False) -> None:
|
||||
"""Push changed books/*/ → Assistent/books/ (content_sha diff)."""
|
||||
import json
|
||||
|
||||
from gpu_rent.books import (
|
||||
SIDEcar_NAME,
|
||||
VM_BOOKS_PREFIX,
|
||||
iter_installed_books,
|
||||
read_meta,
|
||||
remote_book_dir,
|
||||
seed_book_files,
|
||||
)
|
||||
from gpu_rent.paths import app_root
|
||||
from gpu_rent.ssh_ops import put_file, put_text
|
||||
|
||||
root = Path(getattr(cfg, "app_root", None) or app_root())
|
||||
book_ids = iter_installed_books(root)
|
||||
if not book_ids:
|
||||
seed_civitai_examples(cfg, host, log)
|
||||
return
|
||||
|
||||
run_ssh(cfg, host, f"mkdir -p {shlex.quote(VM_BOOKS_PREFIX)}", check=False)
|
||||
pushed = 0
|
||||
for book_id in book_ids:
|
||||
meta = read_meta(book_id, root)
|
||||
local_sha = str((meta or {}).get("content_sha") or "").strip()
|
||||
if not local_sha:
|
||||
log(f"books/{book_id}: нет meta.content_sha — skip")
|
||||
continue
|
||||
remote_dir = remote_book_dir(book_id)
|
||||
sidecar_remote = f"{remote_dir}/{SIDEcar_NAME}"
|
||||
remote_raw = run_ssh(
|
||||
cfg,
|
||||
host,
|
||||
f"test -f {shlex.quote(sidecar_remote)} && cat {shlex.quote(sidecar_remote)} || true",
|
||||
check=False,
|
||||
timeout=20,
|
||||
).strip()
|
||||
remote_sha = ""
|
||||
if remote_raw:
|
||||
try:
|
||||
remote_meta = json.loads(remote_raw)
|
||||
if isinstance(remote_meta, dict):
|
||||
remote_sha = str(remote_meta.get("content_sha") or "")
|
||||
except json.JSONDecodeError:
|
||||
remote_sha = ""
|
||||
|
||||
if not force and remote_sha == local_sha:
|
||||
log(f"books/{book_id}: unchanged")
|
||||
continue
|
||||
|
||||
run_ssh(cfg, host, f"mkdir -p {shlex.quote(remote_dir)}", check=False)
|
||||
local_dir = root / "books" / book_id
|
||||
for local_path, remote_path in seed_book_files(local_dir, remote_dir):
|
||||
put_file(cfg, host, local_path, remote_path)
|
||||
sidecar_payload = json.dumps(
|
||||
{"content_sha": local_sha, "book_id": book_id, **(meta or {})},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
put_text(cfg, host, sidecar_remote, sidecar_payload + "\n")
|
||||
size_kb = max(1, (local_dir / "search.jsonl").stat().st_size // 1024)
|
||||
log(f"books/{book_id} -> {remote_dir} ({size_kb} KB, sha={local_sha[:12]})")
|
||||
pushed += 1
|
||||
|
||||
if pushed:
|
||||
log(f"books: pushed {pushed}/{len(book_ids)}")
|
||||
elif book_ids:
|
||||
log(f"books: all {len(book_ids)} unchanged")
|
||||
|
||||
|
||||
def seed_civitai_examples(cfg: Config, host: str, log: Log) -> None:
|
||||
"""Push datasets/civitai/search.jsonl → Assistent/civitai-examples.jsonl (FTS, no embed)."""
|
||||
"""Legacy FTS path when books/ not built yet."""
|
||||
from gpu_rent.paths import app_root
|
||||
from gpu_rent.ssh_ops import put_file
|
||||
|
||||
|
||||
Reference in New Issue
Block a user