Leonid PershinandCursor
6dfbf4e702
Local civitai-dataset launchers collect ~2000 prompt/params rows without images; search.jsonl is pushed on up for cheap example lookup. Co-authored-by: Cursor <cursoragent@cursor.com>
99 lines
2.8 KiB
Bash
99 lines
2.8 KiB
Bash
#!/usr/bin/env bash
|
|
# Linux / macOS / Git Bash: same .venv as gpu-rent → python -m gpu_rent.civitai_dataset
|
|
# Default (no args): discover + scrape --target 2000 + split
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$ROOT"
|
|
|
|
load_vars_file() {
|
|
local file="$1"
|
|
[[ -f "$file" ]] || return 0
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
line="${line#"${line%%[![:space:]]*}"}"
|
|
line="${line%"${line##*[![:space:]]}"}"
|
|
[[ -z "$line" || "$line" == \#* ]] && continue
|
|
if [[ "$line" == [eE][xX][pP][oO][rR][tT][[:space:]]* ]]; then
|
|
line="${line#*[eE][xX][pP][oO][rR][tT]}"
|
|
line="${line#"${line%%[![:space:]]*}"}"
|
|
fi
|
|
[[ "$line" != *=* ]] && continue
|
|
local key="${line%%=*}"
|
|
local val="${line#*=}"
|
|
key="${key%"${key##*[![:space:]]}"}"
|
|
key="${key#"${key%%[![:space:]]*}"}"
|
|
val="${val#"${val%%[![:space:]]*}"}"
|
|
val="${val%"${val##*[![:space:]]}"}"
|
|
if [[ ${#val} -ge 2 ]]; then
|
|
local q="${val:0:1}"
|
|
if [[ ( "$q" == '"' || "$q" == "'" ) && "${val: -1}" == "$q" ]]; then
|
|
val="${val:1:${#val}-2}"
|
|
fi
|
|
fi
|
|
[[ -z "$key" ]] && continue
|
|
if [[ -z "${!key+x}" || -z "${!key}" ]]; then
|
|
export "$key=$val"
|
|
fi
|
|
done <"$file"
|
|
}
|
|
|
|
ok_py() {
|
|
local exe="$1"
|
|
shift || true
|
|
command -v "$exe" >/dev/null 2>&1 || return 1
|
|
"$exe" "$@" -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)" >/dev/null 2>&1
|
|
}
|
|
|
|
if ok_py python3.12; then
|
|
PY=(python3.12)
|
|
elif ok_py python3.11; then
|
|
PY=(python3.11)
|
|
elif ok_py python3; then
|
|
PY=(python3)
|
|
elif ok_py python; then
|
|
PY=(python)
|
|
elif ok_py py -3.12; then
|
|
PY=(py -3.12)
|
|
elif ok_py py -3.11; then
|
|
PY=(py -3.11)
|
|
else
|
|
echo "civitai-dataset: нужен Python 3.11+" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -x "$ROOT/.venv/Scripts/python.exe" ]]; then
|
|
VENV_PY="$ROOT/.venv/Scripts/python.exe"
|
|
elif [[ -x "$ROOT/.venv/bin/python" ]]; then
|
|
VENV_PY="$ROOT/.venv/bin/python"
|
|
else
|
|
echo "civitai-dataset: создаю .venv"
|
|
"${PY[@]}" -m venv "$ROOT/.venv"
|
|
if [[ -x "$ROOT/.venv/Scripts/python.exe" ]]; then
|
|
VENV_PY="$ROOT/.venv/Scripts/python.exe"
|
|
else
|
|
VENV_PY="$ROOT/.venv/bin/python"
|
|
fi
|
|
fi
|
|
|
|
STAMP="$ROOT/.venv/.gpu-rent-installed"
|
|
if [[ ! -f "$STAMP" || "$ROOT/pyproject.toml" -nt "$STAMP" ]]; then
|
|
echo "civitai-dataset: ставлю пакет в .venv"
|
|
"$VENV_PY" -m pip install -U pip
|
|
"$VENV_PY" -m pip install -e "$ROOT"
|
|
date -u +"%Y-%m-%dT%H:%M:%SZ" >"$STAMP"
|
|
fi
|
|
|
|
if [[ ! -f "$ROOT/.env" && -f "$ROOT/env.example" ]]; then
|
|
cp "$ROOT/env.example" "$ROOT/.env"
|
|
echo "civitai-dataset: created .env - fill CIVITAI_API_TOKEN"
|
|
fi
|
|
|
|
load_vars_file "$ROOT/gpu-rent.vars"
|
|
|
|
ARGS=("$@")
|
|
if [[ ${#ARGS[@]} -eq 0 ]]; then
|
|
ARGS=(all)
|
|
fi
|
|
|
|
exec "$VENV_PY" -m gpu_rent.civitai_dataset "${ARGS[@]}"
|