Leonid PershinandCursor
6dfbf4e702
Local civitai-dataset launchers collect ~2000 prompt/params rows without images; search.jsonl is pushed on up for cheap example lookup. Co-authored-by: Cursor <cursoragent@cursor.com>
123 lines
3.9 KiB
PowerShell
123 lines
3.9 KiB
PowerShell
#Requires -Version 5.1
|
|
# Windows launcher: same .venv as gpu-rent, runs python -m gpu_rent.civitai_dataset
|
|
# Default (no args): discover + scrape --target 2000 + split
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$Root = Split-Path -Parent $MyInvocation.MyCommand.Path
|
|
Set-Location -LiteralPath $Root
|
|
try {
|
|
$utf8 = New-Object System.Text.UTF8Encoding $false
|
|
[Console]::OutputEncoding = $utf8
|
|
$script:OutputEncoding = $utf8
|
|
} catch {
|
|
}
|
|
|
|
function Import-GpuRentVars {
|
|
param([Parameter(Mandatory = $true)][string]$Path)
|
|
if (-not (Test-Path -LiteralPath $Path)) { return }
|
|
Get-Content -LiteralPath $Path -Encoding UTF8 | ForEach-Object {
|
|
$line = $_.Trim()
|
|
if (-not $line -or $line.StartsWith("#")) { return }
|
|
if ($line -match '^(?i)export\s+') {
|
|
$line = $line.Substring($Matches[0].Length).Trim()
|
|
}
|
|
$eq = $line.IndexOf("=")
|
|
if ($eq -lt 1) { return }
|
|
$key = $line.Substring(0, $eq).Trim()
|
|
$val = $line.Substring($eq + 1).Trim()
|
|
if ($val.Length -ge 2) {
|
|
$q = $val[0]
|
|
if (($q -eq '"' -or $q -eq "'") -and $val[-1] -eq $q) {
|
|
$val = $val.Substring(1, $val.Length - 2)
|
|
}
|
|
}
|
|
if (-not $key) { return }
|
|
$existing = [Environment]::GetEnvironmentVariable($key, "Process")
|
|
if ([string]::IsNullOrEmpty($existing)) {
|
|
Set-Item -Path "Env:$key" -Value $val
|
|
}
|
|
}
|
|
}
|
|
|
|
function Test-Python311 {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Exe,
|
|
[string[]]$Prefix = @()
|
|
)
|
|
$cmd = Get-Command $Exe -ErrorAction SilentlyContinue
|
|
if (-not $cmd) {
|
|
return $false
|
|
}
|
|
try {
|
|
& $Exe @Prefix -c "import sys; raise SystemExit(0 if sys.version_info >= (3, 11) else 1)" 2>$null | Out-Null
|
|
return ($LASTEXITCODE -eq 0)
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
$PyExe = $null
|
|
$PyPrefix = @()
|
|
$found = $false
|
|
foreach ($row in @(
|
|
@{ Exe = "py"; Prefix = @("-3.12") },
|
|
@{ Exe = "py"; Prefix = @("-3.11") },
|
|
@{ Exe = "py"; Prefix = @("-3") },
|
|
@{ Exe = "python3"; Prefix = @() },
|
|
@{ Exe = "python"; Prefix = @() }
|
|
)) {
|
|
if (Test-Python311 -Exe $row.Exe -Prefix $row.Prefix) {
|
|
$PyExe = $row.Exe
|
|
$PyPrefix = $row.Prefix
|
|
$found = $true
|
|
break
|
|
}
|
|
}
|
|
|
|
if (-not $found) {
|
|
Write-Host "civitai-dataset: нужен Python 3.11+."
|
|
exit 1
|
|
}
|
|
|
|
$VenvPy = Join-Path $Root ".venv\Scripts\python.exe"
|
|
if (-not (Test-Path -LiteralPath $VenvPy)) {
|
|
Write-Host "civitai-dataset: создаю .venv"
|
|
& $PyExe @PyPrefix -m venv (Join-Path $Root ".venv")
|
|
}
|
|
|
|
$Stamp = Join-Path $Root ".venv\.gpu-rent-installed"
|
|
$Pyproject = Join-Path $Root "pyproject.toml"
|
|
$needInstall = -not (Test-Path -LiteralPath $Stamp)
|
|
if (-not $needInstall -and (Test-Path -LiteralPath $Pyproject)) {
|
|
$needInstall = (Get-Item -LiteralPath $Pyproject).LastWriteTime -gt (Get-Item -LiteralPath $Stamp).LastWriteTime
|
|
}
|
|
if ($needInstall) {
|
|
Write-Host "civitai-dataset: ставлю пакет в .venv"
|
|
& $VenvPy -m pip install -U pip
|
|
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
|
& $VenvPy -m pip install -e $Root
|
|
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
|
Get-Date -Format o | Set-Content -LiteralPath $Stamp -Encoding ascii
|
|
}
|
|
|
|
$EnvFile = Join-Path $Root ".env"
|
|
$Example = Join-Path $Root "env.example"
|
|
if (-not (Test-Path -LiteralPath $EnvFile)) {
|
|
if (Test-Path -LiteralPath $Example) {
|
|
Copy-Item -LiteralPath $Example -Destination $EnvFile
|
|
Write-Host "civitai-dataset: created .env - fill CIVITAI_API_TOKEN"
|
|
}
|
|
}
|
|
|
|
Import-GpuRentVars (Join-Path $Root "gpu-rent.vars")
|
|
|
|
$InvokeArgs = @($args)
|
|
if ($InvokeArgs.Count -eq 0) {
|
|
$InvokeArgs = @("all")
|
|
}
|
|
|
|
$ErrorActionPreference = "Continue"
|
|
& $VenvPy -m gpu_rent.civitai_dataset @InvokeArgs
|
|
exit $LASTEXITCODE
|