diff --git a/Assets/assistent.css b/Assets/assistent.css index df63fcd..8774d96 100644 --- a/Assets/assistent.css +++ b/Assets/assistent.css @@ -201,6 +201,11 @@ max-width: 12rem; } +.sa-header-right .sa-model-select { + min-width: 14rem; + max-width: 24rem; +} + .sa-check { flex-direction: row !important; align-items: center; @@ -279,6 +284,55 @@ margin-top: 0.45rem; } +.sa-danger { + opacity: 0.85; +} + +.sa-civitai-list { + display: flex; + flex-direction: column; + gap: 0.45rem; + margin-top: 0.55rem; +} + +.sa-civitai-card { + padding: 0.5rem 0.6rem; + border-radius: 0.45rem; + border: 1px solid color-mix(in srgb, currentColor 22%, transparent); + background: color-mix(in srgb, currentColor 5%, transparent); + font-size: 0.88rem; +} + +.sa-civitai-card.sa-installed { + opacity: 0.7; +} + +.sa-civitai-title { + font-weight: 600; + margin-bottom: 0.2rem; +} + +.sa-civitai-meta { + opacity: 0.8; + font-size: 0.82rem; + margin-bottom: 0.35rem; +} + +.sa-civitai-actions { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; +} + +.sa-streaming { + opacity: 0.9; + font-style: italic; +} + +.sa-interrupt-active { + border-color: color-mix(in srgb, #c44 55%, transparent) !important; +} + .sa-composer { border-top: 1px solid color-mix(in srgb, currentColor 18%, transparent); padding: 0.6rem 0.7rem 0.7rem; diff --git a/Assets/assistent.js b/Assets/assistent.js index da7152e..1eec03d 100644 --- a/Assets/assistent.js +++ b/Assets/assistent.js @@ -1,11 +1,16 @@ /** * Swarm Assistent — Krea 2 collaborative chat (Ollama via Swarm API). + * v0.3: inventory, streaming, auto-apply/generate, Civitai Confirm. */ (function () { const LS_BASE = 'swarm_assistent_base_url'; const LS_MODEL = 'swarm_assistent_model'; const LS_PACK = 'swarm_assistent_pack'; const LS_AUTO_VISION = 'swarm_assistent_auto_vision'; + const LS_AUTO_APPLY = 'swarm_assistent_auto_apply'; + const LS_AUTO_GENERATE = 'swarm_assistent_auto_generate'; + const LS_AUTO_CRITIQUE = 'swarm_assistent_auto_critique'; + const LS_AUTO_DOWNLOAD = 'swarm_assistent_auto_download'; const LS_PANE_WIDTH = 'swarm_assistent_pane_width'; const TAB_BUTTON_ID = 'maintab_assistent'; @@ -13,9 +18,13 @@ history: [], packsLoaded: false, busy: false, + generating: false, lastImageDataUrl: null, preferredModel: null, dragDepth: 0, + inventory: { loras: [], checkpoints: [], wildcards: [], has_civitai_key: false }, + streamEl: null, + critiqueHopUsed: false, }; function $(id) { @@ -29,12 +38,19 @@ } } + function setInterruptVisible(on) { + const btn = $('sa_btn_interrupt'); + if (btn) { + btn.hidden = !on; + btn.classList.toggle('sa-interrupt-active', !!on); + } + } + function looksLikeKrea(text) { const s = String(text || ''); return /krea\s*2|krea2|krea-2/i.test(s) || /krea/i.test(s); } - /** Resolve current checkpoint from modern SwarmUI APIs (getCurrentModel is gone). */ function resolveCurrentCheckpoint() { const out = { name: null, @@ -165,7 +181,6 @@ tab.click(); return true; } - // Fallback: try hash / pane const pane = document.getElementById('assistent'); if (pane && typeof bootstrap !== 'undefined' && bootstrap.Tab) { try { @@ -181,7 +196,6 @@ return; } frame.classList.remove('sa-flash'); - // reflow void frame.offsetWidth; frame.classList.add('sa-flash'); } @@ -246,7 +260,183 @@ } } + function countPromptImages() { + try { + const box = document.getElementById('alt_prompt_textbox') || document.getElementById('input_prompt'); + if (!box) { + return 0; + } + const text = box.value || ''; + const matches = text.match(/)/gi) || text.match(/data:image\//gi); + return matches ? matches.length : 0; + } catch (e) { + return 0; + } + } + + function triggerChangeForEl(el) { + if (!el) { + return; + } + if (typeof triggerChangeFor === 'function') { + triggerChangeFor(el); + return; + } + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + } + + function openInitImageGroup() { + try { + const initEl = document.getElementById('input_initimage'); + if (initEl && typeof toggleGroupOpen === 'function') { + toggleGroupOpen(initEl, true); + } + } catch (e) { /* ignore */ } + const toggler = document.getElementById('input_group_content_initimage_toggle'); + if (toggler) { + toggler.checked = true; + triggerChangeForEl(toggler); + } + } + + function hasFileParam(id) { + const el = document.getElementById(id); + return !!(el && el.files && el.files.length > 0); + } + + function clearFileParam(id) { + const el = document.getElementById(id); + if (!el) { + return false; + } + try { + el.value = ''; + if (el.files && typeof DataTransfer !== 'undefined') { + el.files = new DataTransfer().files; + } + } catch (e) { /* ignore */ } + triggerChangeForEl(el); + return true; + } + + function guessImageMime(src) { + const s = String(src || ''); + if (s.startsWith('data:image/')) { + const m = s.match(/^data:(image\/[a-z0-9.+-]+)/i); + return (m && m[1]) || 'image/png'; + } + const path = s.split('?')[0]; + const ext = path.substring(path.lastIndexOf('.') + 1).toLowerCase(); + if (ext === 'jpg' || ext === 'jpeg') { + return 'image/jpeg'; + } + if (ext === 'webp') { + return 'image/webp'; + } + if (ext === 'gif') { + return 'image/gif'; + } + return 'image/png'; + } + + async function srcToBlob(src) { + if (!src) { + return null; + } + const cleaned = String(src).trim().split(/\s+/)[0]; + if (cleaned.startsWith('data:') || cleaned.startsWith('/') || cleaned.startsWith('View/') || cleaned.startsWith('http')) { + try { + const resp = await fetch(cleaned); + return await resp.blob(); + } catch (e) { + console.warn('Assistent: fetch blob failed', e); + } + } + return await new Promise((resolve) => { + const tmpImg = new Image(); + tmpImg.crossOrigin = 'Anonymous'; + tmpImg.onload = () => { + try { + const canvas = document.createElement('canvas'); + canvas.width = tmpImg.naturalWidth; + canvas.height = tmpImg.naturalHeight; + const ctx = canvas.getContext('2d'); + ctx.drawImage(tmpImg, 0, 0); + canvas.toBlob((blob) => resolve(blob), 'image/png'); + } catch (e) { + resolve(null); + } + }; + tmpImg.onerror = () => resolve(null); + tmpImg.src = cleaned; + }); + } + + async function setFileParamFromSrc(paramId, src, { filename = 'assistent.png' } = {}) { + const el = document.getElementById(paramId); + if (!el) { + setStatus(`Missing ${paramId} on Generate tab`); + return false; + } + const blob = await srcToBlob(src); + if (!blob) { + setStatus('Could not load image for init/mask'); + return false; + } + const mime = blob.type || guessImageMime(src); + const file = new File([blob], filename, { type: mime }); + const container = new DataTransfer(); + container.items.add(file); + el.files = container.files; + triggerChangeForEl(el); + openInitImageGroup(); + return true; + } + + async function setInitFromSrc(src) { + const ok = await setFileParamFromSrc('input_initimage', src, { filename: 'assistent_init.png' }); + if (ok) { + setStatus('Init Image set'); + } + return ok; + } + + async function setMaskFromSrc(src) { + const ok = await setFileParamFromSrc('input_maskimage', src, { filename: 'assistent_mask.png' }); + if (ok) { + setStatus('Mask Image set (white = edit)'); + } + return ok; + } + + function clearInitAndMask() { + clearFileParam('input_initimage'); + clearFileParam('input_maskimage'); + const toggler = document.getElementById('input_group_content_initimage_toggle'); + if (toggler) { + toggler.checked = false; + triggerChangeForEl(toggler); + } + setStatus('Init Image + Mask cleared'); + } + + function readInitContext() { + const creativityRaw = val('input_initimagecreativity'); + const creativity = creativityRaw === '' ? null : parseFloat(creativityRaw); + return { + has_init_image: hasFileParam('input_initimage'), + has_mask_image: hasFileParam('input_maskimage'), + init_creativity: Number.isFinite(creativity) ? creativity : null, + mask_blur: parseFloat(val('input_maskblur') || '') || null, + mask_grow: parseInt(val('input_maskgrow') || val('input_maskshrinkgrow') || '', 10) || null, + init_group_on: !!(document.getElementById('input_group_content_initimage_toggle')?.checked), + }; + } + function collectLiveContext() { + const inv = state.inventory || {}; + const initCtx = readInitContext(); const ctx = { architecture_ok: isKreaSelected(), checkpoint: null, @@ -258,9 +448,18 @@ cfg: parseFloat(val('input_cfgscale') || val('input_cfg') || '') || null, sigma_shift: parseFloat(val('input_sigmashift') || '') || null, seed: val('input_seed') || null, + sampler: val('input_sampler') || val('input_samplerate') || null, + scheduler: val('input_scheduler') || null, + batch: parseInt(val('input_images') || val('input_batchsize') || '0', 10) || null, + prompt_image_count: countPromptImages(), selected_loras: [], - available_loras: [], + available_loras: (inv.loras || []).slice(0, 100), + wildcards: (inv.wildcards || []).map((w) => w.name || w).slice(0, 60), has_vision_image: !!state.lastImageDataUrl, + has_civitai_key: !!inv.has_civitai_key, + auto_apply: !!$('sa_auto_apply')?.checked, + auto_generate: !!$('sa_auto_generate')?.checked, + ...initCtx, }; try { @@ -283,47 +482,65 @@ } } catch (e) { /* ignore */ } - try { - const models = (typeof allModels !== 'undefined' && allModels) || (typeof model_list !== 'undefined' && model_list) || []; - const list = Array.isArray(models) ? models : Object.values(models || {}); - for (const m of list) { - if (!m) { - continue; - } - const folder = `${m.folder || m.path || ''}`; - const isLora = /lora/i.test(m.category || m.type || '') || (m.name && String(m.name).toLowerCase().includes('lora')); - const inLoraFolder = /lora/i.test(folder); - if (!(isLora || inLoraFolder)) { - if (!/lora/i.test(JSON.stringify(m).slice(0, 200))) { + // Fallback if inventory empty + if (!ctx.available_loras.length) { + try { + const models = (typeof allModels !== 'undefined' && allModels) || (typeof model_list !== 'undefined' && model_list) || []; + const list = Array.isArray(models) ? models : Object.values(models || {}); + for (const m of list) { + if (!m) { continue; } - } - ctx.available_loras.push({ - name: m.name || m.title, - title: m.title || m.name, - trigger_phrase: m.trigger_phrase || m.trigger || (m.metadata && (m.metadata.trigger_phrase || m.metadata.trigger)) || null, - architecture: m.architecture || null, - }); - } - if (ctx.available_loras.length > 80) { - ctx.available_loras = ctx.available_loras.slice(0, 80); - } - } catch (e) { /* ignore */ } - - try { - const sel = document.getElementById('input_loras'); - if (sel && sel.options && ctx.available_loras.length === 0) { - for (const opt of sel.options) { - if (opt.value) { - ctx.available_loras.push({ name: opt.value, title: opt.text || opt.value, trigger_phrase: null }); + const folder = `${m.folder || m.path || ''}`; + const isLora = /lora/i.test(m.category || m.type || '') || (m.name && String(m.name).toLowerCase().includes('lora')); + const inLoraFolder = /lora/i.test(folder); + if (!(isLora || inLoraFolder)) { + continue; } + ctx.available_loras.push({ + name: m.name || m.title, + title: m.title || m.name, + trigger_phrase: m.trigger_phrase || m.trigger || (m.metadata && (m.metadata.trigger_phrase || m.metadata.trigger)) || null, + architecture: m.architecture || null, + }); } - } - } catch (e) { /* ignore */ } + if (ctx.available_loras.length > 80) { + ctx.available_loras = ctx.available_loras.slice(0, 80); + } + } catch (e) { /* ignore */ } + } return ctx; } + function isPatchObject(obj) { + if (!obj || typeof obj !== 'object') { + return false; + } + return ( + obj.prompt != null || + obj.loras || + obj.width || + obj.height || + obj.steps || + obj.cfg || + obj.seed != null || + obj.sigma_shift != null || + obj.sampler || + obj.actions || + obj.search_query || + obj.civitai_query || + obj.use_init_image != null || + obj.clear_init_image != null || + obj.init_creativity != null || + obj.denoise != null || + obj.use_mask_image != null || + obj.clear_mask_image != null || + obj.mask_blur != null || + obj.mask_grow != null + ); + } + function extractPatch(text) { if (!text) { return { prose: text || '', patch: null }; @@ -336,7 +553,7 @@ const raw = match[1].trim(); try { const obj = JSON.parse(raw); - if (obj && typeof obj === 'object' && (obj.prompt != null || obj.loras || obj.width || obj.height || obj.steps || obj.cfg)) { + if (isPatchObject(obj)) { lastPatch = obj; prose = (text.slice(0, match.index) + text.slice(match.index + match[0].length)).trim(); } @@ -345,13 +562,39 @@ return { prose, patch: lastPatch }; } - function applyPatch(patch, which) { + function patchHasGenTrigger(patch) { + if (!patch) { + return false; + } + if (Array.isArray(patch.actions) && patch.actions.map(String).includes('generate')) { + return true; + } + return ( + patch.prompt != null || + patch.loras || + patch.width != null || + patch.height != null || + patch.steps != null || + patch.cfg != null || + patch.seed != null || + patch.sigma_shift != null || + patch.use_init_image || + patch.clear_init_image || + patch.init_creativity != null || + patch.denoise != null || + patch.use_mask_image || + patch.clear_mask_image + ); + } + + async function applyPatch(patch, which) { if (!patch) { return; } const doPrompt = !which || which === 'all' || which === 'prompt'; const doLoras = !which || which === 'all' || which === 'loras'; - const doSize = !which || which === 'all' || which === 'size'; + const doParams = !which || which === 'all' || which === 'size' || which === 'params'; + const doInit = !which || which === 'all' || which === 'params' || which === 'init'; if (doPrompt && patch.prompt != null) { const box = document.getElementById('alt_prompt_textbox') || document.getElementById('input_prompt'); @@ -405,7 +648,7 @@ } catch (e) { /* ignore */ } } - if (doSize) { + if (doParams) { if (patch.width != null) { setVal('input_width', String(patch.width)); } @@ -422,14 +665,172 @@ setVal('input_cfg', String(patch.cfg)); } } + if (patch.seed != null) { + setVal('input_seed', String(patch.seed)); + } + if (patch.sigma_shift != null) { + setVal('input_sigmashift', String(patch.sigma_shift)); + } + if (patch.sampler != null) { + if (document.getElementById('input_sampler')) { + setVal('input_sampler', String(patch.sampler)); + } + } + if (patch.scheduler != null && document.getElementById('input_scheduler')) { + setVal('input_scheduler', String(patch.scheduler)); + } + } + + if (doInit) { + const creativity = patch.init_creativity != null ? patch.init_creativity : patch.denoise; + if (creativity != null && document.getElementById('input_initimagecreativity')) { + setVal('input_initimagecreativity', String(creativity)); + openInitImageGroup(); + } + if (patch.mask_blur != null && document.getElementById('input_maskblur')) { + setVal('input_maskblur', String(patch.mask_blur)); + } + if (patch.mask_grow != null) { + if (document.getElementById('input_maskgrow')) { + setVal('input_maskgrow', String(patch.mask_grow)); + } else if (document.getElementById('input_maskshrinkgrow')) { + setVal('input_maskshrinkgrow', String(patch.mask_grow)); + } + } + if (patch.clear_init_image || patch.clear_mask_image) { + if (patch.clear_init_image) { + clearFileParam('input_initimage'); + } + if (patch.clear_mask_image) { + clearFileParam('input_maskimage'); + } + if (patch.clear_init_image && patch.clear_mask_image) { + const toggler = document.getElementById('input_group_content_initimage_toggle'); + if (toggler) { + toggler.checked = false; + triggerChangeForEl(toggler); + } + } + } + const src = state.lastImageDataUrl || findCurrentGenerateSrc(); + const wantInit = patch.use_init_image === true + || (Array.isArray(patch.actions) && patch.actions.map(String).includes('use_init')); + const wantMask = patch.use_mask_image === true + || (Array.isArray(patch.actions) && patch.actions.map(String).includes('use_mask')); + if (wantInit) { + if (src) { + await setInitFromSrc(src); + } else { + setStatus('No image for Init — drop/Send to Assistent first'); + } + } + if (wantMask) { + if (src) { + await setMaskFromSrc(src); + } else { + setStatus('No image for Mask — drop a mask (white=edit) first'); + } + } } setStatus('Applied patch'); } - function appendMessage(role, text, patch) { + function triggerGenerate() { + try { + if (typeof mainGenHandler !== 'undefined' && mainGenHandler && typeof mainGenHandler.doGenerate === 'function') { + mainGenHandler.doGenerate(); + return true; + } + } catch (e) { + console.warn('Assistent: doGenerate failed', e); + } + const btn = + document.getElementById('generate_button') || + document.getElementById('alt_generate_button') || + document.querySelector('button.generate-button') || + document.querySelector('#generate_button, button[id*="generate"]'); + if (btn) { + btn.click(); + return true; + } + return false; + } + + function doInterruptNow() { + try { + if (typeof doInterrupt === 'function') { + doInterrupt(false); + return; + } + } catch (e) { /* ignore */ } + if (typeof genericRequest === 'function') { + genericRequest('InterruptAll', { other_sessions: false }, () => {}, 0, () => {}); + } + } + + function waitForNewImage(prevSrc, timeoutMs = 180000) { + return new Promise((resolve) => { + const start = Date.now(); + const timer = setInterval(() => { + const src = findCurrentGenerateSrc(); + if (src && src !== prevSrc) { + clearInterval(timer); + resolve(src); + } else if (Date.now() - start > timeoutMs) { + clearInterval(timer); + resolve(null); + } + }, 400); + }); + } + + async function runGenerateFromPatch(patch) { + if (!$('sa_auto_generate')?.checked || !patchHasGenTrigger(patch)) { + return null; + } + const prev = findCurrentGenerateSrc(); + setStatus('Generating…'); + state.generating = true; + setInterruptVisible(true); + const ok = triggerGenerate(); + if (!ok) { + state.generating = false; + setInterruptVisible(state.busy); + setStatus('Could not start Generate (UI hook missing)'); + return null; + } + const src = await waitForNewImage(prev); + state.generating = false; + setInterruptVisible(state.busy); + if (src) { + setImageFromSrc(src, { note: 'Result → vision' }); + return src; + } + setStatus('Generate finished (no new image detected)'); + return null; + } + + async function maybeAutoCritique(imageSrc) { + if (!$('sa_auto_critique')?.checked || state.critiqueHopUsed || !imageSrc) { + return; + } + state.critiqueHopUsed = true; + const pack = $('sa_pack'); + if (pack) { + pack.value = 'critique_image'; + saveSettings(); + } + if ($('sa_input')) { + $('sa_input').value = 'Critique this result and improve the prompt for the next generation.'; + } + setStatus('Auto-critique…'); + await sendChat({ fromAutoCritique: true }); + } + + function appendMessage(role, text, patch, civitaiResults) { const box = $('sa_messages'); if (!box) { - return; + return null; } const div = document.createElement('div'); div.className = `sa-msg ${role}`; @@ -448,7 +849,7 @@ ['Apply all', 'all'], ['Prompt', 'prompt'], ['LoRAs', 'loras'], - ['Size', 'size'], + ['Params', 'params'], ]) { const btn = document.createElement('button'); btn.type = 'button'; @@ -457,11 +858,209 @@ btn.addEventListener('click', () => applyPatch(finalPatch, which)); actions.appendChild(btn); } + const genBtn = document.createElement('button'); + genBtn.type = 'button'; + genBtn.className = 'basic-button'; + genBtn.textContent = 'Apply + Generate'; + genBtn.addEventListener('click', async () => { + await applyPatch(finalPatch, 'all'); + await runGenerateFromPatch({ ...finalPatch, actions: ['generate'] }); + }); + actions.appendChild(genBtn); wrap.appendChild(actions); div.appendChild(wrap); } + if (civitaiResults && civitaiResults.length) { + div.appendChild(buildCivitaiCards(civitaiResults)); + } box.appendChild(div); box.scrollTop = box.scrollHeight; + return div; + } + + function beginStreamMessage() { + const box = $('sa_messages'); + if (!box) { + return null; + } + const div = document.createElement('div'); + div.className = 'sa-msg assistant sa-streaming'; + div.textContent = ''; + box.appendChild(div); + box.scrollTop = box.scrollHeight; + state.streamEl = div; + return div; + } + + function appendStreamDelta(delta) { + if (!state.streamEl) { + beginStreamMessage(); + } + if (state.streamEl) { + state.streamEl.textContent += delta; + const box = $('sa_messages'); + if (box) { + box.scrollTop = box.scrollHeight; + } + } + } + + function finalizeStreamMessage(fullReply, civitaiResults) { + const el = state.streamEl; + state.streamEl = null; + if (!el) { + appendMessage('assistant', fullReply, null, civitaiResults); + return; + } + el.classList.remove('sa-streaming'); + const { prose, patch } = extractPatch(fullReply); + el.textContent = prose || fullReply || ''; + if (patch) { + const wrap = document.createElement('div'); + wrap.className = 'sa-patch'; + const pre = document.createElement('pre'); + pre.textContent = JSON.stringify(patch, null, 2); + wrap.appendChild(pre); + const actions = document.createElement('div'); + actions.className = 'sa-patch-actions'; + for (const [label, which] of [ + ['Apply all', 'all'], + ['Prompt', 'prompt'], + ['LoRAs', 'loras'], + ['Params', 'params'], + ]) { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'basic-button'; + btn.textContent = label; + btn.addEventListener('click', () => applyPatch(patch, which)); + actions.appendChild(btn); + } + const genBtn = document.createElement('button'); + genBtn.type = 'button'; + genBtn.className = 'basic-button'; + genBtn.textContent = 'Apply + Generate'; + genBtn.addEventListener('click', async () => { + await applyPatch(patch, 'all'); + await runGenerateFromPatch({ ...patch, actions: ['generate'] }); + }); + actions.appendChild(genBtn); + wrap.appendChild(actions); + el.appendChild(wrap); + } + if (civitaiResults && civitaiResults.length) { + el.appendChild(buildCivitaiCards(civitaiResults)); + } + const box = $('sa_messages'); + if (box) { + box.scrollTop = box.scrollHeight; + } + } + + function buildCivitaiCards(results) { + const list = document.createElement('div'); + list.className = 'sa-civitai-list'; + for (const r of results) { + const card = document.createElement('div'); + card.className = 'sa-civitai-card' + (r.already_installed ? ' sa-installed' : ''); + const title = document.createElement('div'); + title.className = 'sa-civitai-title'; + title.textContent = r.name || r.file_name || 'LoRA'; + card.appendChild(title); + const meta = document.createElement('div'); + meta.className = 'sa-civitai-meta'; + const bits = [ + r.base_model || '?', + r.krea_likely ? 'Krea?' : null, + r.already_installed ? 'already installed' : null, + (r.triggers || []).slice(0, 3).join(', ') || null, + ].filter(Boolean); + meta.textContent = bits.join(' · '); + card.appendChild(meta); + const actions = document.createElement('div'); + actions.className = 'sa-civitai-actions'; + if (r.already_installed) { + const note = document.createElement('span'); + note.textContent = 'Installed'; + actions.appendChild(note); + } else if (r.download_url) { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'basic-button sa-primary'; + btn.textContent = 'Confirm download'; + btn.addEventListener('click', () => downloadCivitaiLoRA(r, btn)); + actions.appendChild(btn); + } else { + const note = document.createElement('span'); + note.textContent = 'No download URL'; + actions.appendChild(note); + } + card.appendChild(actions); + list.appendChild(card); + } + return list; + } + + function downloadCivitaiLoRA(card, btn) { + if (!card.download_url) { + return; + } + if (btn) { + btn.disabled = true; + btn.textContent = 'Downloading…'; + } + setStatus(`Downloading ${card.file_name || card.name}…`); + setInterruptVisible(true); + const payload = { + url: card.download_url, + type: 'LoRA', + name: card.file_name || card.name || 'lora', + }; + const onDone = (ok, msg) => { + setInterruptVisible(state.busy || state.generating); + if (ok) { + setStatus(`Downloaded ${payload.name}`); + if (btn) { + btn.textContent = 'Downloaded'; + } + refreshInventory(() => { + if ($('sa_input')) { + $('sa_input').value = `LoRA "${payload.name}" is now installed. Enable it with its triggers and improve the prompt.`; + } + sendChat({ fromDownload: true }); + }); + } else { + setStatus(msg || 'Download failed'); + if (btn) { + btn.disabled = false; + btn.textContent = 'Confirm download'; + } + appendMessage('error', msg || 'Download failed'); + } + }; + if (typeof makeWSRequest === 'function') { + makeWSRequest( + 'DoModelDownloadWS', + payload, + (data) => { + if (data.error) { + onDone(false, String(data.error)); + return; + } + if (data.success || data.overall_percent >= 1 || data.current_percent >= 1) { + if (data.success || data.overall_percent >= 0.99) { + onDone(true); + } else if (data.current_percent != null) { + setStatus(`Download ${(data.current_percent * 100).toFixed(0)}%`); + } + } + }, + 0, + (err) => onDone(false, String(err || 'Download failed')), + ); + } else { + onDone(false, 'makeWSRequest unavailable'); + } } function findCurrentGenerateSrc() { @@ -530,7 +1129,6 @@ return setImageFromSrc(first, { note: 'Image from drag' }); } } - // HTML img drag const html = dt.getData('text/html') || ''; const m = html.match(/src=["']([^"']+)["']/i); if (m && m[1]) { @@ -571,6 +1169,10 @@ const model = localStorage.getItem(LS_MODEL); const pack = localStorage.getItem(LS_PACK); const auto = localStorage.getItem(LS_AUTO_VISION); + const autoApply = localStorage.getItem(LS_AUTO_APPLY); + const autoGen = localStorage.getItem(LS_AUTO_GENERATE); + const autoCrit = localStorage.getItem(LS_AUTO_CRITIQUE); + const autoDl = localStorage.getItem(LS_AUTO_DOWNLOAD); const paneW = localStorage.getItem(LS_PANE_WIDTH); if (base && $('sa_base_url')) { $('sa_base_url').value = base; @@ -581,6 +1183,18 @@ if (auto != null && $('sa_auto_vision')) { $('sa_auto_vision').checked = auto === '1'; } + if ($('sa_auto_apply')) { + $('sa_auto_apply').checked = autoApply == null ? true : autoApply === '1'; + } + if ($('sa_auto_generate')) { + $('sa_auto_generate').checked = autoGen == null ? true : autoGen === '1'; + } + if ($('sa_auto_critique') && autoCrit != null) { + $('sa_auto_critique').checked = autoCrit === '1'; + } + if ($('sa_auto_download') && autoDl != null) { + $('sa_auto_download').checked = autoDl === '1'; + } if (model) { state.preferredModel = model; } @@ -594,42 +1208,134 @@ localStorage.setItem(LS_MODEL, $('sa_model')?.value || ''); localStorage.setItem(LS_PACK, $('sa_pack')?.value || 'write_prompt'); localStorage.setItem(LS_AUTO_VISION, $('sa_auto_vision')?.checked ? '1' : '0'); + localStorage.setItem(LS_AUTO_APPLY, $('sa_auto_apply')?.checked ? '1' : '0'); + localStorage.setItem(LS_AUTO_GENERATE, $('sa_auto_generate')?.checked ? '1' : '0'); + localStorage.setItem(LS_AUTO_CRITIQUE, $('sa_auto_critique')?.checked ? '1' : '0'); + localStorage.setItem(LS_AUTO_DOWNLOAD, $('sa_auto_download')?.checked ? '1' : '0'); + } + + function setModelOptions(models, { error } = {}) { + const sel = $('sa_model'); + if (!sel) { + return; + } + const names = (models || []).map((n) => String(n || '').trim()).filter(Boolean); + sel.innerHTML = ''; + if (error) { + const opt = document.createElement('option'); + opt.value = ''; + opt.textContent = `⚠ ${String(error).replace(/\s+/g, ' ').slice(0, 90)}`; + sel.appendChild(opt); + sel.disabled = true; + return; + } + sel.disabled = false; + if (!names.length) { + const opt = document.createElement('option'); + opt.value = ''; + opt.textContent = 'No Ollama models — pull / Refresh'; + sel.appendChild(opt); + return; + } + for (const name of names) { + const opt = document.createElement('option'); + opt.value = name; + opt.textContent = name; + sel.appendChild(opt); + } + const prefer = state.preferredModel || localStorage.getItem(LS_MODEL); + if (prefer && names.includes(prefer)) { + sel.value = prefer; + } } function refreshModels() { const baseUrl = $('sa_base_url')?.value || 'http://127.0.0.1:11434'; setStatus('Loading models…'); - genericRequest('AssistentListModels', { baseUrl }, (data) => { - if (data.error) { - setStatus(data.error); - appendMessage('error', data.error); - return; - } - const sel = $('sa_model'); - if (!sel) { - return; - } - sel.innerHTML = ''; - const models = data.models || []; - for (const name of models) { - if (!name) { - continue; + if (typeof genericRequest !== 'function') { + setStatus('SwarmUI API not ready'); + setModelOptions([], { error: 'SwarmUI API not ready' }); + return; + } + genericRequest( + 'AssistentListModels', + { baseUrl }, + (data) => { + const models = data.models || []; + setModelOptions(models); + const prefer = state.preferredModel || localStorage.getItem(LS_MODEL); + if (prefer && models.includes(prefer) && $('sa_model')) { + $('sa_model').value = prefer; } - const opt = document.createElement('option'); - opt.value = name; - opt.textContent = name; - sel.appendChild(opt); - } - const prefer = state.preferredModel || localStorage.getItem(LS_MODEL); - if (prefer && models.includes(prefer)) { - sel.value = prefer; - } - setStatus(models.length ? `${models.length} models` : 'No Ollama models'); - saveSettings(); - }); + setStatus(models.length ? `${models.length} models` : 'No Ollama models (gpu-rent: ollama pull)'); + saveSettings(); + }, + 0, + (err) => { + const msg = String(err || 'Ollama unreachable'); + setStatus(msg); + setModelOptions([], { error: msg }); + appendMessage('error', msg); + }, + ); } - async function sendChat() { + function refreshInventory(done) { + if (typeof genericRequest !== 'function') { + if (done) { + done(); + } + return; + } + genericRequest( + 'AssistentListInventory', + {}, + (data) => { + state.inventory = { + loras: data.loras || [], + checkpoints: data.checkpoints || [], + wildcards: data.wildcards || [], + has_civitai_key: !!data.has_civitai_key, + }; + setStatus(`Inventory: ${state.inventory.loras.length} LoRAs, ${state.inventory.wildcards.length} wildcards`); + if (done) { + done(); + } + }, + 0, + (err) => { + console.warn('Assistent inventory', err); + if (done) { + done(); + } + }, + ); + } + + async function handleReplySideEffects(reply, civitaiResults, { fromAutoCritique } = {}) { + const { patch } = extractPatch(reply); + if (Array.isArray(patch?.actions) && patch.actions.map(String).includes('interrupt')) { + doInterruptNow(); + } + if (civitaiResults && civitaiResults.length && $('sa_auto_download')?.checked) { + const pick = civitaiResults.find((r) => !r.already_installed && r.download_url && r.krea_likely) + || civitaiResults.find((r) => !r.already_installed && r.download_url); + if (pick) { + downloadCivitaiLoRA(pick, null); + } + } + if (patch && $('sa_auto_apply')?.checked) { + await applyPatch(patch, 'all'); + if (!fromAutoCritique) { + const src = await runGenerateFromPatch(patch); + if (src) { + await maybeAutoCritique(src); + } + } + } + } + + async function sendChat(opts = {}) { if (state.busy) { return; } @@ -662,6 +1368,10 @@ } } + if (!opts.fromAutoCritique && !opts.fromDownload) { + state.critiqueHopUsed = false; + } + state.history.push({ role: 'user', content: text }); appendMessage('user', text); $('sa_input').value = ''; @@ -674,35 +1384,122 @@ } state.busy = true; + setInterruptVisible(true); setStatus('Thinking…'); saveSettings(); + const baseUrl = $('sa_base_url')?.value || 'http://127.0.0.1:11434'; + // Flat fields: SwarmUI JObject params receive the whole request body. const payload = { - baseUrl: $('sa_base_url')?.value || 'http://127.0.0.1:11434', + baseUrl, model, pack, includeBase: true, + messages, + context_json: JSON.stringify(context), raw: { messages, context_json: JSON.stringify(context), pack, - base_url: $('sa_base_url')?.value || 'http://127.0.0.1:11434', + base_url: baseUrl, model, }, }; - genericRequest('AssistentChat', payload, (data) => { + const finishOk = async (reply, civitaiResults) => { state.busy = false; - if (data.error) { - setStatus(data.error); - appendMessage('error', data.error); - return; - } - const reply = data.reply || ''; + setInterruptVisible(state.generating); state.history.push({ role: 'assistant', content: reply }); - appendMessage('assistant', reply); setStatus('Done'); - }); + await handleReplySideEffects(reply, civitaiResults, opts); + }; + + const finishErr = (msg) => { + state.busy = false; + setInterruptVisible(state.generating); + setStatus(msg); + if (state.streamEl) { + state.streamEl.classList.remove('sa-streaming'); + state.streamEl.classList.add('error'); + state.streamEl.textContent = msg; + state.streamEl = null; + } else { + appendMessage('error', msg); + } + }; + + if (typeof makeWSRequest === 'function') { + beginStreamMessage(); + makeWSRequest( + 'AssistentChatWS', + payload, + (data) => { + if (data.error) { + finishErr(String(data.error)); + return; + } + if (data.clear_stream) { + if (state.streamEl) { + state.streamEl.textContent = ''; + } + setStatus(data.notice || 'Refining…'); + return; + } + if (data.delta) { + appendStreamDelta(data.delta); + setStatus('Thinking…'); + return; + } + if (data.done || data.reply != null) { + const reply = data.reply || (state.streamEl && state.streamEl.textContent) || ''; + const civitai = data.civitai_results || []; + finalizeStreamMessage(reply, civitai); + finishOk(reply, civitai); + } + }, + 0, + (err) => { + // Fallback to HTTP AssistentChat + console.warn('AssistentChatWS failed, falling back', err); + if (state.streamEl) { + state.streamEl.remove(); + state.streamEl = null; + } + genericRequest( + 'AssistentChat', + payload, + (data) => { + if (data.error) { + finishErr(String(data.error)); + return; + } + const reply = data.reply || ''; + appendMessage('assistant', reply, null, data.civitai_results || []); + finishOk(reply, data.civitai_results || []); + }, + 0, + (err2) => finishErr(String(err2 || err || 'Chat failed')), + ); + }, + ); + return; + } + + genericRequest( + 'AssistentChat', + payload, + (data) => { + if (data.error) { + finishErr(String(data.error)); + return; + } + const reply = data.reply || ''; + appendMessage('assistant', reply, null, data.civitai_results || []); + finishOk(reply, data.civitai_results || []); + }, + 0, + (err) => finishErr(String(err || 'Chat failed')), + ); } function wireDropZone() { @@ -745,7 +1542,6 @@ await handleDropDataTransfer(e.dataTransfer); }); - // Also accept drops on the whole layout (easier target) const layout = $('sa_layout'); layout?.addEventListener('dragover', (e) => { if (e.dataTransfer?.types?.includes('Files') || e.dataTransfer?.types?.includes('text/uri-list')) { @@ -760,31 +1556,12 @@ await handleDropDataTransfer(e.dataTransfer); }); - frame.addEventListener('paste', async (e) => { - const items = e.clipboardData?.items; - if (!items) { - return; - } - for (const item of items) { - if (item.type.startsWith('image/')) { - e.preventDefault(); - const file = item.getAsFile(); - if (file) { - await acceptImageFile(file); - } - return; - } - } - }); - - // Global paste when Assistent tab is visible document.addEventListener('paste', async (e) => { const pane = document.getElementById('assistent'); if (!pane || !pane.classList.contains('active')) { return; } if (e.target && (e.target.tagName === 'TEXTAREA' || e.target.tagName === 'INPUT')) { - // Still allow image paste over text fields when clipboard has image const items = e.clipboardData?.items; let hasImage = false; if (items) { @@ -855,11 +1632,9 @@ function registerSendButton() { if (typeof registerMediaButton !== 'function') { - // SwarmUI core not ready yet — retry briefly setTimeout(registerSendButton, 500); return; } - // Avoid duplicate registrations on hot-reload if (window.__swarmAssistentMediaRegistered) { return; } @@ -871,7 +1646,6 @@ switchTab: true, note: 'Image sent to Assistent', }); - // Prefer critique pack when coming from an image const pack = $('sa_pack'); if (pack && pack.value === 'write_prompt') { pack.value = 'critique_image'; @@ -889,10 +1663,19 @@ if (!$('swarm_assistent_root')) { return; } + if (typeof genericRequest !== 'function') { + setTimeout(wire, 300); + return; + } + if (window.__swarmAssistentWired) { + return; + } + window.__swarmAssistentWired = true; loadSettings(); updateGate(); refreshImagePreview(); refreshModels(); + refreshInventory(); wireDropZone(); wireSplitter(); registerSendButton(); @@ -907,6 +1690,7 @@ saveSettings(); refreshModels(); }); + $('sa_btn_refresh_inventory')?.addEventListener('click', () => refreshInventory()); $('sa_btn_use_current')?.addEventListener('click', () => { const src = findCurrentGenerateSrc(); if (src) { @@ -915,10 +1699,48 @@ setStatus('No current Generate image'); } }); + $('sa_btn_as_init')?.addEventListener('click', async () => { + const src = state.lastImageDataUrl || findCurrentGenerateSrc(); + if (!src) { + setStatus('No vision/Generate image for Init'); + return; + } + if (!state.lastImageDataUrl) { + setImageFromSrc(src); + } + await setInitFromSrc(src); + const pack = $('sa_pack'); + if (pack && pack.value === 'write_prompt') { + pack.value = 'inpaint_edit'; + saveSettings(); + } + }); + $('sa_btn_as_mask')?.addEventListener('click', async () => { + const src = state.lastImageDataUrl || findCurrentGenerateSrc(); + if (!src) { + setStatus('No vision image for Mask'); + return; + } + await setMaskFromSrc(src); + const pack = $('sa_pack'); + if (pack) { + pack.value = 'inpaint_edit'; + saveSettings(); + } + }); + $('sa_btn_clear_init')?.addEventListener('click', clearInitAndMask); $('sa_btn_clear_image')?.addEventListener('click', clearVisionImage); $('sa_btn_send')?.addEventListener('click', () => sendChat()); + $('sa_btn_interrupt')?.addEventListener('click', () => { + doInterruptNow(); + setStatus('Interrupted'); + state.busy = false; + state.generating = false; + setInterruptVisible(false); + }); $('sa_btn_clear')?.addEventListener('click', () => { state.history = []; + state.critiqueHopUsed = false; const box = $('sa_messages'); if (box) { box.innerHTML = ''; @@ -935,12 +1757,19 @@ $('sa_model')?.addEventListener('change', saveSettings); $('sa_pack')?.addEventListener('change', saveSettings); $('sa_auto_vision')?.addEventListener('change', saveSettings); + $('sa_auto_apply')?.addEventListener('change', saveSettings); + $('sa_auto_generate')?.addEventListener('change', saveSettings); + $('sa_auto_critique')?.addEventListener('change', saveSettings); + $('sa_auto_download')?.addEventListener('change', saveSettings); setInterval(updateGate, 2000); - // Soft sync from Generate only when we have no pinned vision image setInterval(() => refreshImagePreview({ onlyIfEmpty: true }), 5000); + setInterval(() => { + if (!state.busy) { + refreshInventory(); + } + }, 120000); - // Expose for console / other extensions window.swarmAssistent = { setImageFromSrc, clearVisionImage, @@ -948,6 +1777,12 @@ sendToAssistent: (src) => setImageFromSrc(src, { switchTab: true, note: 'Image sent to Assistent' }), isKreaSelected, resolveCurrentCheckpoint, + refreshInventory, + applyPatch, + triggerGenerate, + setInitFromSrc, + setMaskFromSrc, + clearInitAndMask, }; } diff --git a/Prompts/base_krea2.md b/Prompts/base_krea2.md index 3709796..2e58099 100644 --- a/Prompts/base_krea2.md +++ b/Prompts/base_krea2.md @@ -15,15 +15,18 @@ You are **Swarm Assistent**, a collaborative art director for **Krea 2** image g A JSON block named "Live SwarmUI context" is attached. Treat it as ground truth: -- Use only LoRAs listed in `available_loras` (by exact `name`). +- Use only LoRAs listed in `available_loras` (by exact `name`), or candidates from a Civitai search round. - Prefer listed `trigger_phrase` / `triggers` — **never invent** trigger words. - When enabling a LoRA, include its triggers in `prompt` if missing. -- Respect current width/height/steps/cfg unless the user asks to change them or the pack is `fix_params`. +- Respect current width/height/steps/cfg/seed/sigma_shift/sampler unless the user asks to change them or the pack is `fix_params`. +- `wildcards` lists installed wildcard names (`__name__` syntax in prompts). +- `prompt_image_count` > 0 means Prompt Images are attached — warn if they may dominate. +- **Init / inpaint:** `has_init_image`, `has_mask_image`, `init_creativity` (aka denoise, 0–1), `mask_blur`, `mask_grow`. `has_vision_image` is the Assistent pane reference (can become Init/Mask). ## Output contract (mandatory) 1. Write a short helpful reply in the user's language (RU or EN). -2. Then emit **one** fenced JSON patch (and only fields you want to change): +2. Then emit **one** fenced JSON patch (only fields you want to change): ```json { @@ -34,14 +37,45 @@ A JSON block named "Live SwarmUI context" is attached. Treat it as ground truth: "height": 1280, "steps": 8, "cfg": 1, + "seed": -1, + "sigma_shift": 1.15, + "sampler": null, + "use_init_image": false, + "clear_init_image": false, + "init_creativity": 0.45, + "use_mask_image": false, + "clear_mask_image": false, + "mask_blur": null, + "mask_grow": null, + "actions": ["generate"], + "search_query": null, "notes": "one-line why" } ``` -Rules for the patch: +### Patch rules - Omit keys you are not changing. - `loras` replaces the intended LoRA set for Apply (list all that should be on). - width/height between 128 and 4096; prefer multiples near 1024 for Turbo. - Do not invent model or LoRA filenames. - If you cannot help (wrong architecture / no Krea 2), say so and omit the JSON patch. + +### Init image / inpaint + +- **img2img:** set `use_init_image: true` (uses Assistent vision / current Generate image) and `init_creativity` (0 = almost copy, 1 = almost new). Typical edits: **0.25–0.45**; restyle: **0.5–0.7**. Alias `denoise` is accepted. +- **Inpaint:** needs Init + Mask. Set `use_init_image: true` and `use_mask_image: true` only when the vision pane holds a proper mask (white = edit, black = keep). If the user has not painted a mask, tell them to use Swarm **Edit Image** / paint a mask, or press **As Mask** with a prepared mask — do not invent pixel masks. +- `clear_init_image` / `clear_mask_image` to leave img2img mode. +- Prompt Images ≠ Init Image. Prefer Init for structural edits; Prompt Images for style refs (and warn they can dominate). + +### Actions (auto-safe) + +- `"generate"` — after Apply, start generation (UI auto-generate is on by default). +- `"use_init"` / `"use_mask"` — same as the boolean flags (optional). +- `"search_civitai"` — Civitai search; user must **Confirm** downloads. +- `"interrupt"` — stop generation. +- Pure Q&A with no prompt/param change: omit the JSON patch entirely (do not burn GPU). + +### Auto-apply note + +The UI may auto-apply your patch and auto-generate when `actions` contains `generate` or when you change prompt/loras/size/init. Keep patches intentional. diff --git a/Prompts/compose_scene.md b/Prompts/compose_scene.md index 3de266b..b185253 100644 --- a/Prompts/compose_scene.md +++ b/Prompts/compose_scene.md @@ -8,7 +8,9 @@ Goal: co-create a scene / moodboard direction for **Krea 2**. - Propose one strong prompt (not five weak ones). - Optionally suggest which available LoRAs fit — only from the live list, with triggers. - Mention Prompt Images only if a reference would help, and warn that refs can overpower text. +- Missing style LoRA → `search_civitai` + `search_query` (Krea-compatible). ## Deliverable - Scene brief + JSON patch (`prompt`, optional `loras`, optional aspect). +- Add `actions: ["generate"]` when ready to try the scene. diff --git a/Prompts/critique_image.md b/Prompts/critique_image.md index d1b5b0f..8163e15 100644 --- a/Prompts/critique_image.md +++ b/Prompts/critique_image.md @@ -9,8 +9,11 @@ Goal: look at the attached image (vision) and improve the next generation for ** - If a LoRA trigger was missing or too strong, adjust weight or prompt placement. - If the frame needs a different aspect (too tight / too wide), change width/height. - Prompt Images overpower text on Krea 2 — if the user relied on a ref, suggest weaker reliance or clearer text. +- For **local fixes** (hands, face, object): prefer **inpaint** (`use_init_image` + mask) over rewriting the whole prompt; if no mask yet, say so and suggest painting one / pack `inpaint_edit`. +- For **global restyle**: img2img with moderate `init_creativity` (≈0.4–0.6) can be better than from-scratch. ## Deliverable - Short critique in the user's language. -- JSON patch with improved `prompt` and any `loras` / size tweaks. +- JSON patch with improved `prompt` and any `loras` / size / init tweaks. +- Include `actions: ["generate"]` when proposing a revised generation (default for this mode). diff --git a/Prompts/fix_params.md b/Prompts/fix_params.md index 492340b..57d1c77 100644 --- a/Prompts/fix_params.md +++ b/Prompts/fix_params.md @@ -7,10 +7,14 @@ Goal: adjust **generation parameters** for Krea 2 Turbo (or Raw if context says - Turbo: prefer steps 4–12 (default 8), CFG ~1, sigma shift ~1.15. - Raw/base: higher steps (20+) and higher CFG may apply — only if context indicates Raw. - Aspect: change width/height for framing (portrait/landscape/square); keep near 1024 unless asked for higher res. +- Seed: set `seed` when the user wants reproducibility; `-1` for random. +- Sampler: only change if the form exposes it and the user asks. +- **Init creativity** (`init_creativity` / denoise 0–1): only when `has_init_image` or enabling img2img — see pack `inpaint_edit`. - Do not change the prompt unless needed to match the new framing. - Keep LoRAs unless the user asks to drop them. ## Deliverable - Explain the param change. -- JSON patch focusing on `width`, `height`, `steps`, `cfg` (and `prompt` only if necessary). +- JSON patch focusing on `width`, `height`, `steps`, `cfg`, `seed`, `sigma_shift`, optional `init_creativity` (and `prompt` only if necessary). +- Include `actions: ["generate"]` if the user wants to re-roll with the new params. diff --git a/Prompts/inpaint_edit.md b/Prompts/inpaint_edit.md new file mode 100644 index 0000000..ba05d64 --- /dev/null +++ b/Prompts/inpaint_edit.md @@ -0,0 +1,37 @@ +# Mode: inpaint_edit + +Goal: guide **img2img** (Init Image) and **inpainting** (Init + Mask) on **Krea 2**. + +## When to use which + +| Need | Setup | +| --- | --- | +| Soft edit / restyle whole frame | Init only + `init_creativity` | +| Change one region (face, hand, logo) | Init + Mask (white = edit) | +| Fresh image from text | Clear init/mask; normal t2i | + +## Creativity (denoise) + +- **0.2–0.35** — small fixes, keep composition +- **0.4–0.55** — noticeable edit, still related +- **0.6–0.8** — strong restyle; structure may drift +- Always set `use_init_image: true` when enabling img2img from the Assistent vision / current image. + +## Mask rules + +- White = regenerate, black = preserve. Gray = partial. +- Only set `use_mask_image: true` if context shows a vision image that is meant as a mask, or the user said they prepared one. +- If `has_mask_image` is false and the user wants regional edit: ask them to paint a mask in Swarm Image Editor (or Assistent **As Mask**), then continue. +- Optional: `mask_blur` / `mask_grow` for softer edges. + +## Prompting + +- Describe **what should appear in the edited region**, not the whole scene dump. +- Keep LoRA triggers if the subject depends on them. +- Match width/height to the init image when possible. + +## Deliverable + +- Short plan (img2img vs inpaint) in the user's language. +- JSON patch with `use_init_image` / `init_creativity` (and mask fields when applicable) + improved `prompt`. +- `actions: ["generate"]` when ready to run. diff --git a/Prompts/write_prompt.md b/Prompts/write_prompt.md index 5844280..6b6cc90 100644 --- a/Prompts/write_prompt.md +++ b/Prompts/write_prompt.md @@ -9,9 +9,11 @@ Goal: craft or improve a **Krea 2** prompt that will generate well on Turbo. - Prefer clarity over keyword stuffing. Krea 2 understands sentences. - If the user wants a style covered by an available LoRA, enable that LoRA and weave its triggers in. - Keep Turbo defaults unless the user asks otherwise (steps 8, cfg 1). +- If a needed LoRA is missing from `available_loras`, use `actions: ["search_civitai"]` with a clear `search_query` (and prefer Krea base). ## Deliverable - Explain briefly what you changed. - Emit a JSON patch with at least `prompt`, and `loras` when relevant. +- Include `actions: ["generate"]` when the user wants to see a new image. - Include `width`/`height` only if aspect should change for the scene (e.g. portrait → taller). diff --git a/README.md b/README.md index 409b966..9b7c90e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # Swarm Assistent -SwarmUI extension for **collaborative Krea 2** prompting via **Ollama**: chat + vision, LoRA/trigger awareness, and applyable prompt/size patches. +SwarmUI extension for **collaborative Krea 2** prompting via **Ollama**: chat + vision, LoRA/trigger awareness, applyable patches, **img2img / inpaint**, auto Generate, and Civitai search with Confirm. ## Layout - **Left:** vision reference (drop / paste / *Send to Assistent* from Generate) - **Splitter:** drag to resize panes - **Right (wider):** chat -- **Top-right:** prompt pack + settings (Ollama URL, model, auto-attach) +- **Top-right:** prompt pack + settings (Ollama URL, model, auto-apply / auto-generate) ## UX @@ -15,13 +15,19 @@ SwarmUI extension for **collaborative Krea 2** prompting via **Ollama**: chat + - Drag images from Generate/History or the OS onto the left pane - Paste (`Ctrl+V`) while the Assistent tab is open - **Use current** / **Clear** for the vision reference +- **As Init** / **As Mask** / **Clear Init** — wire the vision image into Swarm `Init Image` / `Mask Image` (img2img & inpaint) - Enter sends; Shift+Enter newline +- **Auto-apply** + **Auto-generate** (default on): patch from the LLM is applied and Generate runs when the patch changes prompt/params or includes `actions: ["generate"]` +- Pure Q&A without a patch does **not** start Generate +- **Interrupt** stops Swarm generation / clears busy state +- **Civitai** search cards require **Confirm download** (uses Swarm `DoModelDownloadWS` + stored `civitai_api` key). Auto-download is off by default. ## Requirements - SwarmUI with a **Krea 2** checkpoint selected -- Ollama on `http://127.0.0.1:11434` (gpu-rent default when `LLM_RUNTIME=ollama`) -- A chat+vision-capable Ollama model recommended for critique mode +- Ollama on `http://127.0.0.1:11434` **on the GPU VM** (gpu-rent `LLM_RUNTIME=ollama`). The browser talks to SwarmUI; SwarmUI proxies `/api/tags` and `/api/chat`. URL in settings must stay `127.0.0.1:11434`, not the laptop tunnel port 17811. +- At least one pulled model (`ollama pull` / `ollama-models.yaml`). Empty `/api/tags` → empty Model dropdown. +- Optional: Civitai API key in SwarmUI User Settings for search/download. ## Install @@ -41,13 +47,48 @@ Restart / rebuild SwarmUI after clone. | Pack | Role | | --- | --- | -| `base_krea2` | Always injected: Krea 2 rules + JSON patch contract | +| `base_krea2` | Always injected: Krea 2 rules + JSON patch / actions contract | | `write_prompt` | Craft / improve prompts | | `critique_image` | Vision critique → fixes | | `compose_scene` | Scene / moodboard | -| `fix_params` | Width/height/steps/CFG | +| `fix_params` | Width/height/steps/CFG/seed/σ-shift | +| `inpaint_edit` | Init Image img2img + Mask inpaint | -Live context (checkpoint, LoRAs + triggers, current params) is injected every request. +Live context (checkpoint, server inventory LoRAs + triggers, wildcards, current params) is injected every request. + +### Patch actions + +```json +{ + "prompt": "...", + "loras": [{"name": "exact", "weight": 0.8, "triggers": ["..."]}], + "width": 1024, "height": 1280, "steps": 8, "cfg": 1, + "seed": -1, "sigma_shift": 1.15, + "use_init_image": true, + "init_creativity": 0.45, + "use_mask_image": false, + "actions": ["generate"], + "search_query": null +} +``` + +- `generate` — auto-generate after apply (when enabled) +- `use_init` / `use_mask` — set Assistent vision (or current Generate) as Init / Mask +- `search_civitai` + `search_query` — server searches Civitai, second LLM hop, Confirm cards in UI +- `interrupt` — stop current generation + +Mask convention: **white = edit**, black = keep. Creativity ≈ denoise (0–1). + +## API routes + +| Route | Role | +| --- | --- | +| `AssistentListModels` | Ollama `/api/tags` | +| `AssistentListInventory` | LoRA / checkpoint / wildcard inventory from Swarm | +| `AssistentSearchCivitai` | Civitai LoRA search | +| `AssistentGetPacks` | Prompt pack texts | +| `AssistentChat` | HTTP chat (+ Civitai hop) | +| `AssistentChatWS` | Streaming chat WebSocket | ## License diff --git a/SwarmAssistentExtension.cs b/SwarmAssistentExtension.cs index 28584be..4efa5e3 100644 --- a/SwarmAssistentExtension.cs +++ b/SwarmAssistentExtension.cs @@ -3,18 +3,22 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.WebSockets; using System.Text; +using System.Text.RegularExpressions; using System.Threading.Tasks; using FreneticUtilities.FreneticExtensions; using Newtonsoft.Json.Linq; using SwarmUI.Accounts; using SwarmUI.Core; +using SwarmUI.Text2Image; using SwarmUI.Utils; using SwarmUI.WebAPI; namespace Mrleo1nid.SwarmAssistent; -/// Krea 2 collaborative assistant: Ollama chat + vision + prompt/LoRA/params patches. +/// Krea 2 collaborative assistant: Ollama chat + vision + prompt/LoRA/params patches + Generate/Civitai. public class SwarmAssistentExtension : Extension { public static PermInfo PermUse = Permissions.Register(new( @@ -33,17 +37,24 @@ public class SwarmAssistentExtension : Extension "critique_image", "compose_scene", "fix_params", + "inpaint_edit", ]; + const int MaxCivitaiHops = 2; + const int MaxLorasInInventory = 120; + const int MaxWildcardsInInventory = 80; + + static readonly Regex JsonFenceRe = new(@"```(?:json)?\s*([\s\S]*?)```", RegexOptions.IgnoreCase | RegexOptions.Compiled); + public override void OnPreInit() { ScriptFiles.Add("Assets/assistent.js"); StyleSheetFiles.Add("Assets/assistent.css"); ExtensionAuthor = "mrleo1nid"; - Description = "Collaborative Krea 2 assistant via Ollama: chat, vision, prompts, LoRA triggers, size patches."; + Description = "Collaborative Krea 2 assistant via Ollama: chat, vision, img2img/inpaint, Generate loop, Civitai Confirm."; License = "MIT"; - Version = "0.2.0"; - Tags = ["tabs", "ui", "llm", "ollama", "krea"]; + Version = "0.3.1"; + Tags = ["tabs", "ui", "llm", "ollama", "krea", "inpaint"]; } public override void OnInit() @@ -51,8 +62,11 @@ public class SwarmAssistentExtension : Extension HttpClient ??= new HttpClient { Timeout = TimeSpan.FromMinutes(10) }; API.RegisterAPICall(AssistentListModels, false, PermUse); API.RegisterAPICall(AssistentGetPacks, false, PermUse); + API.RegisterAPICall(AssistentListInventory, false, PermUse); + API.RegisterAPICall(AssistentSearchCivitai, false, PermUse); API.RegisterAPICall(AssistentChat, true, PermUse); - Logs.Init("Swarm Assistent extension loaded (Ollama proxy + Krea 2 packs)"); + API.RegisterAPICall(AssistentChatWS, true, PermUse); + Logs.Init("Swarm Assistent extension loaded (Ollama proxy + Krea 2 packs + inventory/Civitai)"); } static string Clip(string text, int max) @@ -128,24 +142,231 @@ public class SwarmAssistentExtension : Extension return new JObject { ["success"] = true, ["packs"] = packs, ["order"] = new JArray(PackNames) }; } - /// - /// Proxy to Ollama /api/chat (non-stream). - /// must include messages (JArray) and optional context_json. - /// - public async Task AssistentChat(Session session, string baseUrl, string model, string pack, bool includeBase, JObject raw) + /// Server-side LoRA / checkpoint / wildcard inventory (not DOM scrape). + public async Task AssistentListInventory(Session session) { - string root = NormalizeBaseUrl(baseUrl ?? raw?["base_url"]?.ToString()); - string modelName = (model ?? raw?["model"]?.ToString() ?? "").Trim(); - if (string.IsNullOrWhiteSpace(modelName)) + await Task.CompletedTask; + JArray loras = []; + JArray checkpoints = []; + JArray wildcards = []; + + if (Program.T2IModelSets.TryGetValue("LoRA", out T2IModelHandler loraHandler)) { - return new JObject { ["error"] = "model is required" }; - } - JArray userMessages = raw?["messages"] as JArray; - if (userMessages is null || userMessages.Count == 0) - { - return new JObject { ["error"] = "messages required" }; + foreach (T2IModel model in loraHandler.Models.Values.OrderBy(m => m.Name).Take(MaxLorasInInventory)) + { + loras.Add(new JObject + { + ["name"] = model.Name, + ["title"] = model.Metadata?.Title ?? model.Title ?? model.Name, + ["trigger_phrase"] = model.Metadata?.TriggerPhrase, + ["architecture"] = model.ModelClass?.ID, + ["compat_class"] = model.ModelClass?.CompatClass?.ID, + ["hash"] = model.Metadata?.Hash ?? "", + }); + } } + if (Program.T2IModelSets.TryGetValue("Stable-Diffusion", out T2IModelHandler ckptHandler)) + { + foreach (T2IModel model in ckptHandler.Models.Values.OrderBy(m => m.Name).Take(60)) + { + checkpoints.Add(new JObject + { + ["name"] = model.Name, + ["title"] = model.Metadata?.Title ?? model.Title ?? model.Name, + ["architecture"] = model.ModelClass?.ID, + ["compat_class"] = model.ModelClass?.CompatClass?.ID, + }); + } + } + + try + { + foreach (string name in WildcardsHelper.ListFiles.OrderBy(n => n).Take(MaxWildcardsInInventory)) + { + wildcards.Add(new JObject { ["name"] = name }); + } + } + catch (Exception ex) + { + Logs.Debug($"AssistentListInventory wildcards: {ex.Message}"); + } + + bool hasCivitaiKey = !string.IsNullOrWhiteSpace(session.User.GetGenericData("civitai_api", "key")); + + return new JObject + { + ["success"] = true, + ["loras"] = loras, + ["checkpoints"] = checkpoints, + ["wildcards"] = wildcards, + ["has_civitai_key"] = hasCivitaiKey, + }; + } + + /// Search Civitai for LoRAs (prefers Krea 2 base). Uses Swarm-stored civitai_api key. + public async Task AssistentSearchCivitai(Session session, string query, int limit = 8) + { + string q = (query ?? "").Trim(); + if (string.IsNullOrWhiteSpace(q)) + { + return new JObject { ["error"] = "query is required" }; + } + limit = Math.Clamp(limit, 1, 20); + string apiKey = session.User.GetGenericData("civitai_api", "key") ?? ""; + HashSet installedNames = CollectInstalledLoraNames(); + HashSet installedHashes = CollectInstalledLoraHashes(); + + string[] hosts = ["civitai.red", "civitai.com"]; + Exception lastEx = null; + foreach (string host in hosts) + { + try + { + string url = $"https://{host}/api/v1/models?limit={limit}&types=LORA&query={Uri.EscapeDataString(q)}"; + using HttpRequestMessage req = new(HttpMethod.Get, url); + if (!string.IsNullOrWhiteSpace(apiKey)) + { + req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey.Trim()); + } + using HttpResponseMessage resp = await HttpClient.SendAsync(req); + string body = await resp.Content.ReadAsStringAsync(); + if (!resp.IsSuccessStatusCode) + { + lastEx = new Exception($"HTTP {(int)resp.StatusCode}: {Clip(body, 200)}"); + continue; + } + JObject parsed = JObject.Parse(body); + JArray items = parsed["items"] as JArray ?? []; + JArray results = []; + foreach (JToken item in items) + { + if (item is not JObject mo) + { + continue; + } + JObject card = BuildCivitaiCard(mo, installedNames, installedHashes); + if (card is not null) + { + results.Add(card); + } + } + // Prefer Krea-compatible first + JArray sorted = new(results.OrderByDescending(t => LooksLikeKrea(t["base_model"]?.ToString())).ThenBy(t => t["name"]?.ToString())); + return new JObject + { + ["success"] = true, + ["query"] = q, + ["host"] = host, + ["results"] = sorted, + ["has_civitai_key"] = !string.IsNullOrWhiteSpace(apiKey), + }; + } + catch (Exception ex) + { + lastEx = ex; + } + } + return new JObject { ["error"] = $"Civitai search failed: {lastEx?.Message ?? "unknown"}" }; + } + + static bool LooksLikeKrea(string text) => !string.IsNullOrEmpty(text) && Regex.IsMatch(text, @"krea", RegexOptions.IgnoreCase); + + static HashSet CollectInstalledLoraNames() + { + HashSet names = new(StringComparer.OrdinalIgnoreCase); + if (!Program.T2IModelSets.TryGetValue("LoRA", out T2IModelHandler handler)) + { + return names; + } + foreach (T2IModel m in handler.Models.Values) + { + names.Add(m.Name); + string leaf = m.Name.Replace('\\', '/').AfterLast('/'); + if (!string.IsNullOrEmpty(leaf)) + { + names.Add(leaf); + names.Add(Path.GetFileNameWithoutExtension(leaf)); + } + } + return names; + } + + static HashSet CollectInstalledLoraHashes() + { + HashSet hashes = new(StringComparer.OrdinalIgnoreCase); + if (!Program.T2IModelSets.TryGetValue("LoRA", out T2IModelHandler handler)) + { + return hashes; + } + foreach (T2IModel m in handler.Models.Values) + { + string h = m.Metadata?.Hash; + if (!string.IsNullOrWhiteSpace(h)) + { + hashes.Add(h.Trim().ToLowerInvariant()); + } + } + return hashes; + } + + static JObject BuildCivitaiCard(JObject model, HashSet installedNames, HashSet installedHashes) + { + string name = model["name"]?.ToString() ?? ""; + JArray versions = model["modelVersions"] as JArray; + JObject ver = versions?.FirstOrDefault() as JObject; + if (ver is null) + { + return null; + } + string baseModel = ver["baseModel"]?.ToString() ?? ""; + JArray trained = ver["trainedWords"] as JArray ?? []; + List triggers = trained.Select(t => t?.ToString()).Where(s => !string.IsNullOrWhiteSpace(s)).Take(8).ToList(); + JObject file = null; + foreach (JToken f in ver["files"] as JArray ?? []) + { + if (f is JObject fo && (fo["primary"]?.Value() == true || (fo["name"]?.ToString() ?? "").EndsWith(".safetensors", StringComparison.OrdinalIgnoreCase))) + { + file = fo; + break; + } + } + file ??= (ver["files"] as JArray)?.FirstOrDefault() as JObject; + string downloadUrl = file?["downloadUrl"]?.ToString() ?? ver["downloadUrl"]?.ToString() ?? ""; + string fileName = file?["name"]?.ToString() ?? ""; + string sha = file?["hashes"]?["SHA256"]?.ToString() ?? file?["hashes"]?["AutoV2"]?.ToString() ?? ""; + string saveName = string.IsNullOrWhiteSpace(fileName) + ? Regex.Replace(name, @"[^\w\-.]+", "_").Trim('_') + : Path.GetFileNameWithoutExtension(fileName); + + bool already = false; + if (!string.IsNullOrWhiteSpace(sha) && installedHashes.Contains(sha.Trim().ToLowerInvariant())) + { + already = true; + } + else if (installedNames.Contains(saveName) || installedNames.Contains(name) || installedNames.Contains(fileName)) + { + already = true; + } + + return new JObject + { + ["id"] = model["id"], + ["version_id"] = ver["id"], + ["name"] = name, + ["base_model"] = baseModel, + ["krea_likely"] = LooksLikeKrea(baseModel), + ["triggers"] = new JArray(triggers), + ["download_url"] = downloadUrl, + ["file_name"] = saveName, + ["sha256"] = sha, + ["already_installed"] = already, + ["n_sfw"] = model["nsfw"]?.Value() ?? false, + }; + } + + List BuildOllamaMessages(string packName, bool includeBase, string contextJson, JArray userMessages, string extraSystem = null) + { List ollamaMessages = []; StringBuilder system = new(); if (includeBase) @@ -156,7 +377,6 @@ public class SwarmAssistentExtension : Extension system.AppendLine(basePack); } } - string packName = (pack ?? raw?["pack"]?.ToString() ?? "write_prompt").Trim(); if (!string.IsNullOrWhiteSpace(packName) && packName != "base_krea2") { string situational = ReadPackFile(packName); @@ -167,7 +387,6 @@ public class SwarmAssistentExtension : Extension system.AppendLine(situational); } } - string contextJson = raw?["context_json"]?.ToString(); if (!string.IsNullOrWhiteSpace(contextJson)) { system.AppendLine(); @@ -176,6 +395,11 @@ public class SwarmAssistentExtension : Extension system.AppendLine(contextJson); system.AppendLine("```"); } + if (!string.IsNullOrWhiteSpace(extraSystem)) + { + system.AppendLine(); + system.AppendLine(extraSystem); + } if (system.Length > 0) { ollamaMessages.Add(new JObject @@ -184,7 +408,7 @@ public class SwarmAssistentExtension : Extension ["content"] = system.ToString(), }); } - foreach (JToken msg in userMessages) + foreach (JToken msg in userMessages ?? []) { if (msg is not JObject mo) { @@ -201,24 +425,264 @@ public class SwarmAssistentExtension : Extension } ollamaMessages.Add(copy); } + return ollamaMessages; + } + static JObject TryParsePatch(string reply) + { + if (string.IsNullOrWhiteSpace(reply)) + { + return null; + } + foreach (Match match in JsonFenceRe.Matches(reply)) + { + string raw = match.Groups[1].Value.Trim(); + try + { + JObject obj = JObject.Parse(raw); + if (obj is not null && (obj["prompt"] != null || obj["loras"] != null || obj["width"] != null + || obj["height"] != null || obj["steps"] != null || obj["cfg"] != null + || obj["seed"] != null || obj["sigma_shift"] != null || obj["sampler"] != null + || obj["actions"] != null || obj["search_query"] != null || obj["civitai_query"] != null + || obj["use_init_image"] != null || obj["clear_init_image"] != null + || obj["init_creativity"] != null || obj["denoise"] != null + || obj["use_mask_image"] != null || obj["clear_mask_image"] != null + || obj["mask_blur"] != null || obj["mask_grow"] != null)) + { + return obj; + } + } + catch + { + // not json + } + } + return null; + } + + static string ExtractSearchQuery(JObject patch) + { + if (patch is null) + { + return null; + } + string q = (patch["search_query"] ?? patch["civitai_query"])?.ToString()?.Trim(); + if (!string.IsNullOrWhiteSpace(q)) + { + return q; + } + if (patch["actions"] is JArray acts) + { + foreach (JToken a in acts) + { + if (string.Equals(a?.ToString(), "search_civitai", StringComparison.OrdinalIgnoreCase)) + { + return q; // may still be null — caller checks + } + } + } + return null; + } + + static bool WantsCivitaiSearch(JObject patch) + { + if (patch is null) + { + return false; + } + if (!string.IsNullOrWhiteSpace(ExtractSearchQuery(patch))) + { + return true; + } + if (patch["actions"] is JArray acts) + { + foreach (JToken a in acts) + { + if (string.Equals(a?.ToString(), "search_civitai", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + } + return false; + } + + async Task<(string reply, JObject raw, JArray civitaiResults)> RunChatWithHops( + Session session, + string root, + string modelName, + string packName, + bool includeBase, + string contextJson, + JArray userMessages, + Func onDelta = null, + Func onHopStart = null) + { + List messages = BuildOllamaMessages(packName, includeBase, contextJson, userMessages); + JArray civitaiResults = []; + string reply = ""; + JObject lastRaw = null; + for (int hop = 0; hop < MaxCivitaiHops; hop++) + { + if (onHopStart is not null) + { + await onHopStart(hop); + } + (reply, lastRaw) = await CallOllamaChat(root, modelName, messages, stream: onDelta is not null, onDelta); + JObject patch = TryParsePatch(reply); + if (hop + 1 >= MaxCivitaiHops || !WantsCivitaiSearch(patch)) + { + break; + } + string query = ExtractSearchQuery(patch); + if (string.IsNullOrWhiteSpace(query)) + { + query = userMessages.LastOrDefault(m => m["role"]?.ToString() == "user")?["content"]?.ToString() ?? ""; + } + if (string.IsNullOrWhiteSpace(query)) + { + break; + } + JObject search = await AssistentSearchCivitai(session, query, 8); + if (search["error"] is not null) + { + messages.Add(new JObject { ["role"] = "assistant", ["content"] = reply }); + messages.Add(new JObject + { + ["role"] = "user", + ["content"] = $"Civitai search failed: {search["error"]}. Continue without download — use only available_loras from context.", + }); + continue; + } + civitaiResults = search["results"] as JArray ?? []; + messages.Add(new JObject { ["role"] = "assistant", ["content"] = reply }); + messages.Add(new JObject + { + ["role"] = "user", + ["content"] = + "Civitai search results (JSON). Prefer `krea_likely: true`. Do NOT download yourself — the UI shows Confirm cards. " + + "Pick useful LoRAs from results or available_loras, emit a normal patch (prompt/loras). " + + "Omit search_civitai from actions unless you need a different query.\n```json\n" + + civitaiResults.ToString(Newtonsoft.Json.Formatting.None) + "\n```", + }); + } + return (reply, lastRaw, civitaiResults); + } + + async Task<(string reply, JObject raw)> CallOllamaChat( + string root, + string modelName, + List ollamaMessages, + bool stream, + Func onDelta) + { JObject payload = new() { ["model"] = modelName, - ["stream"] = false, + ["stream"] = stream, ["messages"] = new JArray(ollamaMessages), }; - try + using StringContent content = new(payload.ToString(Newtonsoft.Json.Formatting.None), Encoding.UTF8, "application/json"); + using HttpRequestMessage req = new(HttpMethod.Post, $"{root}/api/chat") { Content = content }; + using HttpResponseMessage resp = await HttpClient.SendAsync(req, stream + ? HttpCompletionOption.ResponseHeadersRead + : HttpCompletionOption.ResponseContentRead); + if (!resp.IsSuccessStatusCode) + { + string errBody = await resp.Content.ReadAsStringAsync(); + throw new Exception($"Ollama /api/chat HTTP {(int)resp.StatusCode}: {Clip(errBody, 800)}"); + } + if (!stream) { - using StringContent content = new(payload.ToString(Newtonsoft.Json.Formatting.None), Encoding.UTF8, "application/json"); - using HttpResponseMessage resp = await HttpClient.PostAsync($"{root}/api/chat", content); string body = await resp.Content.ReadAsStringAsync(); - if (!resp.IsSuccessStatusCode) - { - return new JObject { ["error"] = $"Ollama /api/chat HTTP {(int)resp.StatusCode}: {Clip(body, 800)}" }; - } JObject parsed = JObject.Parse(body); string reply = parsed["message"]?["content"]?.ToString() ?? parsed["response"]?.ToString() ?? ""; + return (reply, parsed); + } + StringBuilder full = new(); + await using Stream streamBody = await resp.Content.ReadAsStreamAsync(); + using StreamReader reader = new(streamBody, Encoding.UTF8); + JObject last = null; + while (true) + { + string line = await reader.ReadLineAsync(); + if (line is null) + { + break; + } + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + JObject chunk = JObject.Parse(line); + last = chunk; + string delta = chunk["message"]?["content"]?.ToString() ?? ""; + if (!string.IsNullOrEmpty(delta)) + { + full.Append(delta); + if (onDelta is not null) + { + await onDelta(delta); + } + } + if (chunk["done"]?.Value() == true) + { + break; + } + } + return (full.ToString(), last ?? new JObject()); + } + + /// + /// SwarmUI passes the whole request as the JObject param (not only a nested key). + /// Support both flat fields and legacy nested raw. + /// + static void ExtractChatPayload(JObject raw, ref string baseUrl, ref string model, ref string pack, ref bool includeBase, out JArray userMessages, out string contextJson) + { + JObject whole = raw ?? []; + JObject nested = whole["raw"] as JObject; + if (string.IsNullOrWhiteSpace(baseUrl)) + { + baseUrl = whole["base_url"]?.ToString() + ?? whole["baseUrl"]?.ToString() + ?? nested?["base_url"]?.ToString() + ?? nested?["baseUrl"]?.ToString(); + } + if (string.IsNullOrWhiteSpace(model)) + { + model = whole["model"]?.ToString() ?? nested?["model"]?.ToString(); + } + if (string.IsNullOrWhiteSpace(pack)) + { + pack = whole["pack"]?.ToString() ?? nested?["pack"]?.ToString(); + } + if (whole["includeBase"] is not null) + { + includeBase = whole.Value("includeBase") ?? includeBase; + } + userMessages = (whole["messages"] as JArray) ?? (nested?["messages"] as JArray); + contextJson = whole["context_json"]?.ToString() ?? nested?["context_json"]?.ToString(); + } + + /// Proxy to Ollama /api/chat (non-stream), with optional Civitai search hop. + public async Task AssistentChat(Session session, string baseUrl, string model, string pack, bool includeBase, JObject raw) + { + ExtractChatPayload(raw, ref baseUrl, ref model, ref pack, ref includeBase, out JArray userMessages, out string contextJson); + string root = NormalizeBaseUrl(baseUrl); + string modelName = (model ?? "").Trim(); + if (string.IsNullOrWhiteSpace(modelName)) + { + return new JObject { ["error"] = "model is required" }; + } + if (userMessages is null || userMessages.Count == 0) + { + return new JObject { ["error"] = "messages required" }; + } + string packName = (pack ?? "write_prompt").Trim(); + try + { + (string reply, JObject parsed, JArray civitai) = await RunChatWithHops( + session, root, modelName, packName, includeBase, contextJson, userMessages); return new JObject { ["success"] = true, @@ -226,6 +690,7 @@ public class SwarmAssistentExtension : Extension ["model"] = modelName, ["pack"] = packName, ["raw"] = parsed, + ["civitai_results"] = civitai, }; } catch (Exception ex) @@ -233,4 +698,62 @@ public class SwarmAssistentExtension : Extension return new JObject { ["error"] = $"Ollama chat failed: {ex.Message}" }; } } + + /// WebSocket streaming chat (Ollama stream:true) + Civitai hops. + public async Task AssistentChatWS(Session session, WebSocket ws, string baseUrl, string model, string pack, bool includeBase, JObject raw) + { + ExtractChatPayload(raw, ref baseUrl, ref model, ref pack, ref includeBase, out JArray userMessages, out string contextJson); + string root = NormalizeBaseUrl(baseUrl); + string modelName = (model ?? "").Trim(); + if (string.IsNullOrWhiteSpace(modelName)) + { + await ws.SendJson(new JObject { ["error"] = "model is required" }, API.WebsocketTimeout); + return null; + } + if (userMessages is null || userMessages.Count == 0) + { + await ws.SendJson(new JObject { ["error"] = "messages required" }, API.WebsocketTimeout); + return null; + } + string packName = (pack ?? "write_prompt").Trim(); + try + { + async Task OnDelta(string delta) + { + if (ws.State == WebSocketState.Open) + { + await ws.SendJson(new JObject { ["delta"] = delta }, API.WebsocketTimeout); + } + } + async Task OnHopStart(int hop) + { + if (ws.State == WebSocketState.Open && hop > 0) + { + await ws.SendJson(new JObject + { + ["clear_stream"] = true, + ["hop"] = hop + 1, + ["notice"] = "Civitai search done — refining…", + }, API.WebsocketTimeout); + } + } + (string reply, JObject parsed, JArray civitai) = await RunChatWithHops( + session, root, modelName, packName, includeBase, contextJson, userMessages, OnDelta, OnHopStart); + await ws.SendJson(new JObject + { + ["success"] = true, + ["done"] = true, + ["reply"] = reply, + ["model"] = modelName, + ["pack"] = packName, + ["raw"] = parsed, + ["civitai_results"] = civitai, + }, API.WebsocketTimeout); + } + catch (Exception ex) + { + await ws.SendJson(new JObject { ["error"] = $"Ollama chat failed: {ex.Message}" }, API.WebsocketTimeout); + } + return null; + } } diff --git a/Tabs/Text2Image/Assistent.html b/Tabs/Text2Image/Assistent.html index b934645..dc256af 100644 --- a/Tabs/Text2Image/Assistent.html +++ b/Tabs/Text2Image/Assistent.html @@ -15,6 +15,9 @@
+ + +
@@ -29,23 +32,30 @@ + + +
- +
+