commit 5ce3fbf27d5c7b46e407c08ff83785ff3e2bcc10 Author: mrleo1nid Date: Fri Aug 21 05:52:43 2026 +0300 Initial Swarm Assistent extension for Krea 2 + Ollama diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8f27a56 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +bin/ +obj/ +.vs/ +*.user +*.suo +.DS_Store diff --git a/Assets/assistent.css b/Assets/assistent.css new file mode 100644 index 0000000..1e93c87 --- /dev/null +++ b/Assets/assistent.css @@ -0,0 +1,214 @@ +.swarm-assistent-root { + display: flex; + flex-direction: column; + height: calc(100vh - 8rem); + min-height: 28rem; + padding: 0.5rem; + box-sizing: border-box; +} + +.sa-gate { + padding: 1.25rem; + opacity: 0.9; +} + +.sa-layout { + display: flex; + flex: 1; + gap: 0.75rem; + min-height: 0; +} + +.sa-image-pane { + flex: 0 0 32%; + max-width: 22rem; + display: flex; + flex-direction: column; + gap: 0.5rem; + min-width: 12rem; +} + +.sa-image-frame { + flex: 1; + display: flex; + align-items: center; + justify-content: center; + border: 1px solid color-mix(in srgb, currentColor 25%, transparent); + border-radius: 0.35rem; + overflow: hidden; + background: color-mix(in srgb, currentColor 6%, transparent); + min-height: 14rem; +} + +.sa-image-frame img { + max-width: 100%; + max-height: 100%; + object-fit: contain; +} + +.sa-image-empty { + padding: 1rem; + text-align: center; + opacity: 0.65; + font-size: 0.95rem; +} + +.sa-image-actions { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + align-items: center; +} + +.sa-chat-pane { + flex: 1 1 68%; + display: flex; + flex-direction: column; + min-width: 0; + min-height: 0; + border: 1px solid color-mix(in srgb, currentColor 25%, transparent); + border-radius: 0.35rem; +} + +.sa-chat-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 0.5rem; + padding: 0.45rem 0.65rem; + border-bottom: 1px solid color-mix(in srgb, currentColor 20%, transparent); +} + +.sa-chat-title { + font-weight: 600; + letter-spacing: 0.02em; +} + +.sa-header-right { + display: flex; + align-items: center; + gap: 0.4rem; +} + +.sa-icon-btn { + min-width: 2rem; + padding-left: 0.45rem; + padding-right: 0.45rem; +} + +.sa-settings { + display: grid; + gap: 0.45rem; + padding: 0.65rem; + border-bottom: 1px solid color-mix(in srgb, currentColor 20%, transparent); + background: color-mix(in srgb, currentColor 5%, transparent); +} + +.sa-settings label { + display: flex; + flex-direction: column; + gap: 0.2rem; + font-size: 0.9rem; +} + +.sa-settings input[type="text"], +.sa-select { + width: 100%; + box-sizing: border-box; +} + +.sa-check { + flex-direction: row !important; + align-items: center; + gap: 0.4rem !important; +} + +.sa-messages { + flex: 1; + overflow: auto; + padding: 0.75rem; + display: flex; + flex-direction: column; + gap: 0.65rem; +} + +.sa-msg { + max-width: 95%; + padding: 0.55rem 0.7rem; + border-radius: 0.35rem; + white-space: pre-wrap; + word-break: break-word; + line-height: 1.35; +} + +.sa-msg.user { + align-self: flex-end; + background: color-mix(in srgb, currentColor 12%, transparent); +} + +.sa-msg.assistant { + align-self: flex-start; + background: color-mix(in srgb, currentColor 7%, transparent); +} + +.sa-msg.error { + align-self: stretch; + border: 1px solid color-mix(in srgb, #c44 50%, transparent); +} + +.sa-patch { + margin-top: 0.5rem; + padding-top: 0.45rem; + border-top: 1px dashed color-mix(in srgb, currentColor 25%, transparent); + font-size: 0.9rem; +} + +.sa-patch-actions { + display: flex; + flex-wrap: wrap; + gap: 0.35rem; + margin-top: 0.4rem; +} + +.sa-composer { + border-top: 1px solid color-mix(in srgb, currentColor 20%, transparent); + padding: 0.55rem; + display: flex; + flex-direction: column; + gap: 0.4rem; +} + +.sa-composer textarea { + width: 100%; + resize: vertical; + box-sizing: border-box; + min-height: 4rem; +} + +.sa-composer-actions { + display: flex; + flex-wrap: wrap; + gap: 0.4rem; + align-items: center; +} + +.sa-status { + font-size: 0.85rem; + opacity: 0.75; +} + +.sa-disabled { + opacity: 0.45; + pointer-events: none; +} + +@media (max-width: 900px) { + .sa-layout { + flex-direction: column; + } + .sa-image-pane { + flex: 0 0 auto; + max-width: none; + max-height: 40%; + } +} diff --git a/Assets/assistent.js b/Assets/assistent.js new file mode 100644 index 0000000..8dd5754 --- /dev/null +++ b/Assets/assistent.js @@ -0,0 +1,546 @@ +/** + * Swarm Assistent — Krea 2 collaborative chat (Ollama via Swarm API). + */ +(function () { + const LS_BASE = 'swarm_assistent_base_url'; + const LS_MODEL = 'swarm_assistent_model'; + const LS_PACK = 'swarm_assistent_pack'; + const LS_AUTO_VISION = 'swarm_assistent_auto_vision'; + + const state = { + history: [], + packsLoaded: false, + busy: false, + lastImageDataUrl: null, + }; + + function $(id) { + return document.getElementById(id); + } + + function setStatus(text) { + const el = $('sa_status'); + if (el) { + el.textContent = text || ''; + } + } + + function isKreaSelected() { + try { + const model = getCurrentModel && getCurrentModel(); + if (!model) { + return false; + } + const arch = `${model.architecture || ''} ${model.title || ''} ${model.name || ''} ${model.class || ''}`; + return /krea\s*2|krea2/i.test(arch) || /krea/i.test(arch); + } catch (e) { + return false; + } + } + + function updateGate() { + const ok = isKreaSelected(); + const gate = $('sa_gate'); + const layout = $('sa_layout'); + if (gate) { + gate.hidden = ok; + } + if (layout) { + layout.classList.toggle('sa-disabled', !ok); + } + return ok; + } + + function val(id) { + const el = document.getElementById(id); + return el ? el.value : ''; + } + + function setVal(id, value) { + const el = document.getElementById(id); + if (!el) { + return; + } + el.value = value; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + } + + function collectLiveContext() { + const ctx = { + architecture_ok: isKreaSelected(), + checkpoint: null, + prompt: val('alt_prompt_textbox') || val('input_prompt') || '', + negative: val('input_negativeprompt') || val('alt_negativeprompt_textbox') || '', + width: parseInt(val('input_width') || '0', 10) || null, + height: parseInt(val('input_height') || '0', 10) || null, + steps: parseInt(val('input_steps') || '0', 10) || null, + cfg: parseFloat(val('input_cfgscale') || val('input_cfg') || '') || null, + sigma_shift: parseFloat(val('input_sigmashift') || '') || null, + seed: val('input_seed') || null, + selected_loras: [], + available_loras: [], + has_vision_image: !!state.lastImageDataUrl, + }; + + try { + const model = getCurrentModel && getCurrentModel(); + if (model) { + ctx.checkpoint = { + name: model.name || model.title || null, + architecture: model.architecture || model.class || null, + title: model.title || null, + }; + } + } catch (e) { /* ignore */ } + + try { + if (typeof loraHelper !== 'undefined' && loraHelper && Array.isArray(loraHelper.selected)) { + ctx.selected_loras = loraHelper.selected.map((l) => ({ + name: l.name || l, + weight: (loraHelper.loraWeightPref && loraHelper.loraWeightPref[l.name || l]) || 1, + })); + } + } catch (e) { /* ignore */ } + + try { + const models = (typeof allModels !== 'undefined' && allModels) || (typeof model_list !== 'undefined' && model_list) || []; + const list = Array.isArray(models) ? models : Object.values(models || {}); + for (const m of list) { + if (!m) { + continue; + } + const arch = `${m.architecture || ''} ${m.class || ''} ${m.name || ''} ${m.title || ''}`; + const isLora = /lora/i.test(m.category || m.type || '') || (m.name && String(m.name).toLowerCase().includes('lora')); + const folder = `${m.folder || m.path || ''}`; + const inLoraFolder = /lora/i.test(folder); + if (!(isLora || inLoraFolder)) { + // Still include if metadata says lora + if (!/lora/i.test(JSON.stringify(m).slice(0, 200))) { + continue; + } + } + // Prefer Krea-tagged or unknown; skip obvious FLUX/SDXL-only names when tagged + ctx.available_loras.push({ + name: m.name || m.title, + title: m.title || m.name, + trigger_phrase: m.trigger_phrase || m.trigger || (m.metadata && (m.metadata.trigger_phrase || m.metadata.trigger)) || null, + architecture: m.architecture || null, + }); + } + // Cap list size for context window + if (ctx.available_loras.length > 80) { + ctx.available_loras = ctx.available_loras.slice(0, 80); + } + } catch (e) { /* ignore */ } + + // Fallback: parse multi-select input_loras options as available names + try { + const sel = document.getElementById('input_loras'); + if (sel && sel.options && ctx.available_loras.length === 0) { + for (const opt of sel.options) { + if (opt.value) { + ctx.available_loras.push({ name: opt.value, title: opt.text || opt.value, trigger_phrase: null }); + } + } + } + } catch (e) { /* ignore */ } + + return ctx; + } + + function extractPatch(text) { + if (!text) { + return { prose: text || '', patch: null }; + } + const re = /```(?:json)?\s*([\s\S]*?)```/gi; + let match; + let lastPatch = null; + let prose = text; + while ((match = re.exec(text)) !== null) { + const raw = match[1].trim(); + try { + const obj = JSON.parse(raw); + if (obj && typeof obj === 'object' && (obj.prompt != null || obj.loras || obj.width || obj.height || obj.steps || obj.cfg)) { + lastPatch = obj; + prose = (text.slice(0, match.index) + text.slice(match.index + match[0].length)).trim(); + } + } catch (e) { /* not json */ } + } + return { prose, patch: lastPatch }; + } + + function applyPatch(patch, which) { + if (!patch) { + return; + } + const doPrompt = !which || which === 'all' || which === 'prompt'; + const doLoras = !which || which === 'all' || which === 'loras'; + const doSize = !which || which === 'all' || which === 'size'; + + if (doPrompt && patch.prompt != null) { + const box = document.getElementById('alt_prompt_textbox') || document.getElementById('input_prompt'); + if (box) { + box.value = patch.prompt; + box.dispatchEvent(new Event('input', { bubbles: true })); + box.dispatchEvent(new Event('change', { bubbles: true })); + } + if (patch.negative != null) { + setVal('input_negativeprompt', patch.negative); + } + // Ensure triggers present + if (Array.isArray(patch.loras)) { + for (const l of patch.loras) { + const triggers = l.triggers || (l.trigger_phrase ? [l.trigger_phrase] : []); + for (const t of triggers) { + if (t && box && box.value && !box.value.includes(t)) { + box.value = `${box.value.trim()}, ${t}`; + box.dispatchEvent(new Event('input', { bubbles: true })); + } + } + } + } + } + + if (doLoras && Array.isArray(patch.loras) && typeof loraHelper !== 'undefined' && loraHelper) { + try { + if (typeof loraHelper.clearLoras === 'function') { + loraHelper.clearLoras(); + } + } catch (e) { /* ignore */ } + for (const l of patch.loras) { + const name = l.name; + if (!name) { + continue; + } + try { + if (typeof loraHelper.selectLora === 'function') { + loraHelper.selectLora(name); + } + if (loraHelper.loraWeightPref && l.weight != null) { + loraHelper.loraWeightPref[name] = l.weight; + } + } catch (e) { + console.warn('Assistent: selectLora failed', name, e); + } + } + try { + if (typeof loraHelper.rebuildUI === 'function') { + loraHelper.rebuildUI(); + } + } catch (e) { /* ignore */ } + } + + if (doSize) { + if (patch.width != null) { + setVal('input_width', String(patch.width)); + } + if (patch.height != null) { + setVal('input_height', String(patch.height)); + } + if (patch.steps != null) { + setVal('input_steps', String(patch.steps)); + } + if (patch.cfg != null) { + if (document.getElementById('input_cfgscale')) { + setVal('input_cfgscale', String(patch.cfg)); + } else { + setVal('input_cfg', String(patch.cfg)); + } + } + } + setStatus('Applied patch'); + } + + function appendMessage(role, text, patch) { + const box = $('sa_messages'); + if (!box) { + return; + } + const div = document.createElement('div'); + div.className = `sa-msg ${role}`; + const { prose, patch: extracted } = role === 'assistant' ? extractPatch(text) : { prose: text, patch: null }; + const finalPatch = patch || extracted; + div.textContent = prose || text || ''; + if (finalPatch) { + const wrap = document.createElement('div'); + wrap.className = 'sa-patch'; + const pre = document.createElement('pre'); + pre.textContent = JSON.stringify(finalPatch, null, 2); + wrap.appendChild(pre); + const actions = document.createElement('div'); + actions.className = 'sa-patch-actions'; + for (const [label, which] of [ + ['Apply all', 'all'], + ['Prompt', 'prompt'], + ['LoRAs', 'loras'], + ['Size', 'size'], + ]) { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'basic-button'; + btn.textContent = label; + btn.addEventListener('click', () => applyPatch(finalPatch, which)); + actions.appendChild(btn); + } + wrap.appendChild(actions); + div.appendChild(wrap); + } + box.appendChild(div); + box.scrollTop = box.scrollHeight; + } + + function refreshImagePreview() { + let src = null; + try { + const cur = document.getElementById('current_image_img') || document.querySelector('#current_image img') || document.querySelector('.current-image img'); + if (cur && cur.src) { + src = cur.src; + } + } catch (e) { /* ignore */ } + try { + if (!src && typeof currentMetadataMap !== 'undefined' && currentMetadataMap && currentMetadataMap.image) { + src = currentMetadataMap.image; + } + } catch (e) { /* ignore */ } + + const img = $('sa_image_preview'); + const empty = $('sa_image_empty'); + if (src) { + state.lastImageDataUrl = src; + if (img) { + img.src = src; + img.hidden = false; + } + if (empty) { + empty.hidden = true; + } + } else { + state.lastImageDataUrl = null; + if (img) { + img.hidden = true; + } + if (empty) { + empty.hidden = false; + } + } + } + + async function imageToBase64ForOllama(src) { + if (!src) { + return null; + } + // Already data URL + if (src.startsWith('data:')) { + const i = src.indexOf(','); + return i >= 0 ? src.slice(i + 1) : null; + } + try { + const resp = await fetch(src); + const blob = await resp.blob(); + return await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const data = String(reader.result || ''); + const i = data.indexOf(','); + resolve(i >= 0 ? data.slice(i + 1) : null); + }; + reader.onerror = reject; + reader.readAsDataURL(blob); + }); + } catch (e) { + console.warn('Assistent: vision fetch failed', e); + return null; + } + } + + function loadSettings() { + const base = localStorage.getItem(LS_BASE); + const model = localStorage.getItem(LS_MODEL); + const pack = localStorage.getItem(LS_PACK); + const auto = localStorage.getItem(LS_AUTO_VISION); + if (base && $('sa_base_url')) { + $('sa_base_url').value = base; + } + if (pack && $('sa_pack')) { + $('sa_pack').value = pack; + } + if (auto != null && $('sa_auto_vision')) { + $('sa_auto_vision').checked = auto === '1'; + } + if (model) { + state.preferredModel = model; + } + } + + function saveSettings() { + localStorage.setItem(LS_BASE, $('sa_base_url')?.value || ''); + localStorage.setItem(LS_MODEL, $('sa_model')?.value || ''); + localStorage.setItem(LS_PACK, $('sa_pack')?.value || 'write_prompt'); + localStorage.setItem(LS_AUTO_VISION, $('sa_auto_vision')?.checked ? '1' : '0'); + } + + function refreshModels() { + const baseUrl = $('sa_base_url')?.value || 'http://127.0.0.1:11434'; + setStatus('Loading models…'); + genericRequest('AssistentListModels', { baseUrl }, (data) => { + if (data.error) { + setStatus(data.error); + appendMessage('error', data.error); + return; + } + const sel = $('sa_model'); + if (!sel) { + return; + } + sel.innerHTML = ''; + const models = data.models || []; + for (const name of models) { + if (!name) { + continue; + } + const opt = document.createElement('option'); + opt.value = name; + opt.textContent = name; + sel.appendChild(opt); + } + const prefer = state.preferredModel || localStorage.getItem(LS_MODEL); + if (prefer && models.includes(prefer)) { + sel.value = prefer; + } + setStatus(models.length ? `${models.length} models` : 'No Ollama models'); + saveSettings(); + }); + } + + async function sendChat() { + if (state.busy) { + return; + } + if (!updateGate()) { + setStatus('Select a Krea 2 model'); + return; + } + const text = ($('sa_input')?.value || '').trim(); + if (!text) { + return; + } + const pack = $('sa_pack')?.value || 'write_prompt'; + const model = $('sa_model')?.value; + if (!model) { + setStatus('Pick an Ollama model in ⚙'); + refreshModels(); + return; + } + + refreshImagePreview(); + const attach = ($('sa_attach_vision')?.checked || $('sa_auto_vision')?.checked) && state.lastImageDataUrl; + let images = null; + if (attach) { + setStatus('Encoding image…'); + const b64 = await imageToBase64ForOllama(state.lastImageDataUrl); + if (b64) { + images = [b64]; + } + } + + const userMsg = { role: 'user', content: text }; + if (images) { + userMsg.images = images; + } + state.history.push({ role: 'user', content: text }); + appendMessage('user', text); + $('sa_input').value = ''; + + const context = collectLiveContext(); + context.has_vision_image = !!images; + // Strip huge fields from history replay — only send recent turns without images in history payload + const messages = state.history.slice(-12).map((m) => ({ role: m.role, content: m.content })); + // Last user message may include images + if (images && messages.length) { + messages[messages.length - 1].images = images; + } + + state.busy = true; + setStatus('Thinking…'); + saveSettings(); + + const payload = { + baseUrl: $('sa_base_url')?.value || 'http://127.0.0.1:11434', + model, + pack, + includeBase: true, + raw: { + messages, + context_json: JSON.stringify(context), + pack, + base_url: $('sa_base_url')?.value || 'http://127.0.0.1:11434', + model, + }, + }; + + genericRequest('AssistentChat', payload, (data) => { + state.busy = false; + if (data.error) { + setStatus(data.error); + appendMessage('error', data.error); + return; + } + const reply = data.reply || ''; + state.history.push({ role: 'assistant', content: reply }); + appendMessage('assistant', reply); + setStatus('Done'); + }); + } + + function wire() { + if (!$('swarm_assistent_root')) { + return; + } + loadSettings(); + updateGate(); + refreshImagePreview(); + refreshModels(); + + $('sa_btn_settings')?.addEventListener('click', () => { + const s = $('sa_settings'); + if (s) { + s.hidden = !s.hidden; + } + }); + $('sa_btn_refresh_models')?.addEventListener('click', () => { + saveSettings(); + refreshModels(); + }); + $('sa_btn_refresh_image')?.addEventListener('click', refreshImagePreview); + $('sa_btn_send')?.addEventListener('click', () => sendChat()); + $('sa_btn_clear')?.addEventListener('click', () => { + state.history = []; + const box = $('sa_messages'); + if (box) { + box.innerHTML = ''; + } + setStatus(''); + }); + $('sa_input')?.addEventListener('keydown', (e) => { + if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) { + e.preventDefault(); + sendChat(); + } + }); + $('sa_base_url')?.addEventListener('change', saveSettings); + $('sa_model')?.addEventListener('change', saveSettings); + $('sa_pack')?.addEventListener('change', saveSettings); + $('sa_auto_vision')?.addEventListener('change', saveSettings); + + // Re-check Krea gate when user may swap models + setInterval(updateGate, 2000); + setInterval(refreshImagePreview, 4000); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', wire); + } else { + wire(); + } +})(); diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c7c596e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 mrleo1nid + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Prompts/base_krea2.md b/Prompts/base_krea2.md new file mode 100644 index 0000000..3709796 --- /dev/null +++ b/Prompts/base_krea2.md @@ -0,0 +1,47 @@ +# Base: Krea 2 + Swarm Assistent + +You are **Swarm Assistent**, a collaborative art director for **Krea 2** image generation inside SwarmUI. + +## Model facts (do not contradict) + +- Architecture: Krea 2 (12B DiT). Not FLUX, not SDXL, not FLUX.1-Krea. +- Text encoder: Qwen3-VL 4B. VAE: Qwen Image VAE. +- **Turbo** defaults: steps **8**, CFG **1**, sigma shift **1.15**, side length ~**1024**. +- **Prompt Images** (refs in the prompt box) often **overpower** the text prompt — suggest them sparingly and warn the user. +- Built-in NSFW text-refiner may strip risque words; LoRAs may change that — do not lecture; stay practical. +- LoRAs: **only Krea2-trained**. Never suggest FLUX/SDXL LoRAs. + +## Live context + +A JSON block named "Live SwarmUI context" is attached. Treat it as ground truth: + +- Use only LoRAs listed in `available_loras` (by exact `name`). +- Prefer listed `trigger_phrase` / `triggers` — **never invent** trigger words. +- When enabling a LoRA, include its triggers in `prompt` if missing. +- Respect current width/height/steps/cfg unless the user asks to change them or the pack is `fix_params`. + +## Output contract (mandatory) + +1. Write a short helpful reply in the user's language (RU or EN). +2. Then emit **one** fenced JSON patch (and only fields you want to change): + +```json +{ + "prompt": "...", + "negative": null, + "loras": [{"name": "exact_name_from_list", "weight": 0.8, "triggers": ["..."]}], + "width": 1024, + "height": 1280, + "steps": 8, + "cfg": 1, + "notes": "one-line why" +} +``` + +Rules for the patch: + +- Omit keys you are not changing. +- `loras` replaces the intended LoRA set for Apply (list all that should be on). +- width/height between 128 and 4096; prefer multiples near 1024 for Turbo. +- Do not invent model or LoRA filenames. +- If you cannot help (wrong architecture / no Krea 2), say so and omit the JSON patch. diff --git a/Prompts/compose_scene.md b/Prompts/compose_scene.md new file mode 100644 index 0000000..3de266b --- /dev/null +++ b/Prompts/compose_scene.md @@ -0,0 +1,14 @@ +# Mode: compose_scene + +Goal: co-create a scene / moodboard direction for **Krea 2**. + +## Approach + +- Clarify subject, setting, time of day, camera distance, style. +- Propose one strong prompt (not five weak ones). +- Optionally suggest which available LoRAs fit — only from the live list, with triggers. +- Mention Prompt Images only if a reference would help, and warn that refs can overpower text. + +## Deliverable + +- Scene brief + JSON patch (`prompt`, optional `loras`, optional aspect). diff --git a/Prompts/critique_image.md b/Prompts/critique_image.md new file mode 100644 index 0000000..d1b5b0f --- /dev/null +++ b/Prompts/critique_image.md @@ -0,0 +1,16 @@ +# Mode: critique_image + +Goal: look at the attached image (vision) and improve the next generation for **Krea 2**. + +## How to critique + +- Describe what you see: subject, composition, lighting, defects (anatomy, blur, wrong style). +- Tie feedback to **actionable** prompt / LoRA / size changes. +- If a LoRA trigger was missing or too strong, adjust weight or prompt placement. +- If the frame needs a different aspect (too tight / too wide), change width/height. +- Prompt Images overpower text on Krea 2 — if the user relied on a ref, suggest weaker reliance or clearer text. + +## Deliverable + +- Short critique in the user's language. +- JSON patch with improved `prompt` and any `loras` / size tweaks. diff --git a/Prompts/fix_params.md b/Prompts/fix_params.md new file mode 100644 index 0000000..492340b --- /dev/null +++ b/Prompts/fix_params.md @@ -0,0 +1,16 @@ +# Mode: fix_params + +Goal: adjust **generation parameters** for Krea 2 Turbo (or Raw if context says so). + +## Guidelines + +- Turbo: prefer steps 4–12 (default 8), CFG ~1, sigma shift ~1.15. +- Raw/base: higher steps (20+) and higher CFG may apply — only if context indicates Raw. +- Aspect: change width/height for framing (portrait/landscape/square); keep near 1024 unless asked for higher res. +- Do not change the prompt unless needed to match the new framing. +- Keep LoRAs unless the user asks to drop them. + +## Deliverable + +- Explain the param change. +- JSON patch focusing on `width`, `height`, `steps`, `cfg` (and `prompt` only if necessary). diff --git a/Prompts/write_prompt.md b/Prompts/write_prompt.md new file mode 100644 index 0000000..5844280 --- /dev/null +++ b/Prompts/write_prompt.md @@ -0,0 +1,17 @@ +# Mode: write_prompt + +Goal: craft or improve a **Krea 2** prompt that will generate well on Turbo. + +## How to write the prompt + +- Natural language + concrete visual details (subject, lighting, lens/mood, composition). +- Put **LoRA trigger phrases** near the subject they affect; do not dump unrelated tags. +- Prefer clarity over keyword stuffing. Krea 2 understands sentences. +- If the user wants a style covered by an available LoRA, enable that LoRA and weave its triggers in. +- Keep Turbo defaults unless the user asks otherwise (steps 8, cfg 1). + +## Deliverable + +- Explain briefly what you changed. +- Emit a JSON patch with at least `prompt`, and `loras` when relevant. +- Include `width`/`height` only if aspect should change for the scene (e.g. portrait → taller). diff --git a/README.md b/README.md new file mode 100644 index 0000000..19411f6 --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# Swarm Assistent + +SwarmUI extension for **collaborative Krea 2** prompting via **Ollama**: chat + vision, LoRA/trigger awareness, and applyable prompt/size patches. + +## Layout + +- **Left:** current / selected image preview (attach for vision) +- **Right (wider):** chat +- **Top-right:** settings (Ollama URL, model, auto-attach) + +## Requirements + +- SwarmUI with a **Krea 2** checkpoint selected +- Ollama on `http://127.0.0.1:11434` (gpu-rent default when `LLM_RUNTIME=ollama`) +- A chat+vision-capable Ollama model recommended for critique mode + +## Install + +Clone into SwarmUI `src/Extensions/swarm-assistent` (or let **gpu-rent** seed it from `extensions.yaml` with `requires: ollama`): + +```yaml +swarmui: + - url: https://gitea.hsrv.site/mrleo1nid/swarm-assistent.git + ref: main + dir: swarm-assistent + requires: ollama +``` + +Restart / rebuild SwarmUI after clone. + +## Prompt packs + +| Pack | Role | +| --- | --- | +| `base_krea2` | Always injected: Krea 2 rules + JSON patch contract | +| `write_prompt` | Craft / improve prompts | +| `critique_image` | Vision critique → fixes | +| `compose_scene` | Scene / moodboard | +| `fix_params` | Width/height/steps/CFG | + +Live context (checkpoint, LoRAs + triggers, current params) is injected every request. + +## License + +MIT diff --git a/SwarmAssistentExtension.cs b/SwarmAssistentExtension.cs new file mode 100644 index 0000000..7d55436 --- /dev/null +++ b/SwarmAssistentExtension.cs @@ -0,0 +1,236 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using FreneticUtilities.FreneticExtensions; +using Newtonsoft.Json.Linq; +using SwarmUI.Accounts; +using SwarmUI.Core; +using SwarmUI.Utils; +using SwarmUI.WebAPI; + +namespace Mrleo1nid.SwarmAssistent; + +/// Krea 2 collaborative assistant: Ollama chat + vision + prompt/LoRA/params patches. +public class SwarmAssistentExtension : Extension +{ + public static PermInfo PermUse = Permissions.Register(new( + "swarm_assistent_use", + "[Swarm Assistent] Use", + "Allows using the Swarm Assistent chat (Ollama proxy).", + PermissionDefault.USER, + Permissions.GroupUser)); + + public static HttpClient HttpClient; + + public static readonly string[] PackNames = + [ + "base_krea2", + "write_prompt", + "critique_image", + "compose_scene", + "fix_params", + ]; + + public override void OnPreInit() + { + ScriptFiles.Add("Assets/assistent.js"); + StyleSheetFiles.Add("Assets/assistent.css"); + ExtensionAuthor = "mrleo1nid"; + Description = "Collaborative Krea 2 assistant via Ollama: chat, vision, prompts, LoRA triggers, size patches."; + License = "MIT"; + Version = "0.1.0"; + Tags = ["tabs", "ui", "llm", "ollama", "krea"]; + } + + public override void OnInit() + { + HttpClient ??= new HttpClient { Timeout = TimeSpan.FromMinutes(10) }; + API.RegisterAPICall(AssistentListModels, false, PermUse); + API.RegisterAPICall(AssistentGetPacks, false, PermUse); + API.RegisterAPICall(AssistentChat, true, PermUse); + Logs.Init("Swarm Assistent extension loaded (Ollama proxy + Krea 2 packs)"); + } + + static string Clip(string text, int max) + { + if (string.IsNullOrEmpty(text) || text.Length <= max) + { + return text ?? ""; + } + return text[..max] + "…"; + } + + public static string NormalizeBaseUrl(string raw) + { + string url = (raw ?? "").Trim(); + if (string.IsNullOrWhiteSpace(url)) + { + url = "http://127.0.0.1:11434"; + } + return url.TrimEnd('/'); + } + + public string ReadPackFile(string name) + { + string safe = name.Replace('\\', '/').AfterLast('/').Replace("..", ""); + if (!PackNames.Contains(safe)) + { + return null; + } + string path = Path.Combine(FilePath, "Prompts", $"{safe}.md"); + if (!File.Exists(path)) + { + return null; + } + return File.ReadAllText(path, Encoding.UTF8); + } + + public async Task AssistentListModels(Session session, string baseUrl) + { + string root = NormalizeBaseUrl(baseUrl); + try + { + using HttpResponseMessage resp = await HttpClient.GetAsync($"{root}/api/tags"); + string body = await resp.Content.ReadAsStringAsync(); + if (!resp.IsSuccessStatusCode) + { + return new JObject { ["error"] = $"Ollama /api/tags HTTP {(int)resp.StatusCode}: {Clip(body, 400)}" }; + } + JObject parsed = JObject.Parse(body); + JArray models = []; + foreach (JToken m in parsed["models"] as JArray ?? []) + { + models.Add(m["name"]?.ToString() ?? ""); + } + return new JObject { ["success"] = true, ["base_url"] = root, ["models"] = models }; + } + catch (Exception ex) + { + return new JObject { ["error"] = $"Ollama unreachable at {root}: {ex.Message}" }; + } + } + + public async Task AssistentGetPacks(Session session) + { + JObject packs = new(); + foreach (string name in PackNames) + { + string text = ReadPackFile(name); + if (text is not null) + { + packs[name] = text; + } + } + return new JObject { ["success"] = true, ["packs"] = packs, ["order"] = new JArray(PackNames) }; + } + + /// + /// Proxy to Ollama /api/chat (non-stream). + /// must include messages (JArray) and optional context_json. + /// + public async Task AssistentChat(Session session, string baseUrl, string model, string pack, bool includeBase, JObject raw) + { + string root = NormalizeBaseUrl(baseUrl ?? raw?["base_url"]?.ToString()); + string modelName = (model ?? raw?["model"]?.ToString() ?? "").Trim(); + if (string.IsNullOrWhiteSpace(modelName)) + { + return new JObject { ["error"] = "model is required" }; + } + JArray userMessages = raw?["messages"] as JArray; + if (userMessages is null || userMessages.Count == 0) + { + return new JObject { ["error"] = "messages required" }; + } + + List ollamaMessages = []; + StringBuilder system = new(); + if (includeBase) + { + string basePack = ReadPackFile("base_krea2"); + if (!string.IsNullOrWhiteSpace(basePack)) + { + system.AppendLine(basePack); + } + } + string packName = (pack ?? raw?["pack"]?.ToString() ?? "write_prompt").Trim(); + if (!string.IsNullOrWhiteSpace(packName) && packName != "base_krea2") + { + string situational = ReadPackFile(packName); + if (!string.IsNullOrWhiteSpace(situational)) + { + system.AppendLine(); + system.AppendLine($"## Active mode: {packName}"); + system.AppendLine(situational); + } + } + string contextJson = raw?["context_json"]?.ToString(); + if (!string.IsNullOrWhiteSpace(contextJson)) + { + system.AppendLine(); + system.AppendLine("## Live SwarmUI context (JSON — trust this over guesses)"); + system.AppendLine("```json"); + system.AppendLine(contextJson); + system.AppendLine("```"); + } + if (system.Length > 0) + { + ollamaMessages.Add(new JObject + { + ["role"] = "system", + ["content"] = system.ToString(), + }); + } + foreach (JToken msg in userMessages) + { + if (msg is not JObject mo) + { + continue; + } + JObject copy = new() + { + ["role"] = mo["role"]?.ToString() ?? "user", + ["content"] = mo["content"]?.ToString() ?? "", + }; + if (mo["images"] is JArray images && images.Count > 0) + { + copy["images"] = images; + } + ollamaMessages.Add(copy); + } + + JObject payload = new() + { + ["model"] = modelName, + ["stream"] = false, + ["messages"] = new JArray(ollamaMessages), + }; + try + { + using StringContent content = new(payload.ToString(Newtonsoft.Json.Formatting.None), Encoding.UTF8, "application/json"); + using HttpResponseMessage resp = await HttpClient.PostAsync($"{root}/api/chat", content); + string body = await resp.Content.ReadAsStringAsync(); + if (!resp.IsSuccessStatusCode) + { + return new JObject { ["error"] = $"Ollama /api/chat HTTP {(int)resp.StatusCode}: {Clip(body, 800)}" }; + } + JObject parsed = JObject.Parse(body); + string reply = parsed["message"]?["content"]?.ToString() ?? parsed["response"]?.ToString() ?? ""; + return new JObject + { + ["success"] = true, + ["reply"] = reply, + ["model"] = modelName, + ["pack"] = packName, + ["raw"] = parsed, + }; + } + catch (Exception ex) + { + return new JObject { ["error"] = $"Ollama chat failed: {ex.Message}" }; + } + } +} diff --git a/SwarmAssistentExtension.csproj b/SwarmAssistentExtension.csproj new file mode 100644 index 0000000..29e3dec --- /dev/null +++ b/SwarmAssistentExtension.csproj @@ -0,0 +1,6 @@ + + + SwarmAssistentExtension + + + diff --git a/Tabs/Text2Image/Assistent.html b/Tabs/Text2Image/Assistent.html new file mode 100644 index 0000000..ec2be17 --- /dev/null +++ b/Tabs/Text2Image/Assistent.html @@ -0,0 +1,48 @@ +
+ +
+ +
+
+
Assistent
+
+ + +
+
+ +
+
+ +
+ + + +
+
+
+
+