Improve Assistent UX: send from Generate, drag-drop, resize.

Add Send to Assistent media button, paste/drop vision image, pane splitter, and polish layout for Krea collaborative chat.

Co-authored-by: Cursor <cursoragent@cursor.com>

Assets/assistent.js

@@ -6,12 +6,16 @@
     const LS_MODEL = 'swarm_assistent_model';
     const LS_PACK = 'swarm_assistent_pack';
     const LS_AUTO_VISION = 'swarm_assistent_auto_vision';
+    const LS_PANE_WIDTH = 'swarm_assistent_pane_width';
+    const TAB_BUTTON_ID = 'maintab_assistent';
 
     const state = {
         history: [],
         packsLoaded: false,
         busy: false,
         lastImageDataUrl: null,
+        preferredModel: null,
+        dragDepth: 0,
     };
 
     function $(id) {
@@ -66,6 +70,93 @@
         el.dispatchEvent(new Event('change', { bubbles: true }));
     }
 
+    function openAssistentTab() {
+        const tab = document.getElementById(TAB_BUTTON_ID);
+        if (tab) {
+            tab.click();
+            return true;
+        }
+        // Fallback: try hash / pane
+        const pane = document.getElementById('assistent');
+        if (pane && typeof bootstrap !== 'undefined' && bootstrap.Tab) {
+            try {
+                bootstrap.Tab.getOrCreateInstance(tab || pane).show();
+            } catch (e) { /* ignore */ }
+        }
+        return !!tab;
+    }
+
+    function flashImagePane() {
+        const frame = $('sa_image_frame');
+        if (!frame) {
+            return;
+        }
+        frame.classList.remove('sa-flash');
+        // reflow
+        void frame.offsetWidth;
+        frame.classList.add('sa-flash');
+    }
+
+    function setImageFromSrc(src, { switchTab = false, note = null } = {}) {
+        if (!src) {
+            return false;
+        }
+        const cleaned = String(src).trim().split(/\s+/)[0];
+        if (!cleaned || cleaned.startsWith('#')) {
+            return false;
+        }
+        state.lastImageDataUrl = cleaned;
+        const img = $('sa_image_preview');
+        const empty = $('sa_image_empty');
+        const chip = $('sa_vision_chip');
+        const frame = $('sa_image_frame');
+        if (img) {
+            img.src = cleaned;
+            img.hidden = false;
+        }
+        if (empty) {
+            empty.hidden = true;
+        }
+        if (chip) {
+            chip.hidden = false;
+        }
+        if (frame) {
+            frame.classList.add('sa-has-image');
+        }
+        if (switchTab) {
+            openAssistentTab();
+        }
+        flashImagePane();
+        if (note) {
+            setStatus(note);
+        }
+        return true;
+    }
+
+    function clearVisionImage({ silent = false } = {}) {
+        state.lastImageDataUrl = null;
+        const img = $('sa_image_preview');
+        const empty = $('sa_image_empty');
+        const chip = $('sa_vision_chip');
+        const frame = $('sa_image_frame');
+        if (img) {
+            img.removeAttribute('src');
+            img.hidden = true;
+        }
+        if (empty) {
+            empty.hidden = false;
+        }
+        if (chip) {
+            chip.hidden = true;
+        }
+        if (frame) {
+            frame.classList.remove('sa-has-image');
+        }
+        if (!silent) {
+            setStatus('Vision image cleared');
+        }
+    }
+
     function collectLiveContext() {
         const ctx = {
             architecture_ok: isKreaSelected(),
@@ -110,17 +201,14 @@
                 if (!m) {
                     continue;
                 }
-                const arch = `${m.architecture || ''} ${m.class || ''} ${m.name || ''} ${m.title || ''}`;
-                const isLora = /lora/i.test(m.category || m.type || '') || (m.name && String(m.name).toLowerCase().includes('lora'));
                 const folder = `${m.folder || m.path || ''}`;
+                const isLora = /lora/i.test(m.category || m.type || '') || (m.name && String(m.name).toLowerCase().includes('lora'));
                 const inLoraFolder = /lora/i.test(folder);
                 if (!(isLora || inLoraFolder)) {
-                    // Still include if metadata says lora
                     if (!/lora/i.test(JSON.stringify(m).slice(0, 200))) {
                         continue;
                     }
                 }
-                // Prefer Krea-tagged or unknown; skip obvious FLUX/SDXL-only names when tagged
                 ctx.available_loras.push({
                     name: m.name || m.title,
                     title: m.title || m.name,
@@ -128,13 +216,11 @@
                     architecture: m.architecture || null,
                 });
             }
-            // Cap list size for context window
             if (ctx.available_loras.length > 80) {
                 ctx.available_loras = ctx.available_loras.slice(0, 80);
             }
         } catch (e) { /* ignore */ }
 
-        // Fallback: parse multi-select input_loras options as available names
         try {
             const sel = document.getElementById('input_loras');
             if (sel && sel.options && ctx.available_loras.length === 0) {
@@ -188,7 +274,6 @@
             if (patch.negative != null) {
                 setVal('input_negativeprompt', patch.negative);
             }
-            // Ensure triggers present
             if (Array.isArray(patch.loras)) {
                 for (const l of patch.loras) {
                     const triggers = l.triggers || (l.trigger_phrase ? [l.trigger_phrase] : []);
@@ -290,47 +375,85 @@
         box.scrollTop = box.scrollHeight;
     }
 
-    function refreshImagePreview() {
-        let src = null;
+    function findCurrentGenerateSrc() {
         try {
-            const cur = document.getElementById('current_image_img') || document.querySelector('#current_image img') || document.querySelector('.current-image img');
-            if (cur && cur.src) {
-                src = cur.src;
+            const cur = document.getElementById('current_image_img')
+                || document.querySelector('#current_image img')
+                || document.querySelector('.current-image img')
+                || document.querySelector('#current_image_batch img');
+            if (cur) {
+                return cur.dataset?.src || cur.src || null;
             }
         } catch (e) { /* ignore */ }
         try {
-            if (!src && typeof currentMetadataMap !== 'undefined' && currentMetadataMap && currentMetadataMap.image) {
-                src = currentMetadataMap.image;
+            if (typeof currentMetadataMap !== 'undefined' && currentMetadataMap && currentMetadataMap.image) {
+                return currentMetadataMap.image;
             }
         } catch (e) { /* ignore */ }
+        return null;
+    }
 
-        const img = $('sa_image_preview');
-        const empty = $('sa_image_empty');
+    function refreshImagePreview({ onlyIfEmpty = false } = {}) {
+        if (onlyIfEmpty && state.lastImageDataUrl) {
+            return;
+        }
+        const src = findCurrentGenerateSrc();
         if (src) {
-            state.lastImageDataUrl = src;
-            if (img) {
-                img.src = src;
-                img.hidden = false;
-            }
-            if (empty) {
-                empty.hidden = true;
-            }
-        } else {
-            state.lastImageDataUrl = null;
-            if (img) {
-                img.hidden = true;
-            }
-            if (empty) {
-                empty.hidden = false;
+            setImageFromSrc(src);
+        } else if (!state.lastImageDataUrl) {
+            clearVisionImage({ silent: true });
+        }
+    }
+
+    function fileToDataUrl(file) {
+        return new Promise((resolve, reject) => {
+            const reader = new FileReader();
+            reader.onload = () => resolve(String(reader.result || ''));
+            reader.onerror = reject;
+            reader.readAsDataURL(file);
+        });
+    }
+
+    async function acceptImageFile(file) {
+        if (!file || !String(file.type || '').startsWith('image/')) {
+            setStatus('Not an image file');
+            return false;
+        }
+        const dataUrl = await fileToDataUrl(file);
+        return setImageFromSrc(dataUrl, { note: `Loaded ${file.name || 'image'}` });
+    }
+
+    async function handleDropDataTransfer(dt) {
+        if (!dt) {
+            return false;
+        }
+        if (dt.files && dt.files.length) {
+            for (const file of dt.files) {
+                if (String(file.type || '').startsWith('image/')) {
+                    return acceptImageFile(file);
+                }
             }
         }
+        const uri = (dt.getData('text/uri-list') || dt.getData('text/plain') || '').trim();
+        if (uri) {
+            const first = uri.split('\n').map((l) => l.trim()).find((l) => l && !l.startsWith('#'));
+            if (first) {
+                return setImageFromSrc(first, { note: 'Image from drag' });
+            }
+        }
+        // HTML img drag
+        const html = dt.getData('text/html') || '';
+        const m = html.match(/src=["']([^"']+)["']/i);
+        if (m && m[1]) {
+            return setImageFromSrc(m[1], { note: 'Image from drag' });
+        }
+        return false;
     }
 
     async function imageToBase64ForOllama(src) {
         if (!src) {
             return null;
         }
-        // Already data URL
         if (src.startsWith('data:')) {
             const i = src.indexOf(',');
             return i >= 0 ? src.slice(i + 1) : null;
@@ -359,6 +482,7 @@
         const model = localStorage.getItem(LS_MODEL);
         const pack = localStorage.getItem(LS_PACK);
         const auto = localStorage.getItem(LS_AUTO_VISION);
+        const paneW = localStorage.getItem(LS_PANE_WIDTH);
         if (base && $('sa_base_url')) {
             $('sa_base_url').value = base;
         }
@@ -371,6 +495,9 @@
         if (model) {
             state.preferredModel = model;
         }
+        if (paneW) {
+            document.documentElement.style.setProperty('--sa-image-width', paneW);
+        }
     }
 
     function saveSettings() {
@@ -433,7 +560,9 @@
             return;
         }
 
-        refreshImagePreview();
+        if (!state.lastImageDataUrl) {
+            refreshImagePreview();
+        }
         const attach = ($('sa_attach_vision')?.checked || $('sa_auto_vision')?.checked) && state.lastImageDataUrl;
         let images = null;
         if (attach) {
@@ -444,19 +573,13 @@
             }
         }
 
-        const userMsg = { role: 'user', content: text };
-        if (images) {
-            userMsg.images = images;
-        }
         state.history.push({ role: 'user', content: text });
         appendMessage('user', text);
         $('sa_input').value = '';
 
         const context = collectLiveContext();
         context.has_vision_image = !!images;
-        // Strip huge fields from history replay — only send recent turns without images in history payload
         const messages = state.history.slice(-12).map((m) => ({ role: m.role, content: m.content }));
-        // Last user message may include images
         if (images && messages.length) {
             messages[messages.length - 1].images = images;
         }
@@ -493,6 +616,186 @@
         });
     }
 
+    function wireDropZone() {
+        const frame = $('sa_image_frame');
+        const overlay = $('sa_drop_overlay');
+        if (!frame) {
+            return;
+        }
+
+        const setDrag = (on) => {
+            frame.classList.toggle('sa-dragover', on);
+            if (overlay) {
+                overlay.hidden = !on;
+            }
+        };
+
+        ['dragenter', 'dragover'].forEach((ev) => {
+            frame.addEventListener(ev, (e) => {
+                e.preventDefault();
+                e.stopPropagation();
+                state.dragDepth += ev === 'dragenter' ? 1 : 0;
+                setDrag(true);
+                if (e.dataTransfer) {
+                    e.dataTransfer.dropEffect = 'copy';
+                }
+            });
+        });
+        frame.addEventListener('dragleave', (e) => {
+            e.preventDefault();
+            state.dragDepth = Math.max(0, state.dragDepth - 1);
+            if (state.dragDepth === 0) {
+                setDrag(false);
+            }
+        });
+        frame.addEventListener('drop', async (e) => {
+            e.preventDefault();
+            e.stopPropagation();
+            state.dragDepth = 0;
+            setDrag(false);
+            await handleDropDataTransfer(e.dataTransfer);
+        });
+
+        // Also accept drops on the whole layout (easier target)
+        const layout = $('sa_layout');
+        layout?.addEventListener('dragover', (e) => {
+            if (e.dataTransfer?.types?.includes('Files') || e.dataTransfer?.types?.includes('text/uri-list')) {
+                e.preventDefault();
+            }
+        });
+        layout?.addEventListener('drop', async (e) => {
+            if (!e.dataTransfer) {
+                return;
+            }
+            e.preventDefault();
+            await handleDropDataTransfer(e.dataTransfer);
+        });
+
+        frame.addEventListener('paste', async (e) => {
+            const items = e.clipboardData?.items;
+            if (!items) {
+                return;
+            }
+            for (const item of items) {
+                if (item.type.startsWith('image/')) {
+                    e.preventDefault();
+                    const file = item.getAsFile();
+                    if (file) {
+                        await acceptImageFile(file);
+                    }
+                    return;
+                }
+            }
+        });
+
+        // Global paste when Assistent tab is visible
+        document.addEventListener('paste', async (e) => {
+            const pane = document.getElementById('assistent');
+            if (!pane || !pane.classList.contains('active')) {
+                return;
+            }
+            if (e.target && (e.target.tagName === 'TEXTAREA' || e.target.tagName === 'INPUT')) {
+                // Still allow image paste over text fields when clipboard has image
+                const items = e.clipboardData?.items;
+                let hasImage = false;
+                if (items) {
+                    for (const item of items) {
+                        if (item.type.startsWith('image/')) {
+                            hasImage = true;
+                            break;
+                        }
+                    }
+                }
+                if (!hasImage) {
+                    return;
+                }
+            }
+            const items = e.clipboardData?.items;
+            if (!items) {
+                return;
+            }
+            for (const item of items) {
+                if (item.type.startsWith('image/')) {
+                    e.preventDefault();
+                    const file = item.getAsFile();
+                    if (file) {
+                        await acceptImageFile(file);
+                    }
+                    return;
+                }
+            }
+        });
+    }
+
+    function wireSplitter() {
+        const splitter = $('sa_splitter');
+        const layout = $('sa_layout');
+        const pane = $('sa_image_pane');
+        if (!splitter || !layout || !pane) {
+            return;
+        }
+        let dragging = false;
+        splitter.addEventListener('mousedown', (e) => {
+            e.preventDefault();
+            dragging = true;
+            splitter.classList.add('sa-dragging');
+            document.body.style.cursor = 'col-resize';
+            document.body.style.userSelect = 'none';
+        });
+        window.addEventListener('mousemove', (e) => {
+            if (!dragging) {
+                return;
+            }
+            const rect = layout.getBoundingClientRect();
+            const x = e.clientX - rect.left;
+            const pct = Math.min(45, Math.max(18, (x / rect.width) * 100));
+            const value = `${pct}%`;
+            document.documentElement.style.setProperty('--sa-image-width', value);
+            localStorage.setItem(LS_PANE_WIDTH, value);
+        });
+        window.addEventListener('mouseup', () => {
+            if (!dragging) {
+                return;
+            }
+            dragging = false;
+            splitter.classList.remove('sa-dragging');
+            document.body.style.cursor = '';
+            document.body.style.userSelect = '';
+        });
+    }
+
+    function registerSendButton() {
+        if (typeof registerMediaButton !== 'function') {
+            // SwarmUI core not ready yet — retry briefly
+            setTimeout(registerSendButton, 500);
+            return;
+        }
+        // Avoid duplicate registrations on hot-reload
+        if (window.__swarmAssistentMediaRegistered) {
+            return;
+        }
+        window.__swarmAssistentMediaRegistered = true;
+        registerMediaButton(
+            'Send to Assistent',
+            (src) => {
+                setImageFromSrc(src, {
+                    switchTab: true,
+                    note: 'Image sent to Assistent',
+                });
+                // Prefer critique pack when coming from an image
+                const pack = $('sa_pack');
+                if (pack && pack.value === 'write_prompt') {
+                    pack.value = 'critique_image';
+                    saveSettings();
+                }
+            },
+            'Open Assistent with this image (vision / critique / prompt help)',
+            ['image'],
+            true,
+            true,
+        );
+    }
+
     function wire() {
         if (!$('swarm_assistent_root')) {
             return;
@@ -501,6 +804,9 @@
         updateGate();
         refreshImagePreview();
         refreshModels();
+        wireDropZone();
+        wireSplitter();
+        registerSendButton();
 
         $('sa_btn_settings')?.addEventListener('click', () => {
             const s = $('sa_settings');
@@ -512,7 +818,15 @@
             saveSettings();
             refreshModels();
         });
-        $('sa_btn_refresh_image')?.addEventListener('click', refreshImagePreview);
+        $('sa_btn_use_current')?.addEventListener('click', () => {
+            const src = findCurrentGenerateSrc();
+            if (src) {
+                setImageFromSrc(src, { note: 'Using current Generate image' });
+            } else {
+                setStatus('No current Generate image');
+            }
+        });
+        $('sa_btn_clear_image')?.addEventListener('click', clearVisionImage);
         $('sa_btn_send')?.addEventListener('click', () => sendChat());
         $('sa_btn_clear')?.addEventListener('click', () => {
             state.history = [];
@@ -523,7 +837,7 @@
             setStatus('');
         });
         $('sa_input')?.addEventListener('keydown', (e) => {
-            if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) {
+            if (e.key === 'Enter' && !e.shiftKey && !e.altKey) {
                 e.preventDefault();
                 sendChat();
             }
@@ -533,9 +847,17 @@
         $('sa_pack')?.addEventListener('change', saveSettings);
         $('sa_auto_vision')?.addEventListener('change', saveSettings);
 
-        // Re-check Krea gate when user may swap models
         setInterval(updateGate, 2000);
-        setInterval(refreshImagePreview, 4000);
+        // Soft sync from Generate only when we have no pinned vision image
+        setInterval(() => refreshImagePreview({ onlyIfEmpty: true }), 5000);
+
+        // Expose for console / other extensions
+        window.swarmAssistent = {
+            setImageFromSrc,
+            clearVisionImage,
+            openAssistentTab,
+            sendToAssistent: (src) => setImageFromSrc(src, { switchTab: true, note: 'Image sent to Assistent' }),
+        };
     }
 
     if (document.readyState === 'loading') {