From 26e4ef76fb9771c1767e58f03ebf0335402722d3 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Sat, 22 Aug 2026 02:26:41 +0300 Subject: [PATCH] Ship Assistent 0.10.2: lean context packing with quality-first vision flags. Cut always-on system tokens (slim core, inventory, identity shelves, hops) and fix has_vision_image vs images_in_request so look_at gates correctly; enrich rich LoRAs and expose system_chars in debug. Co-authored-by: Cursor --- Assets/assistent.js | 250 +++++++++----- AssistentChatPipeline.cs | 493 +++++++++++++++++++++++---- AssistentConfig.cs | 2 +- AssistentOllama.cs | 6 +- AssistentPatch.cs | 3 +- AssistentPersonaApi.cs | 2 +- Config/_base/core/core.md | 2 +- Config/_base/packs/critique_image.md | 13 +- Config/_base/packs/describe_ref.md | 14 +- README.md | 2 +- SwarmAssistentExtension.cs | 4 +- Tabs/Text2Image/Assistent.html | 2 +- 12 files changed, 622 insertions(+), 171 deletions(-) diff --git a/Assets/assistent.js b/Assets/assistent.js index d96cccd..30731fb 100644 --- a/Assets/assistent.js +++ b/Assets/assistent.js @@ -27,6 +27,8 @@ let MAX_REF_SLOTS = 4; let CONTEXT_PROMPT_MAX = 2000; let HISTORY_KEEP_TURNS = 4; + let INVENTORY_PROMPT_RICH = 12; + let INVENTORY_PROMPT_NAMES = 24; let ASPECT_TABLE = { '1:1': [1024, 1024], @@ -136,6 +138,8 @@ streamMeta: null, critiqueHopUsed: false, visionHopUsed: false, + lastSystemChars: 0, + lastContextChars: 0, busyPhase: 'idle', busyStarted: 0, gotDelta: false, @@ -784,6 +788,12 @@ return state.slots.filter((s) => s.attach && s.src); } + /** Real board frames (not model previews). Used for has_vision_image even when JPEG is not sent. */ + function visionReadySlots() { + ensureBoard(); + return state.slots.filter((s) => s && s.src && !looksLikeModelPreview(s.src)); + } + function setSlotSrc(id, src, { select = true, attach = null, note = null, switchTab = false, allowPreview = false } = {}) { const slot = slotById(id); if (!slot) { @@ -2374,11 +2384,11 @@ batch: parseInt(val('input_images') || val('input_batchsize') || '0', 10) || null, prompt_image_count: countPromptImages(), selected_loras: [], - available_loras: slimInventoryLoras(inv.loras || [], 100), - available_checkpoints: slimInventoryCheckpoints(inv.checkpoints || [], 40), - wildcards: (inv.wildcards || []).map((w) => w.name || w).slice(0, 60), + available_loras: [], + available_checkpoints: slimInventoryCheckpoints(inv.checkpoints || [], 8), + wildcards: (inv.wildcards || []).map((w) => w.name || w).slice(0, 20), inventory_at: inv.inventory_at || null, - has_vision_image: attachableSlots().length > 0, + has_vision_image: visionReadySlots().length > 0, image_slots: slotCatalog(), attached_slot_ids: attachableSlots().map((s) => s.id), has_civitai_key: !!inv.has_civitai_key, @@ -2389,6 +2399,14 @@ user_prefs_count: 0, ...initCtx, }; + { + const slim = slimInventoryLoras(inv.loras || [], INVENTORY_PROMPT_NAMES); + ctx.available_loras = slim; + if ((inv.loras || []).length > slim.length) { + ctx.available_loras_truncated = true; + ctx.available_loras_total = (inv.loras || []).length; + } + } try { const model = resolveCurrentCheckpoint(); @@ -2403,14 +2421,35 @@ try { if (typeof loraHelper !== 'undefined' && loraHelper && Array.isArray(loraHelper.selected)) { - ctx.selected_loras = loraHelper.selected.map((l) => ({ - name: l.name || l, - weight: (loraHelper.loraWeightPref && loraHelper.loraWeightPref[l.name || l]) || 1, - })); + const byName = new Map(); + for (const l of inv.loras || []) { + if (l?.name) { + byName.set(String(l.name).toLowerCase(), l); + } + } + ctx.selected_loras = loraHelper.selected.map((l) => { + const name = l.name || l; + const invRow = byName.get(String(name).toLowerCase()) || {}; + const out = { + name, + weight: (loraHelper.loraWeightPref && loraHelper.loraWeightPref[name]) || invRow.default_weight || 1, + }; + if (invRow.trigger_phrase) { + out.trigger_phrase = invRow.trigger_phrase; + } + if (Array.isArray(invRow.triggers) && invRow.triggers.length) { + out.triggers = invRow.triggers.slice(0, 8); + } + if (invRow.blurb) { + out.blurb = invRow.blurb; + } + return out; + }); + ctx.enabled_loras = ctx.selected_loras; } } catch (e) { /* ignore */ } - // Recommendation cards: checkpoint + selected LoRAs + other has_card entries (capped). + // Recommendation cards: checkpoint + selected LoRAs only (no has_card sweep). const cardKeys = []; const seenCard = new Set(); const addKey = (kind, name) => { @@ -2432,22 +2471,6 @@ addKey('lora', l.name); } } - for (const l of inv.loras || []) { - if (l?.has_card && l?.name) { - addKey('lora', l.name); - } - if (cardKeys.length >= 14) { - break; - } - } - for (const c of inv.checkpoints || []) { - if (c?.has_card && c?.name) { - addKey('checkpoint', c.name); - } - if (cardKeys.length >= 16) { - break; - } - } for (const k of cardKeys) { const cached = state.modelCards[`${k.kind}:${k.name}`]; if (cached) { @@ -2477,8 +2500,8 @@ architecture: m.architecture || null, }); } - if (ctx.available_loras.length > 80) { - ctx.available_loras = ctx.available_loras.slice(0, 80); + if (ctx.available_loras.length > INVENTORY_PROMPT_NAMES) { + ctx.available_loras = ctx.available_loras.slice(0, INVENTORY_PROMPT_NAMES); } } catch (e) { /* ignore */ } } @@ -2491,28 +2514,33 @@ const profile = hasRaw && !hasTurbo ? 'raw' : 'turbo'; ctx.krea_profile = profile; const defaults = mergedGenerationDefaults(profile); - ctx.recommended_params = { + // Only send recommended_params when live UI differs from Exact-backed defaults + // (Exact itself is already in the system prompt). + const rec = { steps: defaults.steps ?? 8, cfg: defaults.cfg ?? 1, sigma_shift: defaults.sigma_shift ?? 1.15, }; if (defaults.aspect) { - ctx.recommended_params.aspect = defaults.aspect; + rec.aspect = defaults.aspect; + } + const liveDiffers = (ctx.steps != null && ctx.steps !== rec.steps) + || (ctx.cfg != null && ctx.cfg !== rec.cfg) + || (ctx.sigma_shift != null && ctx.sigma_shift !== rec.sigma_shift); + if (liveDiffers) { + ctx.recommended_params = rec; } } catch (e) { ctx.krea_profile = 'turbo'; - const defaults = mergedGenerationDefaults('turbo'); - ctx.recommended_params = { - steps: defaults.steps ?? 8, - cfg: defaults.cfg ?? 1, - sigma_shift: defaults.sigma_shift ?? 1.15, - }; } ctx.session_exact = state.sessionExact && Object.keys(state.sessionExact).length ? { ...state.sessionExact } - : {}; + : undefined; // Exact KV is already in the system prompt — do not duplicate the full blob into live context. + if (!ctx.session_exact) { + delete ctx.session_exact; + } return ctx; } @@ -2562,7 +2590,8 @@ } } } catch (e) { /* ignore */ } - const softCap = Math.min(limit || 100, 80); + const namesCap = Math.min(limit || INVENTORY_PROMPT_NAMES, INVENTORY_PROMPT_NAMES); + const richCap = Math.max(4, Math.min(INVENTORY_PROMPT_RICH, namesCap)); const rows = (list || []).map((l) => { const sel = selected.has(String(l.name || '').toLowerCase()); const hasCard = !!l.has_card; @@ -2584,58 +2613,72 @@ }; }); rows.sort((a, b) => b._score - a._score || String(a.name).localeCompare(String(b.name))); - // Full detail for top tier; name+trigger only for the rest within softCap. - const fullDetail = 36; - return rows.slice(0, softCap).map((row, idx) => { - const out = { name: row.name, title: row.title }; - if (row.trigger_phrase) { - out.trigger_phrase = row.trigger_phrase; + // Rich: enabled + top krea/card (triggers/blurb). Rest: name (+ krea_likely) only. + let richUsed = 0; + const out = []; + for (const row of rows) { + if (out.length >= namesCap) { + break; } - if (row.triggers) { - out.triggers = row.triggers; + const sel = selected.has(String(row.name || '').toLowerCase()); + let wantRich = sel; + if (!wantRich && richUsed < richCap && (row.krea_likely || row.has_card || row.blurb)) { + wantRich = true; } - if (row.krea_likely) { - out.krea_likely = true; - } - if (row.has_card) { - out.has_card = true; - } - const rich = idx < fullDetail || row._score >= 200; - if (rich) { - if (row.architecture) { - out.architecture = row.architecture; + if (wantRich) { + const rich = { name: row.name, title: row.title }; + if (row.trigger_phrase) { + rich.trigger_phrase = row.trigger_phrase; } - if (row.compat_class) { - out.compat_class = row.compat_class; + if (row.triggers) { + rich.triggers = row.triggers; + } + if (row.krea_likely) { + rich.krea_likely = true; + } + if (row.has_card) { + rich.has_card = true; } if (row.blurb) { - out.blurb = row.blurb; + rich.blurb = row.blurb; } if (row.default_weight) { - out.default_weight = row.default_weight; + rich.default_weight = row.default_weight; } - if (row.tags) { - out.tags = row.tags; + if (row.architecture) { + rich.architecture = row.architecture; } + out.push(rich); + if (!sel) { + richUsed++; + } + } else { + const nameOnly = { name: row.name }; + if (row.krea_likely) { + nameOnly.krea_likely = true; + } + out.push(nameOnly); } - return out; - }); + } + return out; } function slimInventoryCheckpoints(list, limit) { const rows = (list || []).slice(); rows.sort((a, b) => ((b.krea_likely ? 1 : 0) - (a.krea_likely ? 1 : 0)) || ((b.has_card ? 1 : 0) - (a.has_card ? 1 : 0)) || String(a.name).localeCompare(String(b.name))); - return rows.slice(0, limit || 40).map((c) => { + return rows.slice(0, limit || 8).map((c) => { const out = { name: c.name, title: c.title || c.name, - architecture: c.architecture || null, - compat_class: c.compat_class || null, - has_card: !!c.has_card, - krea_likely: !!c.krea_likely, }; - if (c.blurb) { - out.blurb = c.blurb; + if (c.architecture) { + out.architecture = c.architecture; + } + if (c.krea_likely) { + out.krea_likely = true; + } + if (c.has_card) { + out.has_card = true; } return out; }); @@ -3566,6 +3609,30 @@ await sendChat({ fromAutoCritique: true, forceSlotIds: [GEN_ID] }); } + /** After Generate: send look_at with JPEG when sa_auto_vision is on (skipped if auto-critique already attaches vision). */ + async function maybeAutoVisionLook(imageSrc) { + if (!wantsAutoVision() || $('sa_auto_critique')?.checked || state.visionHopUsed || state.busy) { + return; + } + const src = await resolveFinishedGenerateSrc(imageSrc); + if (!src) { + return; + } + const gen = generateSlot(); + if (gen) { + gen.src = src; + gen.attach = true; + renderBoard(); + } + state.visionHopUsed = true; + setPackValue('critique_image', { flash: true }); + if ($('sa_input')) { + $('sa_input').value = 'Look at the Generate result and briefly say what worked and what to fix next.'; + } + setStatus('Auto look_at…'); + await sendChat({ fromVisionHop: true, forceSlotIds: [GEN_ID], skipAutoPack: true }); + } + /** Board action: attach the finished Generate frame and ask for a verdict. */ async function askLookAtResult() { if (state.busy || state.generating) { @@ -4309,6 +4376,12 @@ if (asst.context_prompt_max != null) { CONTEXT_PROMPT_MAX = Math.max(200, Number(asst.context_prompt_max) || 2000); } + if (asst.inventory_prompt_rich != null) { + INVENTORY_PROMPT_RICH = Math.max(4, Number(asst.inventory_prompt_rich) || 12); + } + if (asst.inventory_prompt_names != null) { + INVENTORY_PROMPT_NAMES = Math.max(INVENTORY_PROMPT_RICH, Number(asst.inventory_prompt_names) || 24); + } fillKnobsFromConfig(data); if (applyDefaults || data.exact) { fillEmptyParamsFromExact(); @@ -6375,6 +6448,7 @@ ); if (src) { await maybeAutoCritique(src); + await maybeAutoVisionLook(src); } } else if (!state.generating) { stopBusyUi(wantsGen ? 'Применено' : ''); @@ -6494,9 +6568,12 @@ ` ckpt=${ctx.checkpoint?.name || '—'} · krea_profile=${ctx.krea_profile || profile}`, ` ${ctx.width || '?'}×${ctx.height || '?'} (${aspect}) · steps=${ctx.steps ?? '—'} · cfg=${ctx.cfg ?? '—'} · sigma=${ctx.sigma_shift ?? '—'} · seed=${ctx.seed ?? '—'} · batch=${ctx.batch ?? '—'}`, ` loras: ${formatDebugLoras(ctx.selected_loras || ctx.enabled_loras)}`, + ` available_loras=${(ctx.available_loras || []).length}${ctx.available_loras_truncated ? ` truncated/${ctx.available_loras_total || '?'}` : ''}`, ` prompt: ${clipDebug(ctx.prompt, 220)}`, ` negative: ${clipDebug(ctx.negative, 120)}`, ` init=${!!ctx.has_init_image} mask=${!!ctx.has_mask_image} prompt_images=${ctx.prompt_image_count || 0}`, + ` has_vision_image=${!!ctx.has_vision_image} · images_in_request=${!!ctx.images_in_request} · vision_ready=${visionReadySlots().length}`, + ` context_json_chars≈${JSON.stringify(ctx).length} · last_system_chars=${state.lastSystemChars || '—'} · last_context_chars=${state.lastContextChars || '—'}`, '', 'Exact defaults (merged):', ` generation=${JSON.stringify(exactGen)}`, @@ -6853,11 +6930,10 @@ } let wantedIds = (opts.forceSlotIds || []).map(normalizeSlotId).filter(Boolean); - if (!wantedIds.length) { - wantedIds = attachableSlots().map((s) => s.id); - } - if (wantsAutoVision() && generateSlot()?.src && !looksLikeModelPreview(generateSlot().src) && !wantedIds.includes(GEN_ID)) { - wantedIds.push(GEN_ID); + // JPEG only on look_at / vision hop — never auto-attach every board ref to ordinary chat. + const sendVision = !!(opts.fromVisionHop || (wantedIds.length && opts.forceSlotIds)); + if (!sendVision) { + wantedIds = []; } const visionSlots = wantedIds.map((id) => slotById(id)).filter((s) => s && s.src && !looksLikeModelPreview(s.src)); let images = null; @@ -6897,8 +6973,12 @@ persistHistory(); const context = collectLiveContext(); - context.has_vision_image = !!images; - context.attached_slot_ids = visionSlots.map((s) => s.id); + // has_vision_image = board has a real frame (even when JPEG is not in this request). + // images_in_request = JPEG bytes are attached to the last user message this turn. + context.has_vision_image = visionReadySlots().length > 0; + context.images_in_request = !!(images && images.length); + context.attached_slot_ids = attachableSlots().map((s) => s.id); + context.vision_slot_ids = visionSlots.map((s) => s.id); context.persona = persona; if (opts.cardTarget) { context.card_target = opts.cardTarget; @@ -6935,10 +7015,18 @@ embed_model: $('sa_embed_model')?.value || state.preferredEmbed || '', }; - const finishOk = async (reply, civitaiResults) => { + const finishOk = async (reply, civitaiResults, meta = {}) => { if (chatEpoch !== state.chatEpoch) { return; } + if (meta.system_chars != null) { + state.lastSystemChars = Number(meta.system_chars) || 0; + } + try { + state.lastContextChars = (context && JSON.stringify(context).length) || 0; + } catch (e) { + state.lastContextChars = 0; + } const prose = extractPatch(reply).prose || reply; state.history.push({ role: 'assistant', content: prose, persona, pack }); persistHistory(); @@ -7018,7 +7106,7 @@ const reply = data.reply || (state.streamEl?.querySelector('.sa-msg-body')?.textContent) || ''; const civitai = data.civitai_results || []; finalizeStreamMessage(reply, civitai); - finishOk(reply, civitai); + finishOk(reply, civitai, { system_chars: data.system_chars }); } }, 0, @@ -7046,7 +7134,7 @@ } const reply = data.reply || ''; appendMessage('assistant', reply, null, data.civitai_results || [], msgMeta); - finishOk(reply, data.civitai_results || []); + finishOk(reply, data.civitai_results || [], { system_chars: data.system_chars }); }, 0, (err2) => finishErr(String(err2 || err || 'Chat failed')), @@ -7069,7 +7157,7 @@ } const reply = data.reply || ''; appendMessage('assistant', reply, null, data.civitai_results || [], msgMeta); - finishOk(reply, data.civitai_results || []); + finishOk(reply, data.civitai_results || [], { system_chars: data.system_chars }); }, 0, (err) => finishErr(String(err || 'Chat failed')), diff --git a/AssistentChatPipeline.cs b/AssistentChatPipeline.cs index 456f49d..c510782 100644 --- a/AssistentChatPipeline.cs +++ b/AssistentChatPipeline.cs @@ -1,10 +1,13 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using Newtonsoft.Json.Linq; using SwarmUI.Accounts; +using SwarmUI.Core; +using SwarmUI.Text2Image; using SwarmUI.Utils; namespace Mrleo1nid.SwarmAssistent; @@ -129,7 +132,7 @@ public partial class SwarmAssistentExtension return ollamaMessages; } - async Task<(string reply, JObject raw, JArray civitaiResults)> RunChatWithHops( + async Task<(string reply, JObject raw, JArray civitaiResults, int systemChars)> RunChatWithHops( Session session, string root, string modelName, @@ -175,6 +178,15 @@ public partial class SwarmAssistentExtension string enrichedContext = InjectMemoryHits(contextJson, hits); enrichedContext = EnrichPersonaContext(enrichedContext, pid, packName); List messages = BuildOllamaMessages(packName, includeBase, enrichedContext, userMessages, personaId: pid, skillIds: skills); + int systemChars = 0; + foreach (JObject m in messages) + { + if (string.Equals(m["role"]?.ToString(), "system", StringComparison.OrdinalIgnoreCase)) + { + systemChars = m["content"]?.ToString()?.Length ?? 0; + break; + } + } JArray civitaiResults = []; string reply = ""; JObject lastRaw = null; @@ -210,10 +222,14 @@ public partial class SwarmAssistentExtension { civitaiResults = civitaiHop; } - messages.Add(new JObject { ["role"] = "assistant", ["content"] = reply }); + // Re-feed only the parsed patch JSON (not full prose) to save hop tokens. + string assistantContent = patch is not null + ? patch.ToString(Newtonsoft.Json.Formatting.None) + : reply; + messages.Add(new JObject { ["role"] = "assistant", ["content"] = assistantContent }); messages.Add(new JObject { ["role"] = "user", ["content"] = follow }); } - return (reply, lastRaw, civitaiResults); + return (reply, lastRaw, civitaiResults, systemChars); } static string BuildRetrieveQuery(JArray userMessages, string contextJson, string packName = null) @@ -281,7 +297,7 @@ public partial class SwarmAssistentExtension JObject a = Config.LoadAssistant(pid) ?? new JObject(); AssistentMemory.RetrieveOptions opt = new() { - TopK = a["memory_top_k"]?.Value() ?? 10, + TopK = a["memory_top_k"]?.Value() ?? 8, MinScore = a["memory_min_score"]?.Value() ?? 0.32f, ApplyQuotas = true, }; @@ -347,7 +363,7 @@ public partial class SwarmAssistentExtension return (null, null); } string kind = patch["memory_kind"]?.ToString(); - int topK = Config.LoadAssistant(pid)["memory_top_k"]?.Value() ?? 10; + int topK = Config.LoadAssistant(pid)["memory_top_k"]?.Value() ?? 8; JArray rows = await Memory.SearchAsync(root, q, kind, topK, embed, chain); return ( "memory_search results (JSON, hybrid FTS+vector). Omit memory_search unless you need a different query.\n```json\n" @@ -368,6 +384,102 @@ public partial class SwarmAssistentExtension + tags.ToString(Newtonsoft.Json.Formatting.None) + "\n```", null); } + if (tool == "list_inventory") + { + string q = patch["inventory_query"]?.ToString()?.Trim() ?? ""; + string sig = "inv:" + q.ToLowerInvariant(); + if (!hopDone.Add(sig)) + { + return (null, null); + } + int lim = Config.LoadAssistant(pid)["inventory_hop_limit"]?.Value() ?? 20; + JArray rows = SearchInventoryForHop(q, lim); + return ( + "list_inventory results (rich LoRA/checkpoint rows). Use exact names + listed triggers; omit list_inventory unless you need a different query.\n```json\n" + + rows.ToString(Newtonsoft.Json.Formatting.None) + "\n```", + null); + } + if (tool == "skill_load") + { + List ids = []; + if (patch["skills"] is JArray skArr) + { + foreach (JToken t in skArr) + { + string sid = AssistentConfig.SafeId(t?.ToString()); + if (!string.IsNullOrWhiteSpace(sid)) + { + ids.Add(sid); + } + } + } + if (ids.Count == 0) + { + ids.Add("memory"); + } + StringBuilder sb = new(); + foreach (string sid in ids) + { + string sig = "skill:" + sid; + if (!hopDone.Add(sig)) + { + continue; + } + string text = Config.LoadSkillPrompt(pid, sid); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + sb.AppendLine($"## Skill: {sid}"); + sb.AppendLine(text); + sb.AppendLine(); + } + if (sb.Length == 0) + { + return (null, null); + } + return ( + "skill_load results. Follow these skill rules on the next reply; omit skill_load unless you need another skill.\n\n" + + sb.ToString().TrimEnd(), + null); + } + if (tool == "persona_read") + { + List shelves = []; + if (patch["persona_shelves"] is JArray shArr) + { + foreach (JToken t in shArr) + { + string name = t?.ToString()?.Trim(); + if (!string.IsNullOrWhiteSpace(name)) + { + shelves.Add(name); + } + } + } + else if (patch["persona_shelves"] is JObject shObj) + { + // Model sometimes echoes shelf objects; treat keys as names. + foreach (JProperty p in shObj.Properties()) + { + shelves.Add(p.Name); + } + } + string sig = "persona_read:" + string.Join(",", shelves); + if (!hopDone.Add(sig)) + { + return (null, null); + } + string body = Config.RenderPersonaReadBlock(pid, shelves.Count > 0 ? shelves : null); + if (string.IsNullOrWhiteSpace(body)) + { + return ("persona_read: no additional lore shelves for this persona.", null); + } + return ( + "persona_read results (lore shelves). Use for roleplay/appearance/outfit detail; omit persona_read unless you need different shelves.\n\n" + + body, + null); + } if (tool == "civitai") { string query = ExtractSearchQuery(patch); @@ -391,7 +503,48 @@ public partial class SwarmAssistentExtension return (null, null); } - static string InjectMemoryHits(string contextJson, JArray hits, JObject exact = null) + /// Substring filter over current LoRA/checkpoint inventory for list_inventory hop. + JArray SearchInventoryForHop(string query, int limit) + { + int lim = Math.Max(1, Math.Min(limit, 40)); + string q = (query ?? "").Trim().ToLowerInvariant(); + JArray outRows = []; + void AddFromHandler(string setName, string kind) + { + if (!Program.T2IModelSets.TryGetValue(setName, out T2IModelHandler handler)) + { + return; + } + IEnumerable models = handler.Models.Values + .OrderByDescending(LooksLikeKreaArch) + .ThenBy(m => m.Name); + foreach (T2IModel model in models) + { + if (outRows.Count >= lim) + { + return; + } + string name = model.Name ?? ""; + if (!string.IsNullOrEmpty(q)) + { + string blob = $"{name} {model.Metadata?.UsageHint} {model.Metadata?.Description}".ToLowerInvariant(); + if (!blob.Contains(q, StringComparison.Ordinal)) + { + continue; + } + } + outRows.Add(BuildInventoryModelEntry(model, kind)); + } + } + AddFromHandler("LoRA", "lora"); + if (outRows.Count < lim) + { + AddFromHandler("Stable-Diffusion", "checkpoint"); + } + return outRows; + } + + string InjectMemoryHits(string contextJson, JArray hits, JObject exact = null) { JObject ctx; try @@ -402,7 +555,36 @@ public partial class SwarmAssistentExtension { ctx = new JObject { ["_raw_context"] = contextJson }; } - ctx["memory_hits"] = hits ?? new JArray(); + + int hitChars = 240; + try + { + string pidHit = AssistentConfig.SafeId(ctx["persona"]?.ToString()) ?? Config?.DefaultPersonaId() ?? "neutral"; + hitChars = Config?.LoadAssistant(pidHit)?["memory_hit_chars"]?.Value() ?? 240; + } + catch + { + hitChars = 240; + } + hitChars = Math.Max(80, Math.Min(hitChars, 800)); + + JArray clippedHits = []; + foreach (JToken t in hits ?? []) + { + if (t is not JObject ho) + { + continue; + } + JObject copy = (JObject)ho.DeepClone(); + string text = copy["text"]?.ToString() ?? ""; + if (text.Length > hitChars) + { + copy["text"] = text[..hitChars] + "…"; + copy["truncated"] = true; + } + clippedHits.Add(copy); + } + ctx["memory_hits"] = clippedHits; ctx.Remove("taste_profile"); try { @@ -418,72 +600,209 @@ public partial class SwarmAssistentExtension } // Never re-inject full Exact into live context (already in system prompt). ctx.Remove("exact"); - if (ctx["session_exact"] is null) + if (ctx["session_exact"] is JObject se && !se.Properties().Any()) { - ctx["session_exact"] = new JObject(); - } - // Slim inventory for LLM: keep enabled + current, drop full dump if present - if (ctx["available_loras"] is JArray allLoras && allLoras.Count > 24) - { - HashSet keep = new(StringComparer.OrdinalIgnoreCase); - if (ctx["enabled_loras"] is JArray en) - { - foreach (JToken t in en) - { - string n = t?["name"]?.ToString() ?? t?.ToString(); - if (!string.IsNullOrWhiteSpace(n)) - { - keep.Add(n); - } - } - } - foreach (JToken hit in hits ?? []) - { - if (string.Equals(hit?["kind"]?.ToString(), "lora", StringComparison.OrdinalIgnoreCase) - || string.Equals(hit?["kind"]?.ToString(), "card", StringComparison.OrdinalIgnoreCase)) - { - string k = hit?["key"]?.ToString(); - if (!string.IsNullOrWhiteSpace(k)) - { - keep.Add(k); - } - } - } - JArray slim = []; - foreach (JToken t in allLoras) - { - string n = t?["name"]?.ToString(); - if (!string.IsNullOrWhiteSpace(n) && (keep.Contains(n) || slim.Count < 12)) - { - if (keep.Contains(n) || t?["krea_likely"]?.Value() == true) - { - slim.Add(t); - } - } - } - if (slim.Count == 0) - { - foreach (JToken t in allLoras.Take(12)) - { - slim.Add(t); - } - } - ctx["available_loras"] = slim; - ctx["available_loras_truncated"] = true; - ctx["available_loras_total"] = allLoras.Count; + ctx.Remove("session_exact"); } + + // Always normalize inventory: keep enabled + rich top-N; name-only for the rest. + SlimAvailableLorasInContext(ctx, hits); + DropNullOrEmpty(ctx); return ctx.ToString(Newtonsoft.Json.Formatting.None); } - static string MemoryWritePersona(JObject mo, string currentPersonaId) + void SlimAvailableLorasInContext(JObject ctx, JArray hits) { - string scope = (mo?["scope"]?.ToString() ?? "").Trim().ToLowerInvariant(); - if (scope is "shared" or "common" or "global") + if (ctx["available_loras"] is not JArray allLoras || allLoras.Count == 0) { - return AssistentMemory.SharedPersona; + return; + } + int richCap = 12; + int namesCap = 24; + try + { + string pid = AssistentConfig.SafeId(ctx["persona"]?.ToString()) ?? Config?.DefaultPersonaId() ?? "neutral"; + JObject asst = Config?.LoadAssistant(pid) ?? new JObject(); + richCap = asst["inventory_prompt_rich"]?.Value() ?? 12; + namesCap = asst["inventory_prompt_names"]?.Value() ?? 24; + } + catch { /* defaults */ } + richCap = Math.Max(4, Math.Min(richCap, 40)); + namesCap = Math.Max(richCap, Math.Min(namesCap, 80)); + + HashSet keepRich = new(StringComparer.OrdinalIgnoreCase); + if (ctx["enabled_loras"] is JArray en) + { + foreach (JToken t in en) + { + string n = t?["name"]?.ToString() ?? t?.ToString(); + if (!string.IsNullOrWhiteSpace(n)) + { + keepRich.Add(n); + } + } + } + if (ctx["selected_loras"] is JArray sel) + { + foreach (JToken t in sel) + { + string n = t?["name"]?.ToString() ?? t?.ToString(); + if (!string.IsNullOrWhiteSpace(n)) + { + keepRich.Add(n); + } + } + } + foreach (JToken hit in hits ?? []) + { + if (string.Equals(hit?["kind"]?.ToString(), "lora", StringComparison.OrdinalIgnoreCase) + || string.Equals(hit?["kind"]?.ToString(), "card", StringComparison.OrdinalIgnoreCase)) + { + string k = hit?["key"]?.ToString(); + if (!string.IsNullOrWhiteSpace(k)) + { + keepRich.Add(k); + } + } + } + + List ordered = allLoras + .OrderByDescending(t => keepRich.Contains(t?["name"]?.ToString() ?? "") ? 1000 : 0) + .ThenByDescending(t => t?["krea_likely"]?.Value() == true ? 50 : 0) + .ThenBy(t => t?["name"]?.ToString() ?? "", StringComparer.OrdinalIgnoreCase) + .ToList(); + + JArray slim = []; + int richCount = 0; + foreach (JToken t in ordered) + { + if (slim.Count >= namesCap) + { + break; + } + string n = t?["name"]?.ToString(); + if (string.IsNullOrWhiteSpace(n)) + { + continue; + } + bool wantRich = keepRich.Contains(n) || (t?["krea_likely"]?.Value() == true && richCount < richCap); + if (wantRich) + { + JObject rich = EnrichLoraRowForPrompt(t as JObject ?? new JObject { ["name"] = n }); + slim.Add(rich); + if (!keepRich.Contains(n)) + { + richCount++; + } + } + else + { + JObject nameOnly = new() { ["name"] = n }; + if (t?["krea_likely"]?.Value() == true) + { + nameOnly["krea_likely"] = true; + } + slim.Add(nameOnly); + } + } + ctx["available_loras"] = slim; + if (allLoras.Count > slim.Count) + { + ctx["available_loras_truncated"] = true; + // Prefer client total if already set (full disk inventory count). + if (ctx["available_loras_total"] is null) + { + ctx["available_loras_total"] = allLoras.Count; + } + } + } + + /// Ensure rich LoRA rows have triggers/blurb from Swarm inventory when the client sent name-only. + JObject EnrichLoraRowForPrompt(JObject row) + { + if (row is null) + { + return new JObject(); + } + JObject outRow = (JObject)row.DeepClone(); + string name = outRow["name"]?.ToString(); + bool needsTriggers = string.IsNullOrWhiteSpace(outRow["trigger_phrase"]?.ToString()) + && (outRow["triggers"] is not JArray tr || tr.Count == 0); + bool needsBlurb = string.IsNullOrWhiteSpace(outRow["blurb"]?.ToString()); + if (!needsTriggers && !needsBlurb) + { + return outRow; + } + JObject fromDisk = FindInventoryLoraByName(name); + if (fromDisk is null) + { + return outRow; + } + if (needsTriggers) + { + if (!string.IsNullOrWhiteSpace(fromDisk["trigger_phrase"]?.ToString())) + { + outRow["trigger_phrase"] = fromDisk["trigger_phrase"]; + } + if (fromDisk["triggers"] is JArray ft && ft.Count > 0) + { + outRow["triggers"] = ft.DeepClone(); + } + } + if (needsBlurb && !string.IsNullOrWhiteSpace(fromDisk["blurb"]?.ToString())) + { + outRow["blurb"] = fromDisk["blurb"]; + } + if (outRow["default_weight"] is null && fromDisk["default_weight"] is not null) + { + outRow["default_weight"] = fromDisk["default_weight"]; + } + if (fromDisk["krea_likely"]?.Value() == true) + { + outRow["krea_likely"] = true; + } + if (fromDisk["has_card"]?.Value() == true) + { + outRow["has_card"] = true; + } + return outRow; + } + + JObject FindInventoryLoraByName(string name) + { + if (string.IsNullOrWhiteSpace(name) || !Program.T2IModelSets.TryGetValue("LoRA", out T2IModelHandler handler)) + { + return null; + } + T2IModel model = handler.Models.Values.FirstOrDefault(m => + string.Equals(m.Name, name, StringComparison.OrdinalIgnoreCase) + || string.Equals(Path.GetFileNameWithoutExtension(m.Name), Path.GetFileNameWithoutExtension(name), StringComparison.OrdinalIgnoreCase) + || (m.Name?.EndsWith("/" + name, StringComparison.OrdinalIgnoreCase) ?? false)); + return model is null ? null : BuildInventoryModelEntry(model, "lora"); + } + + static void DropNullOrEmpty(JObject ctx) + { + List remove = []; + foreach (JProperty p in ctx.Properties()) + { + if (p.Value is null || p.Value.Type == JTokenType.Null) + { + remove.Add(p.Name); + } + else if (p.Value is JObject jo && !jo.Properties().Any()) + { + remove.Add(p.Name); + } + else if (p.Value is JArray ja && ja.Count == 0 && p.Name is not "memory_hits") + { + remove.Add(p.Name); + } + } + foreach (string k in remove) + { + ctx.Remove(k); } - // Personal only — never let the model write into another personality's store. - return AssistentConfig.SafeId(currentPersonaId) ?? AssistentMemory.SharedPersona; } string EnrichPersonaContext(string contextJson, string personaId, string packName) @@ -501,13 +820,23 @@ public partial class SwarmAssistentExtension ctx["persona_source"] = Config.PersonaSource(pid); JObject schema = Config.LoadControlsSchema(pid); JObject values = Config.LoadControlValues(pid); - if (schema.Properties().Any()) + bool authorPack = string.Equals(packName, "author_persona", StringComparison.OrdinalIgnoreCase) + || string.Equals(packName, "persona", StringComparison.OrdinalIgnoreCase); + // Values only outside author pack (schema is fat). Author pack gets full schema. + if (values.Properties().Any()) { - ctx["persona_controls"] = new JObject + if (authorPack && schema.Properties().Any()) { - ["schema"] = schema, - ["values"] = values, - }; + ctx["persona_controls"] = new JObject + { + ["schema"] = schema, + ["values"] = values, + }; + } + else + { + ctx["persona_controls"] = new JObject { ["values"] = values }; + } } JArray catalog = []; foreach (var p in Config.ListPersonaCatalog()) @@ -516,21 +845,34 @@ public partial class SwarmAssistentExtension { ["id"] = p.id, ["title"] = p.title, - ["source"] = p.source, }); } ctx["personas"] = catalog; - if (string.Equals(packName, "author_persona", StringComparison.OrdinalIgnoreCase) - || string.Equals(packName, "persona", StringComparison.OrdinalIgnoreCase)) + if (authorPack) { JObject shelves = Config.LoadIdentityParts(pid); shelves.Remove("extra"); ctx["persona_shelves"] = shelves; - ctx["persona_controls_schema"] = schema; + if (schema.Properties().Any()) + { + ctx["persona_controls_schema"] = schema; + } } + DropNullOrEmpty(ctx); return ctx.ToString(Newtonsoft.Json.Formatting.None); } + static string MemoryWritePersona(JObject mo, string currentPersonaId) + { + string scope = (mo?["scope"]?.ToString() ?? "").Trim().ToLowerInvariant(); + if (scope is "shared" or "common" or "global") + { + return AssistentMemory.SharedPersona; + } + // Personal only — never let the model write into another personality's store. + return AssistentConfig.SafeId(currentPersonaId) ?? AssistentMemory.SharedPersona; + } + /// Apply overlay persona clone/write from patch. Ignores persona_delete. Updates pid ref after switch. void ApplyPersonaActions(JObject patch, ref string personaId) { @@ -559,7 +901,8 @@ public partial class SwarmAssistentExtension { wantClone = true; } - if (patch["persona_shelves"] is JObject) + // persona_shelves as object of content = write; as array of names = persona_read hop (ignore here). + if (patch["persona_shelves"] is JObject && !ActionsContain(patch, "persona_read")) { wantWrite = true; } diff --git a/AssistentConfig.cs b/AssistentConfig.cs index 0cb7a17..2913b87 100644 --- a/AssistentConfig.cs +++ b/AssistentConfig.cs @@ -1431,7 +1431,7 @@ public sealed class AssistentConfig ["source"] = p.source, })), ["identity"] = identity, - ["identity_summary"] = RenderIdentityBlock(id), + ["identity_summary"] = RenderIdentityBlock(id, includeAllShelves: true), ["enabled_skills"] = new JArray(ResolveEnabledSkills(id, null)), }; } diff --git a/AssistentOllama.cs b/AssistentOllama.cs index 65e269e..22ffbe0 100644 --- a/AssistentOllama.cs +++ b/AssistentOllama.cs @@ -243,7 +243,7 @@ public partial class SwarmAssistentExtension string embedModel = raw?["embed_model"]?.ToString() ?? Config.LoadSettings()["embed_model"]?.ToString(); try { - (string reply, JObject parsed, JArray civitai) = await RunChatWithHops( + (string reply, JObject parsed, JArray civitai, int systemChars) = await RunChatWithHops( session, root, modelName, packName, includeBase, contextJson, userMessages, personaId: persona, skillIds: skills, embedModel: embedModel); return new JObject { @@ -254,6 +254,7 @@ public partial class SwarmAssistentExtension ["persona"] = persona, ["raw"] = parsed, ["civitai_results"] = civitai, + ["system_chars"] = systemChars, }; } catch (Exception ex) @@ -305,7 +306,7 @@ public partial class SwarmAssistentExtension }, API.WebsocketTimeout); } } - (string reply, JObject parsed, JArray civitai) = await RunChatWithHops( + (string reply, JObject parsed, JArray civitai, int systemChars) = await RunChatWithHops( session, root, modelName, packName, includeBase, contextJson, userMessages, OnDelta, OnHopStart, persona, skills, embedModel); await ws.SendJson(new JObject { @@ -317,6 +318,7 @@ public partial class SwarmAssistentExtension ["persona"] = persona, ["raw"] = parsed, ["civitai_results"] = civitai, + ["system_chars"] = systemChars, }, API.WebsocketTimeout); } catch (Exception ex) diff --git a/AssistentPatch.cs b/AssistentPatch.cs index 0d2ad47..dc6a961 100644 --- a/AssistentPatch.cs +++ b/AssistentPatch.cs @@ -166,5 +166,6 @@ public partial class SwarmAssistentExtension return null; } - static bool WantsCivitaiSearch(JObject patch) => NextToolHop(patch) == "civitai"; + static bool WantsCivitaiSearch(JObject patch) + => ActionsContain(patch, "search_civitai") || !string.IsNullOrWhiteSpace(ExtractSearchQuery(patch)); } diff --git a/AssistentPersonaApi.cs b/AssistentPersonaApi.cs index 360323f..d199ce2 100644 --- a/AssistentPersonaApi.cs +++ b/AssistentPersonaApi.cs @@ -43,7 +43,7 @@ public partial class SwarmAssistentExtension ["shelves"] = Config.LoadIdentityParts(pid), ["controls"] = Config.LoadControlsSchema(pid), ["control_values"] = Config.LoadControlValues(pid), - ["identity_summary"] = Config.RenderIdentityBlock(pid), + ["identity_summary"] = Config.RenderIdentityBlock(pid, includeAllShelves: true), }; } diff --git a/Config/_base/core/core.md b/Config/_base/core/core.md index 6bc7d55..e7cb02d 100644 --- a/Config/_base/core/core.md +++ b/Config/_base/core/core.md @@ -26,7 +26,7 @@ Never write a “JSON Patch” section in prose without an actual fenced ```json - Use only LoRA/checkpoint **names** from `available_loras` / `enabled_loras` (or Civitai hop results). Prefer listed `triggers` / `trigger_phrase` / `blurb` — **never invent**. - Rich entries (blurbs/triggers) are enabled + top krea-likely. Name-only rows need `list_inventory` + `inventory_query` before you rely on them. - `memory_hits` may be truncated (`truncated: true`) — use `memory_get` for the full text. -- `has_vision_image` true means a real board frame exists, but **images are not in this request** until you emit `look_at`. Do not invent what the image looks like. +- `has_vision_image` true means a real board frame exists. `images_in_request` true means JPEG bytes are in **this** request. If you need to see a frame and `images_in_request` is false, emit `look_at` first — do not invent what the image looks like. - Prefer `krea_likely` / Krea architecture; ignore FLUX/SDXL. Respect current params unless asked or pack is `form_params`. - Init/inpaint flags and `image_slots` are in the JSON. Extra pack fields (aspect, inpaint, persona authoring) are documented in the active pack. diff --git a/Config/_base/packs/critique_image.md b/Config/_base/packs/critique_image.md index 48d9ccd..3fa8201 100644 --- a/Config/_base/packs/critique_image.md +++ b/Config/_base/packs/critique_image.md @@ -1,11 +1,18 @@ # Mode: critique_image -Goal: look at the attached board image(s) and improve the next **Krea 2** generation. +Goal: look at board image(s) and improve the next **Krea 2** generation. ## Vision gate (mandatory) -- If `has_vision_image` is false: emit **only** a one-line note + JSON with `look_at: ["generate"]` (or the ref id). Do **not** write a critique checklist. Do **not** invent defects. -- If vision is present: short critique, then a real fenced JSON patch. +- `has_vision_image` — a real frame exists on the board. +- `images_in_request` — JPEG bytes are attached **this turn**. + +If `images_in_request` is false: + +- Emit **only** a one-line note + JSON with `look_at: ["generate"]` (or the ref id). +- Do **not** write a critique checklist. Do **not** invent defects. + +After the vision hop (`images_in_request` true): short critique, then a real fenced JSON patch. ## Critique (keep short) diff --git a/Config/_base/packs/describe_ref.md b/Config/_base/packs/describe_ref.md index 74eb177..3d1d75b 100644 --- a/Config/_base/packs/describe_ref.md +++ b/Config/_base/packs/describe_ref.md @@ -1,8 +1,18 @@ # Mode: describe_ref -Goal: turn an attached / requested board image into a **Krea 2–ready** natural-language prompt (reverse prompt). +Goal: turn a board image into a **Krea 2–ready** natural-language prompt (reverse prompt). -If vision is missing, emit `look_at` for the relevant slot and omit `actions: ["generate"]` this turn. +## Vision gate (mandatory) + +- `has_vision_image` — a real frame exists on the board. +- `images_in_request` — JPEG bytes are attached **this turn**. + +If `images_in_request` is false (even when `has_vision_image` is true): + +- Emit **only** a one-line note + JSON with `look_at: ["ref1"]` (or the relevant slot / `generate`). +- Do **not** invent a description. Do **not** emit `actions: ["generate"]` this turn. + +After the vision hop (`images_in_request` true): write the reverse prompt. ## How to describe diff --git a/README.md b/README.md index a31fd2c..aae3506 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ SwarmUI extension for **collaborative Krea 2** prompting via **Ollama**: chat + board (Generate | Refs tabs), LoRA chips, **persona presets** (`Config/personas/`), **About the user** prefs + craft vector memory, model cards with Civitai fetch, img2img/inpaint, slash commands, auto Generate. -**Version 0.10.0** — Settings panel (Поведение / Модели / Личности / О пользователе / Крафт / Ещё), **UserPrefs** (global + per-persona) with prompt weight, persona export/import, craft memory clear APIs. Legacy `taste` migrates into UserPrefs once. +**Version 0.10.2** — Vision flag fix (`has_vision_image` vs `images_in_request`), enrich rich LoRAs from disk, post-Generate look_at option, `system_chars` in chat debug. Lean Ollama context (quality-first): slim core, always-on voice/likes + rich top LoRAs, hops for inventory/lore/`memory.md`, JPEG only on `look_at`. ## Layout diff --git a/SwarmAssistentExtension.cs b/SwarmAssistentExtension.cs index fc13a1b..3a01184 100644 --- a/SwarmAssistentExtension.cs +++ b/SwarmAssistentExtension.cs @@ -36,7 +36,7 @@ public partial class SwarmAssistentExtension : Extension ExtensionAuthor = "mrleo1nid"; Description = "Collaborative Krea 2 assistant: Ollama chat, persona presets, vector memory, model cards, Generate loop."; License = "MIT"; - Version = "0.10.0"; + Version = "0.10.2"; Tags = ["tabs", "ui", "llm", "ollama", "krea", "inpaint", "memory"]; } @@ -225,7 +225,7 @@ public partial class SwarmAssistentExtension : Extension ["id"] = p.id, ["title"] = p.title, ["accent"] = p.accent, - ["prompt"] = Config.RenderIdentityBlock(p.id), + ["prompt"] = Config.RenderIdentityBlock(p.id, includeAllShelves: true), ["source"] = p.source, }); } diff --git a/Tabs/Text2Image/Assistent.html b/Tabs/Text2Image/Assistent.html index b8f0906..6dd7f8b 100644 --- a/Tabs/Text2Image/Assistent.html +++ b/Tabs/Text2Image/Assistent.html @@ -88,7 +88,7 @@

Автодействия после ответа модели и скилы текущей личности.

- +