Ship Assistent 0.10.2: lean context packing with quality-first vision flags.

Cut always-on system tokens (slim core, inventory, identity shelves, hops) and fix has_vision_image vs images_in_request so look_at gates correctly; enrich rich LoRAs and expose system_chars in debug.

Co-authored-by: Cursor <cursoragent@cursor.com>

AssistentChatPipeline.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using Newtonsoft.Json.Linq;
 using SwarmUI.Accounts;
+using SwarmUI.Core;
+using SwarmUI.Text2Image;
 using SwarmUI.Utils;
 
 namespace Mrleo1nid.SwarmAssistent;
@@ -129,7 +132,7 @@ public partial class SwarmAssistentExtension
         return ollamaMessages;
     }
 
-    async Task<(string reply, JObject raw, JArray civitaiResults)> RunChatWithHops(
+    async Task<(string reply, JObject raw, JArray civitaiResults, int systemChars)> RunChatWithHops(
         Session session,
         string root,
         string modelName,
@@ -175,6 +178,15 @@ public partial class SwarmAssistentExtension
         string enrichedContext = InjectMemoryHits(contextJson, hits);
         enrichedContext = EnrichPersonaContext(enrichedContext, pid, packName);
         List<JObject> messages = BuildOllamaMessages(packName, includeBase, enrichedContext, userMessages, personaId: pid, skillIds: skills);
+        int systemChars = 0;
+        foreach (JObject m in messages)
+        {
+            if (string.Equals(m["role"]?.ToString(), "system", StringComparison.OrdinalIgnoreCase))
+            {
+                systemChars = m["content"]?.ToString()?.Length ?? 0;
+                break;
+            }
+        }
         JArray civitaiResults = [];
         string reply = "";
         JObject lastRaw = null;
@@ -210,10 +222,14 @@ public partial class SwarmAssistentExtension
             {
                 civitaiResults = civitaiHop;
             }
-            messages.Add(new JObject { ["role"] = "assistant", ["content"] = reply });
+            // Re-feed only the parsed patch JSON (not full prose) to save hop tokens.
+            string assistantContent = patch is not null
+                ? patch.ToString(Newtonsoft.Json.Formatting.None)
+                : reply;
+            messages.Add(new JObject { ["role"] = "assistant", ["content"] = assistantContent });
             messages.Add(new JObject { ["role"] = "user", ["content"] = follow });
         }
-        return (reply, lastRaw, civitaiResults);
+        return (reply, lastRaw, civitaiResults, systemChars);
     }
 
     static string BuildRetrieveQuery(JArray userMessages, string contextJson, string packName = null)
@@ -281,7 +297,7 @@ public partial class SwarmAssistentExtension
         JObject a = Config.LoadAssistant(pid) ?? new JObject();
         AssistentMemory.RetrieveOptions opt = new()
         {
-            TopK = a["memory_top_k"]?.Value<int?>() ?? 10,
+            TopK = a["memory_top_k"]?.Value<int?>() ?? 8,
             MinScore = a["memory_min_score"]?.Value<float?>() ?? 0.32f,
             ApplyQuotas = true,
         };
@@ -347,7 +363,7 @@ public partial class SwarmAssistentExtension
                 return (null, null);
             }
             string kind = patch["memory_kind"]?.ToString();
-            int topK = Config.LoadAssistant(pid)["memory_top_k"]?.Value<int?>() ?? 10;
+            int topK = Config.LoadAssistant(pid)["memory_top_k"]?.Value<int?>() ?? 8;
             JArray rows = await Memory.SearchAsync(root, q, kind, topK, embed, chain);
             return (
                 "memory_search results (JSON, hybrid FTS+vector). Omit memory_search unless you need a different query.\n```json\n"
@@ -368,6 +384,102 @@ public partial class SwarmAssistentExtension
                 + tags.ToString(Newtonsoft.Json.Formatting.None) + "\n```",
                 null);
         }
+        if (tool == "list_inventory")
+        {
+            string q = patch["inventory_query"]?.ToString()?.Trim() ?? "";
+            string sig = "inv:" + q.ToLowerInvariant();
+            if (!hopDone.Add(sig))
+            {
+                return (null, null);
+            }
+            int lim = Config.LoadAssistant(pid)["inventory_hop_limit"]?.Value<int?>() ?? 20;
+            JArray rows = SearchInventoryForHop(q, lim);
+            return (
+                "list_inventory results (rich LoRA/checkpoint rows). Use exact names + listed triggers; omit list_inventory unless you need a different query.\n```json\n"
+                + rows.ToString(Newtonsoft.Json.Formatting.None) + "\n```",
+                null);
+        }
+        if (tool == "skill_load")
+        {
+            List<string> ids = [];
+            if (patch["skills"] is JArray skArr)
+            {
+                foreach (JToken t in skArr)
+                {
+                    string sid = AssistentConfig.SafeId(t?.ToString());
+                    if (!string.IsNullOrWhiteSpace(sid))
+                    {
+                        ids.Add(sid);
+                    }
+                }
+            }
+            if (ids.Count == 0)
+            {
+                ids.Add("memory");
+            }
+            StringBuilder sb = new();
+            foreach (string sid in ids)
+            {
+                string sig = "skill:" + sid;
+                if (!hopDone.Add(sig))
+                {
+                    continue;
+                }
+                string text = Config.LoadSkillPrompt(pid, sid);
+                if (string.IsNullOrWhiteSpace(text))
+                {
+                    continue;
+                }
+                sb.AppendLine($"## Skill: {sid}");
+                sb.AppendLine(text);
+                sb.AppendLine();
+            }
+            if (sb.Length == 0)
+            {
+                return (null, null);
+            }
+            return (
+                "skill_load results. Follow these skill rules on the next reply; omit skill_load unless you need another skill.\n\n"
+                + sb.ToString().TrimEnd(),
+                null);
+        }
+        if (tool == "persona_read")
+        {
+            List<string> shelves = [];
+            if (patch["persona_shelves"] is JArray shArr)
+            {
+                foreach (JToken t in shArr)
+                {
+                    string name = t?.ToString()?.Trim();
+                    if (!string.IsNullOrWhiteSpace(name))
+                    {
+                        shelves.Add(name);
+                    }
+                }
+            }
+            else if (patch["persona_shelves"] is JObject shObj)
+            {
+                // Model sometimes echoes shelf objects; treat keys as names.
+                foreach (JProperty p in shObj.Properties())
+                {
+                    shelves.Add(p.Name);
+                }
+            }
+            string sig = "persona_read:" + string.Join(",", shelves);
+            if (!hopDone.Add(sig))
+            {
+                return (null, null);
+            }
+            string body = Config.RenderPersonaReadBlock(pid, shelves.Count > 0 ? shelves : null);
+            if (string.IsNullOrWhiteSpace(body))
+            {
+                return ("persona_read: no additional lore shelves for this persona.", null);
+            }
+            return (
+                "persona_read results (lore shelves). Use for roleplay/appearance/outfit detail; omit persona_read unless you need different shelves.\n\n"
+                + body,
+                null);
+        }
         if (tool == "civitai")
         {
             string query = ExtractSearchQuery(patch);
@@ -391,7 +503,48 @@ public partial class SwarmAssistentExtension
         return (null, null);
     }
 
-    static string InjectMemoryHits(string contextJson, JArray hits, JObject exact = null)
+    /// <summary>Substring filter over current LoRA/checkpoint inventory for list_inventory hop.</summary>
+    JArray SearchInventoryForHop(string query, int limit)
+    {
+        int lim = Math.Max(1, Math.Min(limit, 40));
+        string q = (query ?? "").Trim().ToLowerInvariant();
+        JArray outRows = [];
+        void AddFromHandler(string setName, string kind)
+        {
+            if (!Program.T2IModelSets.TryGetValue(setName, out T2IModelHandler handler))
+            {
+                return;
+            }
+            IEnumerable<T2IModel> models = handler.Models.Values
+                .OrderByDescending(LooksLikeKreaArch)
+                .ThenBy(m => m.Name);
+            foreach (T2IModel model in models)
+            {
+                if (outRows.Count >= lim)
+                {
+                    return;
+                }
+                string name = model.Name ?? "";
+                if (!string.IsNullOrEmpty(q))
+                {
+                    string blob = $"{name} {model.Metadata?.UsageHint} {model.Metadata?.Description}".ToLowerInvariant();
+                    if (!blob.Contains(q, StringComparison.Ordinal))
+                    {
+                        continue;
+                    }
+                }
+                outRows.Add(BuildInventoryModelEntry(model, kind));
+            }
+        }
+        AddFromHandler("LoRA", "lora");
+        if (outRows.Count < lim)
+        {
+            AddFromHandler("Stable-Diffusion", "checkpoint");
+        }
+        return outRows;
+    }
+
+    string InjectMemoryHits(string contextJson, JArray hits, JObject exact = null)
     {
         JObject ctx;
         try
@@ -402,7 +555,36 @@ public partial class SwarmAssistentExtension
         {
             ctx = new JObject { ["_raw_context"] = contextJson };
         }
-        ctx["memory_hits"] = hits ?? new JArray();
+
+        int hitChars = 240;
+        try
+        {
+            string pidHit = AssistentConfig.SafeId(ctx["persona"]?.ToString()) ?? Config?.DefaultPersonaId() ?? "neutral";
+            hitChars = Config?.LoadAssistant(pidHit)?["memory_hit_chars"]?.Value<int?>() ?? 240;
+        }
+        catch
+        {
+            hitChars = 240;
+        }
+        hitChars = Math.Max(80, Math.Min(hitChars, 800));
+
+        JArray clippedHits = [];
+        foreach (JToken t in hits ?? [])
+        {
+            if (t is not JObject ho)
+            {
+                continue;
+            }
+            JObject copy = (JObject)ho.DeepClone();
+            string text = copy["text"]?.ToString() ?? "";
+            if (text.Length > hitChars)
+            {
+                copy["text"] = text[..hitChars] + "…";
+                copy["truncated"] = true;
+            }
+            clippedHits.Add(copy);
+        }
+        ctx["memory_hits"] = clippedHits;
         ctx.Remove("taste_profile");
         try
         {
@@ -418,72 +600,209 @@ public partial class SwarmAssistentExtension
         }
         // Never re-inject full Exact into live context (already in system prompt).
         ctx.Remove("exact");
-        if (ctx["session_exact"] is null)
+        if (ctx["session_exact"] is JObject se && !se.Properties().Any())
         {
-            ctx["session_exact"] = new JObject();
-        }
-        // Slim inventory for LLM: keep enabled + current, drop full dump if present
-        if (ctx["available_loras"] is JArray allLoras && allLoras.Count > 24)
-        {
-            HashSet<string> keep = new(StringComparer.OrdinalIgnoreCase);
-            if (ctx["enabled_loras"] is JArray en)
-            {
-                foreach (JToken t in en)
-                {
-                    string n = t?["name"]?.ToString() ?? t?.ToString();
-                    if (!string.IsNullOrWhiteSpace(n))
-                    {
-                        keep.Add(n);
-                    }
-                }
-            }
-            foreach (JToken hit in hits ?? [])
-            {
-                if (string.Equals(hit?["kind"]?.ToString(), "lora", StringComparison.OrdinalIgnoreCase)
-                    || string.Equals(hit?["kind"]?.ToString(), "card", StringComparison.OrdinalIgnoreCase))
-                {
-                    string k = hit?["key"]?.ToString();
-                    if (!string.IsNullOrWhiteSpace(k))
-                    {
-                        keep.Add(k);
-                    }
-                }
-            }
-            JArray slim = [];
-            foreach (JToken t in allLoras)
-            {
-                string n = t?["name"]?.ToString();
-                if (!string.IsNullOrWhiteSpace(n) && (keep.Contains(n) || slim.Count < 12))
-                {
-                    if (keep.Contains(n) || t?["krea_likely"]?.Value<bool>() == true)
-                    {
-                        slim.Add(t);
-                    }
-                }
-            }
-            if (slim.Count == 0)
-            {
-                foreach (JToken t in allLoras.Take(12))
-                {
-                    slim.Add(t);
-                }
-            }
-            ctx["available_loras"] = slim;
-            ctx["available_loras_truncated"] = true;
-            ctx["available_loras_total"] = allLoras.Count;
+            ctx.Remove("session_exact");
         }
+
+        // Always normalize inventory: keep enabled + rich top-N; name-only for the rest.
+        SlimAvailableLorasInContext(ctx, hits);
+        DropNullOrEmpty(ctx);
         return ctx.ToString(Newtonsoft.Json.Formatting.None);
     }
 
-    static string MemoryWritePersona(JObject mo, string currentPersonaId)
+    void SlimAvailableLorasInContext(JObject ctx, JArray hits)
     {
-        string scope = (mo?["scope"]?.ToString() ?? "").Trim().ToLowerInvariant();
-        if (scope is "shared" or "common" or "global")
+        if (ctx["available_loras"] is not JArray allLoras || allLoras.Count == 0)
         {
-            return AssistentMemory.SharedPersona;
+            return;
+        }
+        int richCap = 12;
+        int namesCap = 24;
+        try
+        {
+            string pid = AssistentConfig.SafeId(ctx["persona"]?.ToString()) ?? Config?.DefaultPersonaId() ?? "neutral";
+            JObject asst = Config?.LoadAssistant(pid) ?? new JObject();
+            richCap = asst["inventory_prompt_rich"]?.Value<int?>() ?? 12;
+            namesCap = asst["inventory_prompt_names"]?.Value<int?>() ?? 24;
+        }
+        catch { /* defaults */ }
+        richCap = Math.Max(4, Math.Min(richCap, 40));
+        namesCap = Math.Max(richCap, Math.Min(namesCap, 80));
+
+        HashSet<string> keepRich = new(StringComparer.OrdinalIgnoreCase);
+        if (ctx["enabled_loras"] is JArray en)
+        {
+            foreach (JToken t in en)
+            {
+                string n = t?["name"]?.ToString() ?? t?.ToString();
+                if (!string.IsNullOrWhiteSpace(n))
+                {
+                    keepRich.Add(n);
+                }
+            }
+        }
+        if (ctx["selected_loras"] is JArray sel)
+        {
+            foreach (JToken t in sel)
+            {
+                string n = t?["name"]?.ToString() ?? t?.ToString();
+                if (!string.IsNullOrWhiteSpace(n))
+                {
+                    keepRich.Add(n);
+                }
+            }
+        }
+        foreach (JToken hit in hits ?? [])
+        {
+            if (string.Equals(hit?["kind"]?.ToString(), "lora", StringComparison.OrdinalIgnoreCase)
+                || string.Equals(hit?["kind"]?.ToString(), "card", StringComparison.OrdinalIgnoreCase))
+            {
+                string k = hit?["key"]?.ToString();
+                if (!string.IsNullOrWhiteSpace(k))
+                {
+                    keepRich.Add(k);
+                }
+            }
+        }
+
+        List<JToken> ordered = allLoras
+            .OrderByDescending(t => keepRich.Contains(t?["name"]?.ToString() ?? "") ? 1000 : 0)
+            .ThenByDescending(t => t?["krea_likely"]?.Value<bool>() == true ? 50 : 0)
+            .ThenBy(t => t?["name"]?.ToString() ?? "", StringComparer.OrdinalIgnoreCase)
+            .ToList();
+
+        JArray slim = [];
+        int richCount = 0;
+        foreach (JToken t in ordered)
+        {
+            if (slim.Count >= namesCap)
+            {
+                break;
+            }
+            string n = t?["name"]?.ToString();
+            if (string.IsNullOrWhiteSpace(n))
+            {
+                continue;
+            }
+            bool wantRich = keepRich.Contains(n) || (t?["krea_likely"]?.Value<bool>() == true && richCount < richCap);
+            if (wantRich)
+            {
+                JObject rich = EnrichLoraRowForPrompt(t as JObject ?? new JObject { ["name"] = n });
+                slim.Add(rich);
+                if (!keepRich.Contains(n))
+                {
+                    richCount++;
+                }
+            }
+            else
+            {
+                JObject nameOnly = new() { ["name"] = n };
+                if (t?["krea_likely"]?.Value<bool>() == true)
+                {
+                    nameOnly["krea_likely"] = true;
+                }
+                slim.Add(nameOnly);
+            }
+        }
+        ctx["available_loras"] = slim;
+        if (allLoras.Count > slim.Count)
+        {
+            ctx["available_loras_truncated"] = true;
+            // Prefer client total if already set (full disk inventory count).
+            if (ctx["available_loras_total"] is null)
+            {
+                ctx["available_loras_total"] = allLoras.Count;
+            }
+        }
+    }
+
+    /// <summary>Ensure rich LoRA rows have triggers/blurb from Swarm inventory when the client sent name-only.</summary>
+    JObject EnrichLoraRowForPrompt(JObject row)
+    {
+        if (row is null)
+        {
+            return new JObject();
+        }
+        JObject outRow = (JObject)row.DeepClone();
+        string name = outRow["name"]?.ToString();
+        bool needsTriggers = string.IsNullOrWhiteSpace(outRow["trigger_phrase"]?.ToString())
+            && (outRow["triggers"] is not JArray tr || tr.Count == 0);
+        bool needsBlurb = string.IsNullOrWhiteSpace(outRow["blurb"]?.ToString());
+        if (!needsTriggers && !needsBlurb)
+        {
+            return outRow;
+        }
+        JObject fromDisk = FindInventoryLoraByName(name);
+        if (fromDisk is null)
+        {
+            return outRow;
+        }
+        if (needsTriggers)
+        {
+            if (!string.IsNullOrWhiteSpace(fromDisk["trigger_phrase"]?.ToString()))
+            {
+                outRow["trigger_phrase"] = fromDisk["trigger_phrase"];
+            }
+            if (fromDisk["triggers"] is JArray ft && ft.Count > 0)
+            {
+                outRow["triggers"] = ft.DeepClone();
+            }
+        }
+        if (needsBlurb && !string.IsNullOrWhiteSpace(fromDisk["blurb"]?.ToString()))
+        {
+            outRow["blurb"] = fromDisk["blurb"];
+        }
+        if (outRow["default_weight"] is null && fromDisk["default_weight"] is not null)
+        {
+            outRow["default_weight"] = fromDisk["default_weight"];
+        }
+        if (fromDisk["krea_likely"]?.Value<bool>() == true)
+        {
+            outRow["krea_likely"] = true;
+        }
+        if (fromDisk["has_card"]?.Value<bool>() == true)
+        {
+            outRow["has_card"] = true;
+        }
+        return outRow;
+    }
+
+    JObject FindInventoryLoraByName(string name)
+    {
+        if (string.IsNullOrWhiteSpace(name) || !Program.T2IModelSets.TryGetValue("LoRA", out T2IModelHandler handler))
+        {
+            return null;
+        }
+        T2IModel model = handler.Models.Values.FirstOrDefault(m =>
+            string.Equals(m.Name, name, StringComparison.OrdinalIgnoreCase)
+            || string.Equals(Path.GetFileNameWithoutExtension(m.Name), Path.GetFileNameWithoutExtension(name), StringComparison.OrdinalIgnoreCase)
+            || (m.Name?.EndsWith("/" + name, StringComparison.OrdinalIgnoreCase) ?? false));
+        return model is null ? null : BuildInventoryModelEntry(model, "lora");
+    }
+
+    static void DropNullOrEmpty(JObject ctx)
+    {
+        List<string> remove = [];
+        foreach (JProperty p in ctx.Properties())
+        {
+            if (p.Value is null || p.Value.Type == JTokenType.Null)
+            {
+                remove.Add(p.Name);
+            }
+            else if (p.Value is JObject jo && !jo.Properties().Any())
+            {
+                remove.Add(p.Name);
+            }
+            else if (p.Value is JArray ja && ja.Count == 0 && p.Name is not "memory_hits")
+            {
+                remove.Add(p.Name);
+            }
+        }
+        foreach (string k in remove)
+        {
+            ctx.Remove(k);
         }
-        // Personal only — never let the model write into another personality's store.
-        return AssistentConfig.SafeId(currentPersonaId) ?? AssistentMemory.SharedPersona;
     }
 
     string EnrichPersonaContext(string contextJson, string personaId, string packName)
@@ -501,13 +820,23 @@ public partial class SwarmAssistentExtension
         ctx["persona_source"] = Config.PersonaSource(pid);
         JObject schema = Config.LoadControlsSchema(pid);
         JObject values = Config.LoadControlValues(pid);
-        if (schema.Properties().Any())
+        bool authorPack = string.Equals(packName, "author_persona", StringComparison.OrdinalIgnoreCase)
+            || string.Equals(packName, "persona", StringComparison.OrdinalIgnoreCase);
+        // Values only outside author pack (schema is fat). Author pack gets full schema.
+        if (values.Properties().Any())
         {
-            ctx["persona_controls"] = new JObject
+            if (authorPack && schema.Properties().Any())
             {
-                ["schema"] = schema,
-                ["values"] = values,
-            };
+                ctx["persona_controls"] = new JObject
+                {
+                    ["schema"] = schema,
+                    ["values"] = values,
+                };
+            }
+            else
+            {
+                ctx["persona_controls"] = new JObject { ["values"] = values };
+            }
         }
         JArray catalog = [];
         foreach (var p in Config.ListPersonaCatalog())
@@ -516,21 +845,34 @@ public partial class SwarmAssistentExtension
             {
                 ["id"] = p.id,
                 ["title"] = p.title,
-                ["source"] = p.source,
             });
         }
         ctx["personas"] = catalog;
-        if (string.Equals(packName, "author_persona", StringComparison.OrdinalIgnoreCase)
-            || string.Equals(packName, "persona", StringComparison.OrdinalIgnoreCase))
+        if (authorPack)
         {
             JObject shelves = Config.LoadIdentityParts(pid);
             shelves.Remove("extra");
             ctx["persona_shelves"] = shelves;
-            ctx["persona_controls_schema"] = schema;
+            if (schema.Properties().Any())
+            {
+                ctx["persona_controls_schema"] = schema;
+            }
         }
+        DropNullOrEmpty(ctx);
         return ctx.ToString(Newtonsoft.Json.Formatting.None);
     }
 
+    static string MemoryWritePersona(JObject mo, string currentPersonaId)
+    {
+        string scope = (mo?["scope"]?.ToString() ?? "").Trim().ToLowerInvariant();
+        if (scope is "shared" or "common" or "global")
+        {
+            return AssistentMemory.SharedPersona;
+        }
+        // Personal only — never let the model write into another personality's store.
+        return AssistentConfig.SafeId(currentPersonaId) ?? AssistentMemory.SharedPersona;
+    }
+
     /// <summary>Apply overlay persona clone/write from patch. Ignores persona_delete. Updates pid ref after switch.</summary>
     void ApplyPersonaActions(JObject patch, ref string personaId)
     {
@@ -559,7 +901,8 @@ public partial class SwarmAssistentExtension
         {
             wantClone = true;
         }
-        if (patch["persona_shelves"] is JObject)
+        // persona_shelves as object of content = write; as array of names = persona_read hop (ignore here).
+        if (patch["persona_shelves"] is JObject && !ActionsContain(patch, "persona_read"))
         {
             wantWrite = true;
         }