using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Text.RegularExpressions; using FreneticUtilities.FreneticExtensions; using Newtonsoft.Json.Linq; using SwarmUI.Utils; namespace Mrleo1nid.SwarmAssistent; /// Loads Config/_base + personas/<id> sparse presets with disk overlay merge. public sealed class AssistentConfig { readonly string _bundledRoot; readonly string _overlayRoot; readonly object _lock = new(); public AssistentConfig(string extensionFilePath, string dataRoot) { _bundledRoot = Path.Combine(extensionFilePath ?? "", "Config"); _overlayRoot = Path.Combine(dataRoot ?? "", "Assistent"); } public string BundledRoot => _bundledRoot; public string OverlayRoot => _overlayRoot; public static string SafeId(string id) { string s = (id ?? "").Replace('\\', '/').AfterLast('/').Replace("..", "").Trim(); if (string.IsNullOrWhiteSpace(s) || !Regex.IsMatch(s, @"^[A-Za-z0-9][A-Za-z0-9_\-]{0,63}$")) { return null; } return s; } static string SafeRel(string relative) { if (string.IsNullOrWhiteSpace(relative)) { return null; } string norm = relative.Replace('\\', '/').TrimStart('/'); if (norm.Contains("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)) { return null; } return norm.Replace('/', Path.DirectorySeparatorChar); } public string ResolveUnder(string root, string relative) { string rel = SafeRel(relative); if (rel is null || string.IsNullOrWhiteSpace(root)) { return null; } string full = Path.GetFullPath(Path.Combine(root, rel)); string rootFull = Path.GetFullPath(root); if (!full.StartsWith(rootFull.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) && !string.Equals(full, rootFull, StringComparison.OrdinalIgnoreCase)) { return null; } return full; } public JObject DeepMerge(JObject bottom, JObject top) { if (bottom is null) { return top is null ? new JObject() : (JObject)top.DeepClone(); } if (top is null) { return (JObject)bottom.DeepClone(); } JObject result = (JObject)bottom.DeepClone(); foreach (JProperty prop in top.Properties()) { if (prop.Value is JObject topObj && result[prop.Name] is JObject botObj) { result[prop.Name] = DeepMerge(botObj, topObj); } else if (prop.Value is JArray || prop.Value is null || prop.Value.Type == JTokenType.Null) { // Arrays replace entirely when the top file provides them. if (prop.Value is not null && prop.Value.Type != JTokenType.Null) { result[prop.Name] = prop.Value.DeepClone(); } } else if (prop.Value.Type == JTokenType.String && string.IsNullOrWhiteSpace(prop.Value.ToString())) { // Empty string does not clobber (personas.json empty prompt rule). continue; } else { result[prop.Name] = prop.Value.DeepClone(); } } return result; } public JObject TryReadJson(string path) { if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) { return null; } try { return JObject.Parse(File.ReadAllText(path, Encoding.UTF8)); } catch (Exception ex) { Logs.Debug($"AssistentConfig json {path}: {ex.Message}"); return null; } } public JArray TryReadJsonArray(string path) { if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) { return null; } try { return JArray.Parse(File.ReadAllText(path, Encoding.UTF8)); } catch (Exception ex) { Logs.Debug($"AssistentConfig json-array {path}: {ex.Message}"); return null; } } public string TryReadText(string path) { if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) { return null; } try { string text = File.ReadAllText(path, Encoding.UTF8); return string.IsNullOrWhiteSpace(text) ? null : text; } catch (Exception ex) { Logs.Debug($"AssistentConfig text {path}: {ex.Message}"); return null; } } /// Merge layered copies of the same relative path: bundled base → persona chain → disk base → disk persona. public JObject MergeJsonLayers(string relative, IEnumerable roots) { JObject acc = null; foreach (string root in roots) { string path = ResolveUnder(root, relative); JObject next = TryReadJson(path); if (next is null) { continue; } acc = DeepMerge(acc, next); } return acc ?? new JObject(); } public string MergeTextLayers(string relative, IEnumerable roots) { string last = null; foreach (string root in roots) { string path = ResolveUnder(root, relative); string text = TryReadText(path); if (text is not null) { last = text; } } return last; } /// Ancestor-first chain ending with the current persona (for overlay merge and vector retrieve). public List PersonaExtendsChain(string personaId) { List chain = []; HashSet seen = new(StringComparer.OrdinalIgnoreCase); string cur = SafeId(personaId) ?? "neutral"; for (int i = 0; i < 8 && !string.IsNullOrWhiteSpace(cur); i++) { if (!seen.Add(cur)) { break; } chain.Insert(0, cur); JObject meta = TryReadJson(ResolveUnder(Path.Combine(_bundledRoot, "personas", cur), "persona.json")) ?? TryReadJson(ResolveUnder(Path.Combine(_overlayRoot, "personas", cur), "persona.json")); string parent = SafeId(meta?["extends"]?.ToString()); if (string.IsNullOrWhiteSpace(parent) || string.Equals(parent, cur, StringComparison.OrdinalIgnoreCase)) { break; } cur = parent; } return chain; } public IEnumerable LayerRoots(string personaId) { yield return Path.Combine(_bundledRoot, "_base"); foreach (string id in PersonaExtendsChain(personaId)) { yield return Path.Combine(_bundledRoot, "personas", id); } yield return Path.Combine(_overlayRoot, "_base"); foreach (string id in PersonaExtendsChain(personaId)) { yield return Path.Combine(_overlayRoot, "personas", id); } } public List<(string id, string title, string accent, string source)> ListPersonaCatalog() { Dictionary byId = new(StringComparer.OrdinalIgnoreCase); void Scan(string root, string source) { string dir = Path.Combine(root, "personas"); if (!Directory.Exists(dir)) { return; } foreach (string folder in Directory.GetDirectories(dir)) { string id = SafeId(Path.GetFileName(folder)); if (id is null) { continue; } JObject meta = TryReadJson(Path.Combine(folder, "persona.json")); if (meta is null) { continue; } if (meta["enabled"]?.Value() == false) { byId.Remove(id); continue; } byId[id] = ( meta["title"]?.ToString() ?? id, meta["accent"]?.ToString() ?? "#8b949e", source ); } } Scan(_bundledRoot, "bundled"); Scan(_overlayRoot, "overlay"); // Legacy personas.json titles string overlayJson = Path.Combine(_overlayRoot, "personas.json"); JObject legacy = TryReadJson(overlayJson); if (legacy?["personas"] is JArray arr) { foreach (JToken t in arr) { if (t is not JObject po) { continue; } string id = SafeId(po["id"]?.ToString()); if (id is null) { continue; } if (byId.TryGetValue(id, out var cur)) { byId[id] = (po["title"]?.ToString() ?? cur.title, cur.accent, "overlay+bundled"); } else { byId[id] = (po["title"]?.ToString() ?? id, "#8b949e", "legacy"); } } } return byId.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase) .Select(kv => (kv.Key, kv.Value.title, kv.Value.accent, kv.Value.source)) .ToList(); } public string DefaultPersonaId() { JObject assistant = MergeJsonLayers("assistant.json", LayerRoots("neutral")); string def = SafeId(assistant["default_persona"]?.ToString()) ?? "neutral"; string overlayJson = Path.Combine(_overlayRoot, "personas.json"); JObject legacy = TryReadJson(overlayJson); string fromLegacy = SafeId(legacy?["default"]?.ToString()); if (fromLegacy is not null) { def = fromLegacy; } var catalog = ListPersonaCatalog(); if (catalog.All(p => !string.Equals(p.id, def, StringComparison.OrdinalIgnoreCase)) && catalog.Count > 0) { def = catalog[0].id; } return def; } /// Exact KV for the system prompt: params tables only (facts stay short / in RAG). public JObject LoadExactForPrompt(string personaId) { JObject full = LoadExact(personaId); if (full is null || full.Count == 0) { return full ?? new JObject(); } JObject slim = new(); foreach (string key in new[] { "generation", "profiles", "aspect_table" }) { if (full[key] is not null) { slim[key] = full[key].DeepClone(); } } if (full["facts"] is JObject facts) { // Keep short pointers only — long prose bloats 7B context and kills JSON discipline. JObject shortFacts = new(); foreach (JProperty prop in facts.Properties()) { string text = prop.Value?.ToString() ?? ""; if (text.Length <= 160) { shortFacts[prop.Name] = text; } else { shortFacts[prop.Name] = text.Substring(0, 157) + "…"; } } if (shortFacts.Count > 0) { slim["facts"] = shortFacts; } } return slim; } public JObject LoadAssistant(string personaId) => MergeJsonLayers("assistant.json", LayerRoots(personaId)); public JObject LoadUi(string personaId) => MergeJsonLayers("ui.json", LayerRoots(personaId)); /// Exact (KV) memory: bundled base → persona overlays → disk overlays. Persona keys overwrite base. public JObject LoadExact(string personaId) => MergeJsonLayers("exact.json", LayerRoots(personaId)); public JObject LoadModelProfile(string personaId) { JObject assistant = LoadAssistant(personaId); string arch = assistant["gate"]?["architecture"]?.ToString() ?? "krea2"; string safe = SafeId(arch) ?? "krea2"; return MergeJsonLayers(Path.Combine("models", $"{safe}.json"), LayerRoots(personaId)); } public string LoadCorePrompt(string personaId) { JObject meta = MergeJsonLayers(Path.Combine("core", "core.json"), LayerRoots(personaId)); string file = meta["prompt_file"]?.ToString() ?? "core.md"; return MergeTextLayers(Path.Combine("core", file), LayerRoots(personaId)) ?? ""; } public List<(string id, string title, int order, string[] aliases, bool enabled)> ListPacks(string personaId) { Dictionary byId = new(StringComparer.OrdinalIgnoreCase); foreach (string root in LayerRoots(personaId)) { string dir = Path.Combine(root, "packs"); if (!Directory.Exists(dir)) { continue; } foreach (string file in Directory.GetFiles(dir, "*.json")) { JObject meta = TryReadJson(file); string id = SafeId(meta?["id"]?.ToString() ?? Path.GetFileNameWithoutExtension(file)); if (id is null || meta is null) { continue; } bool enabled = meta["enabled"]?.Value() != false; string[] aliases = (meta["aliases"] as JArray)?.Select(t => t?.ToString()).Where(s => !string.IsNullOrWhiteSpace(s)).ToArray() ?? []; byId[id] = ( meta["title"]?.ToString() ?? id, meta["order"]?.Value() ?? 100, aliases, enabled ); } } return byId.Where(kv => kv.Value.enabled) .OrderBy(kv => kv.Value.order).ThenBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase) .Select(kv => (kv.Key, kv.Value.title, kv.Value.order, kv.Value.aliases, kv.Value.enabled)) .ToList(); } public string LoadPackPrompt(string personaId, string packId) { string id = SafeId(packId); if (id is null) { return null; } JObject meta = MergeJsonLayers(Path.Combine("packs", $"{id}.json"), LayerRoots(personaId)); if (meta["enabled"]?.Value() == false) { return null; } string file = meta["prompt_file"]?.ToString() ?? $"{id}.md"; return MergeTextLayers(Path.Combine("packs", file), LayerRoots(personaId)); } public List<(string id, string title, bool defaultOn, bool enabled)> ListSkills(string personaId) { Dictionary byId = new(StringComparer.OrdinalIgnoreCase); foreach (string root in LayerRoots(personaId)) { string dir = Path.Combine(root, "skills"); if (!Directory.Exists(dir)) { continue; } foreach (string file in Directory.GetFiles(dir, "*.json")) { JObject meta = TryReadJson(file); string id = SafeId(meta?["id"]?.ToString() ?? Path.GetFileNameWithoutExtension(file)); if (id is null || meta is null) { continue; } byId[id] = ( meta["title"]?.ToString() ?? id, meta["default"]?.Value() ?? false, meta["enabled"]?.Value() != false ); } } JObject skillsOverride = MergeJsonLayers("skills.json", LayerRoots(personaId)); foreach (JProperty prop in skillsOverride.Properties()) { string id = SafeId(prop.Name); if (id is null || !byId.ContainsKey(id)) { continue; } if (prop.Value.Type == JTokenType.Boolean) { var cur = byId[id]; byId[id] = (cur.title, prop.Value.Value(), cur.enabled); } } return byId.Where(kv => kv.Value.enabled) .OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase) .Select(kv => (kv.Key, kv.Value.title, kv.Value.defaultOn, kv.Value.enabled)) .ToList(); } public string LoadSkillPrompt(string personaId, string skillId) { string id = SafeId(skillId); if (id is null) { return null; } JObject meta = MergeJsonLayers(Path.Combine("skills", $"{id}.json"), LayerRoots(personaId)); if (meta["enabled"]?.Value() == false) { return null; } string file = meta["prompt_file"]?.ToString() ?? $"{id}.md"; return MergeTextLayers(Path.Combine("skills", file), LayerRoots(personaId)); } public JObject LoadIdentityParts(string personaId) { var roots = LayerRoots(personaId).ToList(); JObject persona = MergeJsonLayers("persona.json", roots); JObject voice = MergeJsonLayers("voice.json", roots); JObject likes = MergeJsonLayers("likes.json", roots); JObject dislikes = MergeJsonLayers("dislikes.json", roots); JObject rules = MergeJsonLayers("rules.json", roots); string extra = MergeTextLayers("extra.md", roots); // Legacy personas.json: only when no overlay persona folder exists for this id // (gpu-rent now seeds personas//extra.md instead of dumping personas.json). string id = SafeId(personaId) ?? "neutral"; string overlayPersonaDir = Path.Combine(_overlayRoot, "personas", id); bool hasOverlayFolder = Directory.Exists(overlayPersonaDir) && (File.Exists(Path.Combine(overlayPersonaDir, "persona.json")) || File.Exists(Path.Combine(overlayPersonaDir, "extra.md"))); if (!hasOverlayFolder) { string overlayJson = Path.Combine(_overlayRoot, "personas.json"); JObject legacy = TryReadJson(overlayJson); if (legacy?["personas"] is JArray arr) { foreach (JToken t in arr) { if (t is JObject po && string.Equals(SafeId(po["id"]?.ToString()), id, StringComparison.OrdinalIgnoreCase)) { string title = po["title"]?.ToString(); if (!string.IsNullOrWhiteSpace(title)) { persona["title"] = title; } string prompt = po["prompt"]?.ToString(); if (!string.IsNullOrWhiteSpace(prompt)) { extra = string.IsNullOrWhiteSpace(extra) ? prompt : extra + "\n\n" + prompt; } break; } } } } return new JObject { ["persona"] = persona, ["voice"] = voice, ["likes"] = likes, ["dislikes"] = dislikes, ["rules"] = rules, ["extra"] = extra ?? "", }; } static string FormatCategoryMap(JObject obj) { if (obj is null || !obj.Properties().Any()) { return ""; } List parts = []; foreach (JProperty prop in obj.Properties()) { if (prop.Value is JArray arr) { string joined = string.Join(", ", arr.Select(t => t?.ToString()).Where(s => !string.IsNullOrWhiteSpace(s))); if (!string.IsNullOrWhiteSpace(joined)) { parts.Add($"{prop.Name}: {joined}"); } } else if (prop.Value?.Type == JTokenType.String) { string s = prop.Value.ToString(); if (!string.IsNullOrWhiteSpace(s)) { parts.Add($"{prop.Name}: {s}"); } } } return string.Join("; ", parts); } public string RenderIdentityBlock(string personaId) { string id = SafeId(personaId) ?? "neutral"; JObject parts = LoadIdentityParts(id); JObject persona = parts["persona"] as JObject ?? new JObject(); JObject voice = parts["voice"] as JObject ?? new JObject(); JObject likes = parts["likes"] as JObject ?? new JObject(); JObject dislikes = parts["dislikes"] as JObject ?? new JObject(); JObject rules = parts["rules"] as JObject ?? new JObject(); string extra = parts["extra"]?.ToString() ?? ""; string title = persona["title"]?.ToString() ?? id; StringBuilder sb = new(); sb.AppendLine($"## Persona: {id} — {title}"); List voiceBits = []; if (voice["verbosity"] != null) { voiceBits.Add(voice["verbosity"].ToString()); } if (voice["tone"] is JArray tones) { voiceBits.AddRange(tones.Select(t => t?.ToString()).Where(s => !string.IsNullOrWhiteSpace(s))); } if (voice["humor"] != null && voice["humor"].ToString() != "none") { voiceBits.Add($"humor:{voice["humor"]}"); } if (voice["nsfw"] != null) { voiceBits.Add($"NSFW {voice["nsfw"]}"); } if (voice["language"] != null) { voiceBits.Add(voice["language"].ToString()); } if (voiceBits.Count > 0) { sb.AppendLine("Voice: " + string.Join(", ", voiceBits)); } string prefers = FormatCategoryMap(likes); if (!string.IsNullOrWhiteSpace(prefers)) { sb.AppendLine("Prefers: " + prefers); } string avoids = FormatCategoryMap(dislikes); if (!string.IsNullOrWhiteSpace(avoids)) { sb.AppendLine("Avoids: " + avoids); } List ruleBits = []; if (rules["always"] is JArray always) { foreach (string s in always.Select(t => t?.ToString()).Where(x => !string.IsNullOrWhiteSpace(x))) { ruleBits.Add("always " + s); } } if (rules["never"] is JArray never) { foreach (string s in never.Select(t => t?.ToString()).Where(x => !string.IsNullOrWhiteSpace(x))) { ruleBits.Add("never " + s); } } if (ruleBits.Count > 0) { sb.AppendLine("Rules: " + string.Join("; ", ruleBits)); } if (!string.IsNullOrWhiteSpace(extra)) { sb.AppendLine(extra.Trim()); } return sb.ToString().TrimEnd(); } static IEnumerable PersonaIdsUnder(string root) { string dir = Path.Combine(root ?? "", "personas"); if (!Directory.Exists(dir)) { yield break; } foreach (string folder in Directory.GetDirectories(dir)) { string id = SafeId(Path.GetFileName(folder)); if (id is not null) { yield return id; } } } /// Seed docs: shared from _base/memory-seed, personal from personas/<id>/memory-seed. Later files overwrite earlier same kind+key in the list; persona is stamped on each doc. public List LoadMemorySeedDocs() { List docs = []; void Scan(string dir, string persona) { if (string.IsNullOrWhiteSpace(dir) || !Directory.Exists(dir)) { return; } string stamp = AssistentMemory.NormalizePersona(persona); foreach (string file in Directory.GetFiles(dir, "*.json").OrderBy(f => f, StringComparer.OrdinalIgnoreCase)) { try { string raw = File.ReadAllText(file, Encoding.UTF8); JToken parsed = JToken.Parse(raw); IEnumerable items = parsed is JArray arr ? arr.OfType() : parsed is JObject single ? new[] { single } : []; foreach (JObject jo in items) { JObject clone = (JObject)jo.DeepClone(); if (clone["persona"] is null || string.IsNullOrWhiteSpace(clone["persona"]?.ToString())) { clone["persona"] = stamp; } docs.Add(clone); } } catch (Exception ex) { Logs.Debug($"AssistentConfig memory-seed {file}: {ex.Message}"); } } } Scan(Path.Combine(_bundledRoot, "_base", "memory-seed"), AssistentMemory.SharedPersona); Scan(Path.Combine(_overlayRoot, "_base", "memory-seed"), AssistentMemory.SharedPersona); Scan(Path.Combine(_overlayRoot, "memory-seed"), AssistentMemory.SharedPersona); foreach (string id in PersonaIdsUnder(_bundledRoot).Concat(PersonaIdsUnder(_overlayRoot)).Distinct(StringComparer.OrdinalIgnoreCase)) { Scan(Path.Combine(_bundledRoot, "personas", id, "memory-seed"), id); Scan(Path.Combine(_overlayRoot, "personas", id, "memory-seed"), id); } return docs; } public JObject LoadSettings() { return TryReadJson(Path.Combine(_overlayRoot, "settings.json")) ?? new JObject(); } public void SaveSettings(JObject settings) { Directory.CreateDirectory(_overlayRoot); string path = Path.Combine(_overlayRoot, "settings.json"); File.WriteAllText(path, (settings ?? new JObject()).ToString(Newtonsoft.Json.Formatting.Indented), Encoding.UTF8); } public JObject LoadOllamaRoles() { return TryReadJson(Path.Combine(_overlayRoot, "ollama-roles.json")) ?? new JObject { ["chat"] = new JArray(), ["memory"] = new JArray() }; } public List ResolveEnabledSkills(string personaId, JArray clientSkills) { var catalog = ListSkills(personaId); HashSet enabled = new(StringComparer.OrdinalIgnoreCase); foreach (var s in catalog.Where(x => x.defaultOn)) { enabled.Add(s.id); } JObject settings = LoadSettings(); string pid = SafeId(personaId) ?? "neutral"; if (settings[pid]?["skills"] is JObject perPersona) { foreach (JProperty prop in perPersona.Properties()) { string id = SafeId(prop.Name); if (id is null) { continue; } if (prop.Value.Type == JTokenType.Boolean) { if (prop.Value.Value()) { enabled.Add(id); } else { enabled.Remove(id); } } } } if (clientSkills is not null && clientSkills.Count > 0) { enabled.Clear(); foreach (JToken t in clientSkills) { string id = SafeId(t?.ToString()); if (id is not null && catalog.Any(c => string.Equals(c.id, id, StringComparison.OrdinalIgnoreCase))) { enabled.Add(id); } } } return catalog.Select(c => c.id).Where(enabled.Contains).ToList(); } public JObject BuildMergedConfigPayload(string personaId) { string id = SafeId(personaId) ?? DefaultPersonaId(); JObject assistant = LoadAssistant(id); JObject ui = LoadUi(id); JObject model = LoadModelProfile(id); JObject exact = LoadExact(id); var packs = ListPacks(id); var skills = ListSkills(id); var personas = ListPersonaCatalog(); JObject identity = LoadIdentityParts(id); return new JObject { ["success"] = true, ["persona"] = id, ["default_persona"] = DefaultPersonaId(), ["assistant"] = assistant, ["ui"] = ui, ["model"] = model, ["exact"] = exact, ["packs"] = new JArray(packs.Select(p => new JObject { ["id"] = p.id, ["title"] = p.title, ["order"] = p.order, ["aliases"] = new JArray(p.aliases), })), ["skills"] = new JArray(skills.Select(s => new JObject { ["id"] = s.id, ["title"] = s.title, ["default"] = s.defaultOn, })), ["personas"] = new JArray(personas.Select(p => new JObject { ["id"] = p.id, ["title"] = p.title, ["accent"] = p.accent, ["source"] = p.source, })), ["identity"] = identity, ["identity_summary"] = RenderIdentityBlock(id), ["enabled_skills"] = new JArray(ResolveEnabledSkills(id, null)), }; } }