diff --git a/Assets/assistent.css b/Assets/assistent.css index 4d20304..f43e566 100644 --- a/Assets/assistent.css +++ b/Assets/assistent.css @@ -427,6 +427,23 @@ letter-spacing: 0.03em; } +.sa-chats-search { + appearance: none; + width: 100%; + box-sizing: border-box; + border: 1px solid color-mix(in srgb, currentColor 22%, transparent); + background: color-mix(in srgb, #000 28%, transparent); + color: inherit; + border-radius: 0.35rem; + padding: 0.35rem 0.5rem; + font: inherit; + font-size: 0.82rem; +} + +.sa-chats-search:focus { + outline: 1px solid color-mix(in srgb, #6cf 55%, currentColor); +} + .sa-chats-panel-hint { font-size: 0.72rem; opacity: 0.65; diff --git a/Assets/assistent.js b/Assets/assistent.js index 5b110a4..491b8dd 100644 --- a/Assets/assistent.js +++ b/Assets/assistent.js @@ -4047,7 +4047,7 @@ } /** - * Fills fields the browser has never seen from Assistent/ui-state.json, so a fresh + * Fills fields the browser has never seen from sqlite ui_state, so a fresh * browser on the same volume inherits the previous session. Existing localStorage wins. * auto_download is only ever restored when it is off — the danger flag stays opt-in. */ @@ -5361,7 +5361,7 @@ } const remoteUpdated = remote.updated || 0; const localUpdated = state.taste?.updated || 0; - // Disk taste.json is the source of truth; localStorage only wins when it is strictly newer. + // sqlite taste is the source of truth; localStorage only wins when it is strictly newer. const localEmpty = !localUpdated && !(state.taste?.styles?.length || state.taste?.likes?.length || state.taste?.avoid?.length); if (localEmpty || remoteUpdated >= localUpdated) { diff --git a/Assets/assistent.persist.js b/Assets/assistent.persist.js index e17ab10..144d6ab 100644 --- a/Assets/assistent.persist.js +++ b/Assets/assistent.persist.js @@ -1,5 +1,5 @@ /** - * Swarm Assistent — disk persistence for chats + UI state (Assistent/chats/, Assistent/ui-state.json). + * Swarm Assistent — sqlite persistence for chats + UI state (Assistent/memory/assistent.sqlite). * Loaded after assistent.api.js and before assistent.js. */ window.SA = window.SA || {}; @@ -40,6 +40,7 @@ window.SA = window.SA || {}; createdAt: Number(raw.createdAt) || Date.now(), updatedAt: Number(raw.updatedAt) || Number(raw.createdAt) || Date.now(), messages: Array.isArray(raw.messages) ? raw.messages : [], + messages_count: Number(raw.messages_count) || (Array.isArray(raw.messages) ? raw.messages.length : 0), params: raw.params && typeof raw.params === 'object' ? raw.params : null, }; } @@ -74,7 +75,7 @@ window.SA = window.SA || {}; return local; } - /** Disk chats, newest first. Falls back to a localStorage migration when the volume is empty. */ + /** Sqlite chats, newest first. Falls back to a localStorage migration when the store is empty. */ async function loadChats() { let chats = []; try { @@ -103,6 +104,15 @@ window.SA = window.SA || {}; return normalizeChat(data?.chat); } + async function searchChats(q) { + const query = String(q || '').trim(); + if (query.length < 2) { + return []; + } + const data = await request('AssistentListChats', { q: query, with_messages: false, limit: 40 }); + return (data?.chats || []).map(normalizeChat).filter(Boolean); + } + function saveChat(chat, { immediate = false } = {}) { const clean = normalizeChat(chat); if (!clean) { @@ -183,6 +193,7 @@ window.SA = window.SA || {}; LS_CHATS, loadChats, getChat, + searchChats, saveChat, deleteChat, loadUiState, diff --git a/AssistentMemory.Store.cs b/AssistentMemory.Store.cs new file mode 100644 index 0000000..4fe9a60 --- /dev/null +++ b/AssistentMemory.Store.cs @@ -0,0 +1,485 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Data.Sqlite; +using Newtonsoft.Json.Linq; +using SwarmUI.Utils; + +namespace Mrleo1nid.SwarmAssistent; + +/// Runtime store in the same sqlite file: chats, ui-state, taste. +/// Config overlays, sidecar cards, and ollama-roles stay on disk. +public sealed partial class AssistentMemory +{ + public const int MaxChatsStored = 200; + public const string KvUiState = "ui_state"; + public const string KvTaste = "taste"; + + void EnsureStoreSchema() + { + Exec( + """ + CREATE TABLE IF NOT EXISTS kv ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS chats ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + params_json TEXT, + messages_json TEXT NOT NULL DEFAULT '[]', + messages_count INTEGER NOT NULL DEFAULT 0, + body TEXT NOT NULL DEFAULT '' + ); + CREATE INDEX IF NOT EXISTS idx_chats_updated ON chats(updated_at DESC); + """); + EnsureChatsFts(); + MigrateJsonStoreOnce(); + } + + void EnsureChatsFts() + { + try + { + Exec( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS chats_fts USING fts5( + title, + body, + tokenize = 'unicode61 remove_diacritics 2' + ); + """); + Exec( + """ + CREATE TRIGGER IF NOT EXISTS chats_fts_ai AFTER INSERT ON chats BEGIN + INSERT INTO chats_fts(rowid, title, body) VALUES (new.rowid, new.title, new.body); + END; + """); + Exec( + """ + CREATE TRIGGER IF NOT EXISTS chats_fts_ad AFTER DELETE ON chats BEGIN + INSERT INTO chats_fts(chats_fts, rowid) VALUES('delete', old.rowid); + END; + """); + Exec( + """ + CREATE TRIGGER IF NOT EXISTS chats_fts_au AFTER UPDATE ON chats BEGIN + INSERT INTO chats_fts(chats_fts, rowid) VALUES('delete', old.rowid); + INSERT INTO chats_fts(rowid, title, body) VALUES (new.rowid, new.title, new.body); + END; + """); + int n = 0, fts = 0; + using (SqliteCommand c = _conn.CreateCommand()) + { + c.CommandText = "SELECT COUNT(*) FROM chats"; + n = Convert.ToInt32(c.ExecuteScalar()); + } + using (SqliteCommand c = _conn.CreateCommand()) + { + c.CommandText = "SELECT COUNT(*) FROM chats_fts"; + fts = Convert.ToInt32(c.ExecuteScalar()); + } + if (n != fts) + { + Exec("DELETE FROM chats_fts"); + Exec("INSERT INTO chats_fts(rowid, title, body) SELECT rowid, title, body FROM chats"); + } + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory chats FTS5 unavailable: {ex.Message}"); + } + } + + void MigrateJsonStoreOnce() + { + if (GetMeta("json_store_migrated") == "1") + { + return; + } + string root = Path.Combine(_dataRoot, "Assistent"); + int chats = 0; + string chatsDir = Path.Combine(root, "chats"); + if (Directory.Exists(chatsDir)) + { + foreach (string file in Directory.EnumerateFiles(chatsDir, "*.json")) + { + try + { + JObject chat = JObject.Parse(File.ReadAllText(file, Encoding.UTF8)); + string id = (chat["id"]?.ToString() ?? Path.GetFileNameWithoutExtension(file) ?? "").Trim(); + if (string.IsNullOrWhiteSpace(id) || GetChatUnlocked(id) is not null) + { + continue; + } + UpsertChatUnlocked(chat, id); + chats++; + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory migrate chat {file}: {ex.Message}"); + } + } + } + ImportKvFile(Path.Combine(root, "ui-state.json"), KvUiState); + ImportKvFile(Path.Combine(root, "taste.json"), KvTaste); + SetMeta("json_store_migrated", "1"); + TryArchiveMigratedJson(root, chatsDir); + if (chats > 0) + { + Logs.Debug($"AssistentMemory: migrated {chats} chats from JSON into sqlite"); + } + } + + void ImportKvFile(string path, string key) + { + if (!File.Exists(path) || !string.IsNullOrEmpty(GetKvUnlocked(key))) + { + return; + } + try + { + SetKvUnlocked(key, File.ReadAllText(path, Encoding.UTF8)); + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory migrate {key}: {ex.Message}"); + } + } + + void TryArchiveMigratedJson(string root, string chatsDir) + { + try + { + string dest = Path.Combine(root, "_migrated_json"); + Directory.CreateDirectory(dest); + MoveIfExists(Path.Combine(root, "ui-state.json"), Path.Combine(dest, "ui-state.json")); + MoveIfExists(Path.Combine(root, "taste.json"), Path.Combine(dest, "taste.json")); + if (!Directory.Exists(chatsDir)) + { + return; + } + string chatsDest = Path.Combine(dest, "chats"); + Directory.CreateDirectory(chatsDest); + foreach (string file in Directory.EnumerateFiles(chatsDir, "*.json")) + { + MoveIfExists(file, Path.Combine(chatsDest, Path.GetFileName(file))); + } + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory archive json: {ex.Message}"); + } + } + + static void MoveIfExists(string src, string dest) + { + if (!File.Exists(src)) + { + return; + } + if (File.Exists(dest)) + { + File.Delete(src); + return; + } + Directory.CreateDirectory(Path.GetDirectoryName(dest)!); + File.Move(src, dest); + } + + public string GetKv(string key) + { + lock (_lock) + { + EnsureOpen(); + return GetKvUnlocked(key); + } + } + + public JObject GetKvObject(string key) + { + string raw = GetKv(key); + if (string.IsNullOrWhiteSpace(raw)) + { + return null; + } + try + { + return JObject.Parse(raw); + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory kv {key}: {ex.Message}"); + return null; + } + } + + public void SetKvObject(string key, JObject value) + { + SetKv(key, value?.ToString(Newtonsoft.Json.Formatting.None) ?? ""); + } + + public void SetKv(string key, string value) + { + lock (_lock) + { + EnsureOpen(); + SetKvUnlocked(key, value); + } + } + + string GetKvUnlocked(string key) + { + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT value FROM kv WHERE key = $k"; + cmd.Parameters.AddWithValue("$k", key ?? ""); + return cmd.ExecuteScalar()?.ToString(); + } + + void SetKvUnlocked(string key, string value) + { + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = + """ + INSERT INTO kv(key, value, updated) VALUES($k, $v, $u) + ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated = excluded.updated + """; + cmd.Parameters.AddWithValue("$k", key ?? ""); + cmd.Parameters.AddWithValue("$v", value ?? ""); + cmd.Parameters.AddWithValue("$u", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + cmd.ExecuteNonQuery(); + } + + public List ListChats(bool withMessages, int limit, string query = null) + { + lock (_lock) + { + EnsureOpen(); + int take = Math.Clamp(limit, 1, MaxChatsStored); + List list = []; + string match = BuildFtsMatch(query); + bool searched = false; + if (!string.IsNullOrWhiteSpace(match) && TableExists("chats_fts")) + { + try + { + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = + """ + SELECT id, title, created_at, updated_at, params_json, messages_json, messages_count + FROM chats + WHERE rowid IN (SELECT rowid FROM chats_fts WHERE chats_fts MATCH $q) + ORDER BY updated_at DESC LIMIT $lim + """; + cmd.Parameters.AddWithValue("$q", match); + cmd.Parameters.AddWithValue("$lim", take); + ReadChats(cmd, withMessages, list); + searched = true; + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory chats FTS: {ex.Message}"); + } + } + if (!searched && !string.IsNullOrWhiteSpace(query)) + { + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = + """ + SELECT id, title, created_at, updated_at, params_json, messages_json, messages_count + FROM chats + WHERE title LIKE $like ESCAPE '\' OR body LIKE $like ESCAPE '\' + ORDER BY updated_at DESC LIMIT $lim + """; + cmd.Parameters.AddWithValue("$like", "%" + EscapeLike(query.Trim()) + "%"); + cmd.Parameters.AddWithValue("$lim", take); + ReadChats(cmd, withMessages, list); + searched = true; + } + if (!searched) + { + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = + """ + SELECT id, title, created_at, updated_at, params_json, messages_json, messages_count + FROM chats + ORDER BY updated_at DESC LIMIT $lim + """; + cmd.Parameters.AddWithValue("$lim", take); + ReadChats(cmd, withMessages, list); + } + return list; + } + } + + public int CountChats() + { + lock (_lock) + { + EnsureOpen(); + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM chats"; + return Convert.ToInt32(cmd.ExecuteScalar()); + } + } + + public JObject GetChat(string id) + { + lock (_lock) + { + EnsureOpen(); + return GetChatUnlocked(id); + } + } + + public void SaveChat(JObject chat) + { + lock (_lock) + { + EnsureOpen(); + string id = chat?["id"]?.ToString()?.Trim(); + if (string.IsNullOrWhiteSpace(id)) + { + throw new ArgumentException("chat id required"); + } + UpsertChatUnlocked(chat, id); + PruneChatsUnlocked(); + } + } + + public bool DeleteChat(string id) + { + lock (_lock) + { + EnsureOpen(); + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = "DELETE FROM chats WHERE id = $id"; + cmd.Parameters.AddWithValue("$id", (id ?? "").Trim()); + return cmd.ExecuteNonQuery() > 0; + } + } + + JObject GetChatUnlocked(string id) + { + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = + """ + SELECT id, title, created_at, updated_at, params_json, messages_json, messages_count + FROM chats WHERE id = $id LIMIT 1 + """; + cmd.Parameters.AddWithValue("$id", (id ?? "").Trim()); + List list = []; + ReadChats(cmd, withMessages: true, list); + return list.Count > 0 ? list[0] : null; + } + + void UpsertChatUnlocked(JObject chat, string id) + { + JArray messages = chat["messages"] as JArray ?? []; + string title = chat["title"]?.ToString() ?? "Новый чат"; + long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + long created = chat["createdAt"]?.Value() ?? now; + long updated = chat["updatedAt"]?.Value() ?? now; + string paramsJson = chat["params"] is JObject p ? p.ToString(Newtonsoft.Json.Formatting.None) : null; + string messagesJson = messages.ToString(Newtonsoft.Json.Formatting.None); + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = + """ + INSERT INTO chats(id, title, created_at, updated_at, params_json, messages_json, messages_count, body) + VALUES($id, $title, $created, $updated, $params, $messages, $count, $body) + ON CONFLICT(id) DO UPDATE SET + title = excluded.title, + created_at = excluded.created_at, + updated_at = excluded.updated_at, + params_json = excluded.params_json, + messages_json = excluded.messages_json, + messages_count = excluded.messages_count, + body = excluded.body + """; + cmd.Parameters.AddWithValue("$id", id); + cmd.Parameters.AddWithValue("$title", title); + cmd.Parameters.AddWithValue("$created", created); + cmd.Parameters.AddWithValue("$updated", updated); + cmd.Parameters.AddWithValue("$params", (object)paramsJson ?? DBNull.Value); + cmd.Parameters.AddWithValue("$messages", messagesJson); + cmd.Parameters.AddWithValue("$count", messages.Count); + cmd.Parameters.AddWithValue("$body", ChatFtsBody(title, messages)); + cmd.ExecuteNonQuery(); + } + + void PruneChatsUnlocked() + { + using SqliteCommand cmd = _conn.CreateCommand(); + cmd.CommandText = + """ + DELETE FROM chats WHERE rowid IN ( + SELECT rowid FROM chats ORDER BY updated_at DESC LIMIT -1 OFFSET $keep + ) + """; + cmd.Parameters.AddWithValue("$keep", MaxChatsStored); + cmd.ExecuteNonQuery(); + } + + static void ReadChats(SqliteCommand cmd, bool withMessages, List list) + { + using SqliteDataReader reader = cmd.ExecuteReader(); + while (reader.Read()) + { + list.Add(ReadChatRow(reader, withMessages)); + } + } + + static JObject ReadChatRow(SqliteDataReader reader, bool withMessages) + { + JObject chat = new() + { + ["id"] = reader.GetString(0), + ["title"] = reader.IsDBNull(1) ? "Новый чат" : reader.GetString(1), + ["createdAt"] = reader.GetInt64(2), + ["updatedAt"] = reader.GetInt64(3), + ["messages_count"] = reader.GetInt32(6), + }; + if (!reader.IsDBNull(4)) + { + try + { + chat["params"] = JObject.Parse(reader.GetString(4)); + } + catch + { + chat["params"] = null; + } + } + if (withMessages) + { + try + { + chat["messages"] = reader.IsDBNull(5) ? new JArray() : JArray.Parse(reader.GetString(5)); + } + catch + { + chat["messages"] = new JArray(); + } + } + return chat; + } + + static string ChatFtsBody(string title, JArray messages) + { + StringBuilder sb = new(); + sb.Append(title).Append('\n'); + foreach (JToken token in messages ?? []) + { + if (token is JObject mo) + { + sb.Append(mo["content"]?.ToString()).Append('\n'); + } + } + return sb.ToString(); + } +} diff --git a/AssistentMemory.cs b/AssistentMemory.cs index 27d91d4..613aeb5 100644 --- a/AssistentMemory.cs +++ b/AssistentMemory.cs @@ -202,7 +202,7 @@ public sealed partial class AssistentMemory : IDisposable { using SqliteCommand cmd = _conn.CreateCommand(); cmd.CommandText = "PRAGMA " + pragma; - cmd.ExecuteNonQuery(); + _ = cmd.ExecuteScalar(); } catch (Exception ex) { diff --git a/AssistentPersist.cs b/AssistentPersist.cs index 4de6d2a..16de492 100644 --- a/AssistentPersist.cs +++ b/AssistentPersist.cs @@ -1,9 +1,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Text; -using System.Text.RegularExpressions; using System.Threading.Tasks; using Newtonsoft.Json.Linq; using SwarmUI.Accounts; @@ -11,15 +8,14 @@ using SwarmUI.Utils; namespace Mrleo1nid.SwarmAssistent; -/// Disk persistence for chat sessions and UI state under DataRoot()/Assistent/. -/// Chats survive browser storage wipes and follow the data volume across gpu-rent VMs. +/// Chat sessions and UI state in Assistent/memory/assistent.sqlite. +/// Survives browser wipes and follows the data volume across gpu-rent VMs. public partial class SwarmAssistentExtension { - const int MaxChatsOnDisk = 60; const int MaxChatMessagesOnDisk = 40; const int MaxChatMessageChars = 4000; - static readonly Regex ChatIdRe = new(@"^[A-Za-z0-9][A-Za-z0-9_\-]{0,63}$", RegexOptions.Compiled); + static readonly System.Text.RegularExpressions.Regex ChatIdRe = new(@"^[A-Za-z0-9][A-Za-z0-9_\-]{0,63}$", System.Text.RegularExpressions.RegexOptions.Compiled); /// UI-state keys accepted from the browser — anything else is dropped. static readonly string[] UiStateKeys = @@ -28,93 +24,35 @@ public partial class SwarmAssistentExtension "auto_download", "pane_width", "embed_model", "base_url", "model", "view", "board_tab", ]; - string AssistentDataDir() => Path.Combine(DataRoot(), "Assistent"); - - string AssistentChatsDir() => Path.Combine(AssistentDataDir(), "chats"); - - string AssistentUiStatePath() => Path.Combine(AssistentDataDir(), "ui-state.json"); - static string SafeChatId(string id) { string s = (id ?? "").Trim(); return ChatIdRe.IsMatch(s) ? s : null; } - string ChatFilePath(string id) - { - string safe = SafeChatId(id); - return safe is null ? null : Path.Combine(AssistentChatsDir(), $"{safe}.json"); - } - - JObject ReadChatFile(string path) - { - if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) - { - return null; - } - try - { - JObject chat = JObject.Parse(File.ReadAllText(path, Encoding.UTF8)); - string id = SafeChatId(chat["id"]?.ToString() ?? Path.GetFileNameWithoutExtension(path)); - if (id is null) - { - return null; - } - chat["id"] = id; - return chat; - } - catch (Exception ex) - { - Logs.Debug($"AssistentPersist read {path}: {ex.Message}"); - return null; - } - } - static JObject ChatSummary(JObject chat) { - JArray messages = chat["messages"] as JArray ?? []; return new JObject { ["id"] = chat["id"], ["title"] = chat["title"]?.ToString() ?? "Новый чат", ["createdAt"] = chat["createdAt"] ?? 0, ["updatedAt"] = chat["updatedAt"] ?? 0, - ["messages_count"] = messages.Count, + ["messages_count"] = chat["messages_count"] ?? (chat["messages"] as JArray)?.Count ?? 0, ["params"] = chat["params"], }; } - List LoadAllChats() - { - string dir = AssistentChatsDir(); - if (!Directory.Exists(dir)) - { - return []; - } - List chats = []; - foreach (string file in Directory.EnumerateFiles(dir, "*.json")) - { - JObject chat = ReadChatFile(file); - if (chat is not null) - { - chats.Add(chat); - } - } - return chats - .OrderByDescending(c => c["updatedAt"]?.Value() ?? 0) - .ToList(); - } - - /// All chats on disk, newest first. Pass with_messages to get full transcripts. - public async Task AssistentListChats(Session session, bool with_messages = false, int limit = MaxChatsOnDisk) + /// All chats, newest first. searches title+body via FTS. + public async Task AssistentListChats(Session session, bool with_messages = false, int limit = 60, string q = null) { await Task.CompletedTask; try { - List chats = LoadAllChats(); - int take = Math.Clamp(limit, 1, MaxChatsOnDisk); + int take = Math.Clamp(limit, 1, AssistentMemory.MaxChatsStored); + List chats = Memory.ListChats(with_messages, take, q); JArray list = []; - foreach (JObject chat in chats.Take(take)) + foreach (JObject chat in chats) { list.Add(with_messages ? chat : ChatSummary(chat)); } @@ -122,8 +60,8 @@ public partial class SwarmAssistentExtension { ["success"] = true, ["chats"] = list, - ["total"] = chats.Count, - ["path"] = AssistentChatsDir(), + ["total"] = string.IsNullOrWhiteSpace(q) ? Memory.CountChats() : list.Count, + ["path"] = "Assistent/memory/assistent.sqlite", }; } catch (Exception ex) @@ -135,22 +73,29 @@ public partial class SwarmAssistentExtension public async Task AssistentGetChat(Session session, string id) { await Task.CompletedTask; - string path = ChatFilePath(id); - if (path is null) + string safe = SafeChatId(id); + if (safe is null) { return new JObject { ["error"] = "valid id required" }; } - JObject chat = ReadChatFile(path); - return new JObject + try { - ["success"] = true, - ["id"] = SafeChatId(id), - ["found"] = chat is not null, - ["chat"] = chat, - }; + JObject chat = Memory.GetChat(safe); + return new JObject + { + ["success"] = true, + ["id"] = safe, + ["found"] = chat is not null, + ["chat"] = chat, + }; + } + catch (Exception ex) + { + return new JObject { ["error"] = $"chat get: {ex.Message}" }; + } } - /// Writes one chat to Assistent/chats/<id>.json. + /// Writes one chat into sqlite. /// SwarmUI hands the whole request body to a JObject param, so messages (array) /// and params (object) are read out of . public async Task AssistentSaveChat(Session session, string id, string title, JObject raw) @@ -187,8 +132,15 @@ public partial class SwarmAssistentExtension } long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - string path = ChatFilePath(safe); - JObject existing = ReadChatFile(path); + JObject existing = null; + try + { + existing = Memory.GetChat(safe); + } + catch (Exception ex) + { + Logs.Debug($"AssistentSaveChat existing: {ex.Message}"); + } JObject chat = new() { ["id"] = safe, @@ -196,14 +148,12 @@ public partial class SwarmAssistentExtension ["createdAt"] = raw?["createdAt"]?.Value() ?? existing?["createdAt"]?.Value() ?? now, ["updatedAt"] = raw?["updatedAt"]?.Value() ?? now, ["messages"] = trimmed, - ["params"] = chatParams ?? existing?["params"], + ["params"] = chatParams ?? existing?["params"] as JObject, }; try { - Directory.CreateDirectory(AssistentChatsDir()); - File.WriteAllText(path, chat.ToString(Newtonsoft.Json.Formatting.Indented), Encoding.UTF8); - PruneChatsOnDisk(); - return new JObject { ["success"] = true, ["id"] = safe, ["path"] = path }; + Memory.SaveChat(chat); + return new JObject { ["success"] = true, ["id"] = safe, ["path"] = "Assistent/memory/assistent.sqlite" }; } catch (Exception ex) { @@ -214,19 +164,15 @@ public partial class SwarmAssistentExtension public async Task AssistentDeleteChat(Session session, string id) { await Task.CompletedTask; - string path = ChatFilePath(id); - if (path is null) + string safe = SafeChatId(id); + if (safe is null) { return new JObject { ["error"] = "valid id required" }; } try { - bool existed = File.Exists(path); - if (existed) - { - File.Delete(path); - } - return new JObject { ["success"] = true, ["deleted"] = existed, ["id"] = SafeChatId(id) }; + bool existed = Memory.DeleteChat(safe); + return new JObject { ["success"] = true, ["deleted"] = existed, ["id"] = safe }; } catch (Exception ex) { @@ -234,49 +180,20 @@ public partial class SwarmAssistentExtension } } - void PruneChatsOnDisk() - { - try - { - List chats = LoadAllChats(); - if (chats.Count <= MaxChatsOnDisk) - { - return; - } - foreach (JObject stale in chats.Skip(MaxChatsOnDisk)) - { - string path = ChatFilePath(stale["id"]?.ToString()); - if (path is not null && File.Exists(path)) - { - File.Delete(path); - } - } - } - catch (Exception ex) - { - Logs.Debug($"AssistentPersist prune: {ex.Message}"); - } - } - public async Task AssistentGetUiState(Session session) { await Task.CompletedTask; - string path = AssistentUiStatePath(); - if (!File.Exists(path)) - { - return new JObject { ["success"] = true, ["ui_state"] = null }; - } try { return new JObject { ["success"] = true, - ["ui_state"] = JObject.Parse(File.ReadAllText(path, Encoding.UTF8)), + ["ui_state"] = Memory.GetKvObject(AssistentMemory.KvUiState), }; } catch (Exception ex) { - return new JObject { ["error"] = $"ui-state.json: {ex.Message}" }; + return new JObject { ["error"] = $"ui-state: {ex.Message}" }; } } @@ -307,10 +224,8 @@ public partial class SwarmAssistentExtension clean["updated"] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); try { - Directory.CreateDirectory(AssistentDataDir()); - string path = AssistentUiStatePath(); - File.WriteAllText(path, clean.ToString(Newtonsoft.Json.Formatting.Indented), Encoding.UTF8); - return new JObject { ["success"] = true, ["path"] = path }; + Memory.SetKvObject(AssistentMemory.KvUiState, clean); + return new JObject { ["success"] = true, ["path"] = "Assistent/memory/assistent.sqlite" }; } catch (Exception ex) { diff --git a/Config/_base/skills/memory.md b/Config/_base/skills/memory.md index bd67c2c..6042e86 100644 --- a/Config/_base/skills/memory.md +++ b/Config/_base/skills/memory.md @@ -28,7 +28,7 @@ Omit the tool action on the follow-up turn once you have results. ## When not to write - Do not dump Exact defaults or the full inventory into vector memory. -- Do not store the user's taste profile (`taste.json`). +- Do not store the user's taste profile (sqlite `kv.taste`). - Do not upsert trivia already in `memory_hits`. - Do not upsert Danbooru tags — the csv catalog already has them. - `memory_forget` without `scope` only removes the personal overlay. diff --git a/README.md b/README.md index d3d5fc2..25a6d42 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ SwarmUI extension for **collaborative Krea 2** prompting via **Ollama**: chat + board (Generate | Refs tabs), LoRA chips, **persona presets** (`Config/personas/`), **vector memory**, model cards with Civitai fetch, img2img/inpaint, slash commands, auto Generate. -**Version 0.8.2** — Hybrid memory (FTS5 + cosine, kind quotas, min_score) plus model hops `memory_get` / `memory_search` / `lookup_tags`. Danbooru csv is a shared FTS catalog (no embeddings); Krea prompts stay prose. +**Version 0.8.3** — Chats, UI state and taste live in `assistent.sqlite` (FTS search in History). Memory stays hybrid FTS5 + cosine. Danbooru csv is a shared FTS catalog (no embeddings); Krea prompts stay prose. ## Layout @@ -24,12 +24,10 @@ Disk overlay (wins over bundled): `/mnt/swarm_data/Assistent/` — same folder l ``` Assistent/ _base/ personas// # overlay presets — same names as Config/, sparse - settings.json # embed_model, base_url, per-persona skills - ui-state.json # pack / persona / auto_* / pane_width / models — seeds a fresh browser - taste.json # learned taste profile (wins over localStorage) - chats/.json # chat history, newest 60 kept - ollama-roles.json # chat vs memory model tags - memory/assistent.sqlite # vector store (shared + per-persona) + settings.json # embed_model, base_url, per-persona skills (config overlay) + ollama-roles.json # chat vs memory model tags (gpu-rent writes this) + memory/assistent.sqlite # vector memory + tags FTS + chats + ui_state + taste + _migrated_json/ # one-shot archive of old chats/*.json, ui-state.json, taste.json ``` Copy `personas/cinema/` → `noir/`, edit only differing JSON files. Persona prompt overrides belong in `personas//` — the old flat `personas.json` is legacy and only read when no overlay folder exists for that id. @@ -54,12 +52,13 @@ Two layers in `memory/assistent.sqlite` (`persona` column; empty = shared): - Soft notes only — Exact and the user beat RAG for params - ⚙ → **Память** lists every row (scope · source · date) with a per-row forget; bundled rows are read-only because reseed brings them back -## Chats on disk - -- Every chat is written to `Assistent/chats/.json` (messages + a Generate params snapshot), so History survives a cleared browser and follows the data volume across VMs -- localStorage stays as a fast cache; on first run with an empty `chats/` the old `swarm_assistent_chats_v1` store is migrated up once -- `ui-state.json` seeds a **fresh** browser only — anything already in localStorage wins, and `auto_download` is never restored as on +## Chats and runtime KV +- Every chat (messages + Generate params snapshot) is a row in `assistent.sqlite`, newest **200** kept. History search uses FTS over title + body. +- First launch after 0.8.3 copies `chats/*.json`, `ui-state.json`, and `taste.json` into sqlite, then archives them under `_migrated_json/`. +- localStorage stays as a fast cache; on first run with an empty store the old `swarm_assistent_chats_v1` browser history is migrated up once. +- UI state seeds a **fresh** browser only — anything already in localStorage wins, and `auto_download` is never restored as on +- `settings.json` and persona overlays stay files (layered merge + git). `.assistent.json` cards stay next to weights. ## VRAM handover - Before every Generate the chat model is unloaded (`keep_alive: 0`) so Krea 2 gets the whole GPU @@ -132,14 +131,14 @@ Restart / rebuild SwarmUI after clone. gpu-rent: `seed-extensions` + restart. | `AssistentGetCard` / `AssistentSaveCard` | `.assistent.json` cards (+ memory ingest) | | `AssistentGetCardMeta` | Local sidecar + optional Civitai by-hash | | `AssistentEnqueueWanted` / `AssistentListWanted` | Wanted YAML queue (write / read + count) | -| `AssistentGetTaste` / `AssistentSaveTaste` | Persistent taste profile | +| `AssistentGetTaste` / `AssistentSaveTaste` | sqlite `kv.taste` | | `AssistentSearchCivitai` | Civitai LoRA search | | `AssistentChat` / `AssistentChatWS` | Chat (+ hybrid memory + Civitai/tag hops) | | `AssistentListMemory` / `AssistentUpsertMemory` / `AssistentForgetMemory` | Vector store (optional `scope` / `persona`) | | `AssistentSearchMemory` / `AssistentGetMemory` | Hybrid search / exact kind+key | | `AssistentLookupTags` | Danbooru csv FTS (no embeddings) | -| `AssistentListChats` / `AssistentGetChat` / `AssistentSaveChat` / `AssistentDeleteChat` | `Assistent/chats/.json` | -| `AssistentGetUiState` / `AssistentSaveUiState` | `Assistent/ui-state.json` | +| `AssistentListChats` / `AssistentGetChat` / `AssistentSaveChat` / `AssistentDeleteChat` | sqlite `chats` (optional `q` FTS) | +| `AssistentGetUiState` / `AssistentSaveUiState` | sqlite `kv.ui_state` | | `AssistentParkLlm` / `AssistentWarmLlm` | Unload / reload the chat model in VRAM | ## License diff --git a/SwarmAssistentExtension.cs b/SwarmAssistentExtension.cs index b514a85..6a6acf5 100644 --- a/SwarmAssistentExtension.cs +++ b/SwarmAssistentExtension.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using Newtonsoft.Json.Linq; diff --git a/Tabs/Text2Image/Assistent.html b/Tabs/Text2Image/Assistent.html index 33b439f..b8beb97 100644 --- a/Tabs/Text2Image/Assistent.html +++ b/Tabs/Text2Image/Assistent.html @@ -50,6 +50,7 @@