mrleo1nid/swarm-assistent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ship Assistent 0.8.1: split modules and shared+personal vector memory.

Browse Source 
Personal RAG never leaks into the shared store; retrieve merges shared plus the persona chain, with personal overwrite on kind+key.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-22 00:38:54 +03:00
co-authored by Cursor
parent 6d8aaefc38
commit 880e2dbea2
21 changed files with 3946 additions and 2040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AssistentMemory.cs
+257 -46
View File
@@ -11,9 +11,13 @@ using SwarmUI.Utils;
namespace Mrleo1nid.SwarmAssistent;
/// <summary>Local SQLite vector memory with Ollama /api/embed.</summary>
/// <summary>Local SQLite vector memory with Ollama /api/embed.
/// Two layers: shared (persona='') is visible to every personality; personal (persona=id)
/// is not written back to shared. On retrieve, personal overwrites shared on the same kind+key.</summary>
public sealed class AssistentMemory : IDisposable
{
public const string SharedPersona = "";
readonly string _dbPath;
readonly HttpClient _http;
readonly object _lock = new();
@@ -35,6 +39,22 @@ public sealed class AssistentMemory : IDisposable
public int Dims => _dims;
public int SeedVersion => _seedVersion;
public static string NormalizePersona(string raw)
{
if (string.IsNullOrWhiteSpace(raw))
{
return SharedPersona;
}
string s = raw.Trim();
if (s is "_" or "*" or "shared" or "common" or "global" or "_shared")
{
return SharedPersona;
}
return AssistentConfig.SafeId(s) ?? SharedPersona;
}
public static bool IsShared(string persona) => string.IsNullOrEmpty(NormalizePersona(persona));
void EnsureOpen()
{
if (_conn is not null)
@@ -55,22 +75,85 @@ public sealed class AssistentMemory : IDisposable
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
key TEXT NOT NULL,
persona TEXT NOT NULL DEFAULT '',
text TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'user',
meta_json TEXT,
embedding BLOB,
updated INTEGER NOT NULL,
UNIQUE(kind, key, source)
UNIQUE(kind, key, source, persona)
);
CREATE INDEX IF NOT EXISTS idx_memories_kind ON memories(kind);
""";
cmd.ExecuteNonQuery();
}
MigratePersonaColumn();
using (SqliteCommand idx = _conn.CreateCommand())
{
idx.CommandText =
"""
CREATE INDEX IF NOT EXISTS idx_memories_kind ON memories(kind);
CREATE INDEX IF NOT EXISTS idx_memories_persona ON memories(persona);
""";
idx.ExecuteNonQuery();
}
_embedModel = GetMeta("embed_model") ?? _embedModel;
_ = int.TryParse(GetMeta("dims"), out _dims);
_ = int.TryParse(GetMeta("seed_version"), out _seedVersion);
}
bool HasColumn(string table, string column)
{
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = $"PRAGMA table_info({table})";
using SqliteDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
if (string.Equals(reader.GetString(1), column, StringComparison.OrdinalIgnoreCase))
{
return true;
}
}
return false;
}
void MigratePersonaColumn()
{
if (HasColumn("memories", "persona"))
{
return;
}
using SqliteTransaction tx = _conn.BeginTransaction();
using (SqliteCommand cmd = _conn.CreateCommand())
{
cmd.Transaction = tx;
cmd.CommandText =
"""
CREATE TABLE memories_v2 (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL,
key TEXT NOT NULL,
persona TEXT NOT NULL DEFAULT '',
text TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'user',
meta_json TEXT,
embedding BLOB,
updated INTEGER NOT NULL,
UNIQUE(kind, key, source, persona)
);
INSERT INTO memories_v2 (kind, key, persona, text, source, meta_json, embedding, updated)
SELECT kind, key, '', text, source, meta_json, embedding, updated FROM memories;
DROP TABLE memories;
ALTER TABLE memories_v2 RENAME TO memories;
CREATE INDEX IF NOT EXISTS idx_memories_kind ON memories(kind);
CREATE INDEX IF NOT EXISTS idx_memories_persona ON memories(persona);
""";
cmd.ExecuteNonQuery();
}
tx.Commit();
SetMeta("schema_version", "2");
Logs.Debug("AssistentMemory: migrated sqlite to shared+personal persona column (existing rows → shared)");
}
string GetMeta(string key)
{
using SqliteCommand cmd = _conn.CreateCommand();
@@ -126,6 +209,11 @@ public sealed class AssistentMemory : IDisposable
return (float)(dot / (Math.Sqrt(na) * Math.Sqrt(nb)));
}
static int SourceRank(string source)
{
return string.Equals(source, "user", StringComparison.OrdinalIgnoreCase) ? 1 : 0;
}
public async Task<float[]> EmbedAsync(string baseUrl, string model, string text, string keepAlive = "60m")
{
string root = (baseUrl ?? "http://127.0.0.1:11434").TrimEnd('/');
@@ -154,6 +242,16 @@ public sealed class AssistentMemory : IDisposable
return floats;
}
bool BundledExists(string kind, string key, string persona)
{
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT 1 FROM memories WHERE source = 'bundled' AND kind = $kind AND key = $key AND persona = $persona LIMIT 1";
cmd.Parameters.AddWithValue("$kind", kind);
cmd.Parameters.AddWithValue("$key", key);
cmd.Parameters.AddWithValue("$persona", persona ?? SharedPersona);
return cmd.ExecuteScalar() is not null;
}
public async Task EnsureSeedAsync(string baseUrl, AssistentConfig config, string modelOverride = null)
{
lock (_lock)
@@ -167,28 +265,32 @@ public sealed class AssistentMemory : IDisposable
: modelOverride.Trim();
bool needReseed = _seedVersion != wantVersion || !string.Equals(_embedModel, wantModel, StringComparison.OrdinalIgnoreCase);
if (!needReseed)
{
int bundledCount;
lock (_lock)
{
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM memories WHERE source = 'bundled'";
bundledCount = Convert.ToInt32(cmd.ExecuteScalar());
}
if (bundledCount > 0)
{
return;
}
}
List<JObject> docs = config.LoadMemorySeedDocs();
if (docs.Count == 0)
{
return;
}
// Probe embed
if (!needReseed)
{
bool missing;
lock (_lock)
{
EnsureOpen();
missing = docs.Any(d =>
{
string kind = (d["kind"]?.ToString() ?? "note").Trim().ToLowerInvariant();
string key = (d["key"]?.ToString() ?? "").Trim();
string persona = NormalizePersona(d["persona"]?.ToString());
return !string.IsNullOrWhiteSpace(key) && !BundledExists(kind, key, persona);
});
}
if (!missing)
{
return;
}
}
float[] probe;
try
{
@@ -228,14 +330,26 @@ public sealed class AssistentMemory : IDisposable
string kind = (doc["kind"]?.ToString() ?? "note").Trim();
string key = (doc["key"]?.ToString() ?? "").Trim();
string text = (doc["text"]?.ToString() ?? "").Trim();
string persona = NormalizePersona(doc["persona"]?.ToString());
if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(text))
{
continue;
}
if (!needReseed)
{
lock (_lock)
{
EnsureOpen();
if (BundledExists(kind.ToLowerInvariant(), key, persona))
{
continue;
}
}
}
try
{
float[] vec = await EmbedAsync(baseUrl, wantModel, text);
Upsert(kind, key, text, "bundled", doc["tags"], vec);
Upsert(kind, key, text, "bundled", doc["tags"], vec, persona);
}
catch (Exception ex)
{
@@ -244,7 +358,7 @@ public sealed class AssistentMemory : IDisposable
}
}
public void Upsert(string kind, string key, string text, string source, JToken meta, float[] embedding)
public void Upsert(string kind, string key, string text, string source, JToken meta, float[] embedding, string persona = null)
{
lock (_lock)
{
@@ -253,6 +367,7 @@ public sealed class AssistentMemory : IDisposable
key = (key ?? "").Trim();
text = (text ?? "").Trim();
source = string.IsNullOrWhiteSpace(source) ? "user" : source.Trim();
persona = NormalizePersona(persona);
if (string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(text))
{
return;
@@ -268,9 +383,9 @@ public sealed class AssistentMemory : IDisposable
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText =
"""
INSERT INTO memories(kind, key, text, source, meta_json, embedding, updated)
VALUES($kind, $key, $text, $source, $meta, $emb, $upd)
ON CONFLICT(kind, key, source) DO UPDATE SET
INSERT INTO memories(kind, key, persona, text, source, meta_json, embedding, updated)
VALUES($kind, $key, $persona, $text, $source, $meta, $emb, $upd)
ON CONFLICT(kind, key, source, persona) DO UPDATE SET
text = excluded.text,
meta_json = excluded.meta_json,
embedding = excluded.embedding,
@@ -278,6 +393,7 @@ public sealed class AssistentMemory : IDisposable
""";
cmd.Parameters.AddWithValue("$kind", kind);
cmd.Parameters.AddWithValue("$key", key);
cmd.Parameters.AddWithValue("$persona", persona);
cmd.Parameters.AddWithValue("$text", text);
cmd.Parameters.AddWithValue("$source", source);
cmd.Parameters.AddWithValue("$meta", meta?.ToString(Newtonsoft.Json.Formatting.None) ?? "");
@@ -287,28 +403,33 @@ public sealed class AssistentMemory : IDisposable
}
}
public void Forget(string kind, string key, string source = null)
/// <summary>Delete a row in one layer. Default (persona=current) never touches shared.
/// Pass SharedPersona to forget a shared fact. Bundled rows are kept unless source is set.</summary>
public void Forget(string kind, string key, string source = null, string persona = null)
{
lock (_lock)
{
EnsureOpen();
persona = NormalizePersona(persona);
using SqliteCommand cmd = _conn.CreateCommand();
if (string.IsNullOrWhiteSpace(source))
{
cmd.CommandText = "DELETE FROM memories WHERE kind = $kind AND key = $key AND source != 'bundled'";
cmd.CommandText = "DELETE FROM memories WHERE kind = $kind AND key = $key AND persona = $persona AND source != 'bundled'";
}
else
{
cmd.CommandText = "DELETE FROM memories WHERE kind = $kind AND key = $key AND source = $source";
cmd.CommandText = "DELETE FROM memories WHERE kind = $kind AND key = $key AND persona = $persona AND source = $source";
cmd.Parameters.AddWithValue("$source", source);
}
cmd.Parameters.AddWithValue("$kind", (kind ?? "").Trim().ToLowerInvariant());
cmd.Parameters.AddWithValue("$key", (key ?? "").Trim());
cmd.Parameters.AddWithValue("$persona", persona);
cmd.ExecuteNonQuery();
}
}
public async Task<JArray> RetrieveAsync(string baseUrl, string query, int topK = 10, string modelOverride = null)
/// <summary>Retrieve shared + the given persona chain. Personal overwrites shared (and parent personas) on kind+key.</summary>
public async Task<JArray> RetrieveAsync(string baseUrl, string query, int topK = 10, string modelOverride = null, IEnumerable<string> personaChain = null)
{
if (string.IsNullOrWhiteSpace(query))
{
@@ -330,53 +451,143 @@ public sealed class AssistentMemory : IDisposable
return [];
}
List<(float score, JObject row)> scored = [];
Dictionary<string, int> rank = new(StringComparer.OrdinalIgnoreCase)
{
[SharedPersona] = 0,
};
int i = 1;
foreach (string id in personaChain ?? [])
{
string p = NormalizePersona(id);
if (p == SharedPersona)
{
continue;
}
rank[p] = i++;
}
List<(float score, int personaRank, int sourceRank, JObject row)> scored = [];
lock (_lock)
{
EnsureOpen();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT kind, key, text, source, meta_json, embedding FROM memories WHERE embedding IS NOT NULL";
cmd.CommandText = "SELECT kind, key, text, source, meta_json, embedding, persona FROM memories WHERE embedding IS NOT NULL";
using SqliteDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
string persona = reader.IsDBNull(6) ? SharedPersona : reader.GetString(6) ?? SharedPersona;
if (!rank.TryGetValue(persona, out int personaRank))
{
continue;
}
float[] emb = BytesToFloats(reader.IsDBNull(5) ? null : (byte[])reader.GetValue(5));
float score = Cosine(q, emb);
if (float.IsNegativeInfinity(score))
{
continue;
}
scored.Add((score, new JObject
string kind = reader.GetString(0);
string key = reader.GetString(1);
string source = reader.GetString(3);
bool shared = persona == SharedPersona;
scored.Add((score, personaRank, SourceRank(source), new JObject
{
["kind"] = kind,
["key"] = key,
["text"] = reader.GetString(2),
["source"] = source,
["scope"] = shared ? "shared" : "personal",
["persona"] = shared ? "shared" : persona,
["score"] = Math.Round(score, 4),
}));
}
}
// Personal (and later parents) overwrite shared on the same kind+key; user beats bundled.
Dictionary<string, (float score, int personaRank, int sourceRank, JObject row)> best = new(StringComparer.OrdinalIgnoreCase);
foreach (var item in scored)
{
string id = $"{item.row["kind"]}\n{item.row["key"]}";
if (best.TryGetValue(id, out var cur))
{
if (item.personaRank < cur.personaRank
|| (item.personaRank == cur.personaRank && item.sourceRank < cur.sourceRank)
|| (item.personaRank == cur.personaRank && item.sourceRank == cur.sourceRank && item.score <= cur.score))
{
continue;
}
}
best[id] = item;
}
return new JArray(best.Values.OrderByDescending(s => s.score).Take(Math.Clamp(topK, 1, 30)).Select(s => s.row));
}
public JArray ListAll(int limit = 200)
{
lock (_lock)
{
EnsureOpen();
List<JObject> rows = [];
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT kind, key, text, source, persona, updated FROM memories ORDER BY updated DESC LIMIT $lim";
cmd.Parameters.AddWithValue("$lim", Math.Clamp(limit, 1, 2000));
using SqliteDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
string persona = reader.IsDBNull(4) ? SharedPersona : reader.GetString(4) ?? SharedPersona;
bool shared = string.IsNullOrEmpty(persona);
rows.Add(new JObject
{
["kind"] = reader.GetString(0),
["key"] = reader.GetString(1),
["text"] = reader.GetString(2),
["source"] = reader.GetString(3),
["score"] = Math.Round(score, 4),
}));
["scope"] = shared ? "shared" : "personal",
["persona"] = shared ? "shared" : persona,
["updated"] = reader.IsDBNull(5) ? 0 : reader.GetInt64(5),
});
}
return new JArray(rows);
}
return new JArray(scored.OrderByDescending(s => s.score).Take(Math.Clamp(topK, 1, 30)).Select(s => s.row));
}
public async Task UpsertTextAsync(string baseUrl, string kind, string key, string text, string source = "user", JToken meta = null, string modelOverride = null)
public int CountAll()
{
string model = string.IsNullOrWhiteSpace(modelOverride) ? _embedModel : modelOverride;
float[] vec = await EmbedAsync(baseUrl, model, text);
Upsert(kind, key, text, source, meta, vec);
}
public async Task ReembedAllAsync(string baseUrl, string newModel)
{
List<(string kind, string key, string text, string source, string meta)> rows = [];
lock (_lock)
{
EnsureOpen();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT kind, key, text, source, meta_json FROM memories";
cmd.CommandText = "SELECT COUNT(*) FROM memories";
return Convert.ToInt32(cmd.ExecuteScalar());
}
}
public async Task UpsertTextAsync(string baseUrl, string kind, string key, string text, string source = "user", JToken meta = null, string modelOverride = null, string persona = null)
{
string model = string.IsNullOrWhiteSpace(modelOverride) ? _embedModel : modelOverride;
float[] vec = await EmbedAsync(baseUrl, model, text);
Upsert(kind, key, text, source, meta, vec, persona);
}
public async Task ReembedAllAsync(string baseUrl, string newModel)
{
List<(string kind, string key, string text, string source, string meta, string persona)> rows = [];
lock (_lock)
{
EnsureOpen();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT kind, key, text, source, meta_json, persona FROM memories";
using SqliteDataReader reader = cmd.ExecuteReader();
while (reader.Read())
{
rows.Add((reader.GetString(0), reader.GetString(1), reader.GetString(2), reader.GetString(3), reader.IsDBNull(4) ? "" : reader.GetString(4)));
rows.Add((
reader.GetString(0),
reader.GetString(1),
reader.GetString(2),
reader.GetString(3),
reader.IsDBNull(4) ? "" : reader.GetString(4),
reader.FieldCount > 5 && !reader.IsDBNull(5) ? reader.GetString(5) : SharedPersona));
}
}
if (rows.Count == 0)
@@ -408,7 +619,7 @@ public sealed class AssistentMemory : IDisposable
{
try { meta = JToken.Parse(row.meta); } catch { /* ignore */ }
}
Upsert(row.kind, row.key, row.text, row.source, meta, vec);
Upsert(row.kind, row.key, row.text, row.source, meta, vec, row.persona);
}
catch (Exception ex)
{