Add Knowledge Hub books FTS and remove training UI (0.16.0).
Index Assistent/books search.jsonl, expose knowledge hops/catalog in chat, and drop QLoRA/HF training stack. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
Cursor
@@ -0,0 +1,458 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using SwarmUI.Utils;
|
||||
|
||||
namespace Mrleo1nid.SwarmAssistent;
|
||||
|
||||
/// <summary>Reference books (search.jsonl) — FTS only, seeded by gpu-rent to Assistent/books/.</summary>
|
||||
public sealed partial class AssistentMemory
|
||||
{
|
||||
const string BooksMetaPrefix = "books_fp:";
|
||||
|
||||
void TryIndexBooks()
|
||||
{
|
||||
try
|
||||
{
|
||||
EnsureBooksIndex();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logs.Debug($"AssistentMemory books index: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
public string BooksRoot()
|
||||
=> Path.Combine(_dataRoot, "Assistent", "books");
|
||||
|
||||
void EnsureBooksSchema()
|
||||
{
|
||||
Exec(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS books_rows (
|
||||
id TEXT PRIMARY KEY,
|
||||
book_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
tags TEXT NOT NULL DEFAULT '',
|
||||
text TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
rating TEXT NOT NULL DEFAULT '',
|
||||
meta_json TEXT NOT NULL DEFAULT '{}',
|
||||
search_blob TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
""");
|
||||
Exec("CREATE INDEX IF NOT EXISTS idx_books_rows_book ON books_rows(book_id);");
|
||||
Exec(
|
||||
"""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS books_fts USING fts5(
|
||||
book_id,
|
||||
title,
|
||||
tags,
|
||||
text,
|
||||
body,
|
||||
search_blob,
|
||||
tokenize = 'unicode61 remove_diacritics 2'
|
||||
);
|
||||
""");
|
||||
Exec(
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS books_fts_ai AFTER INSERT ON books_rows BEGIN
|
||||
INSERT INTO books_fts(rowid, book_id, title, tags, text, body, search_blob)
|
||||
VALUES (new.rowid, new.book_id, new.title, new.tags, new.text, new.body, new.search_blob);
|
||||
END;
|
||||
""");
|
||||
Exec(
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS books_fts_ad AFTER DELETE ON books_rows BEGIN
|
||||
INSERT INTO books_fts(books_fts, rowid) VALUES('delete', old.rowid);
|
||||
END;
|
||||
""");
|
||||
Exec(
|
||||
"""
|
||||
CREATE TRIGGER IF NOT EXISTS books_fts_au AFTER UPDATE ON books_rows BEGIN
|
||||
INSERT INTO books_fts(books_fts, rowid) VALUES('delete', old.rowid);
|
||||
INSERT INTO books_fts(rowid, book_id, title, tags, text, body, search_blob)
|
||||
VALUES (new.rowid, new.book_id, new.title, new.tags, new.text, new.body, new.search_blob);
|
||||
END;
|
||||
""");
|
||||
}
|
||||
|
||||
static string ReadBookContentSha(string bookDir)
|
||||
{
|
||||
foreach (string name in new[] { ".gpu-rent-meta.json", "meta.json" })
|
||||
{
|
||||
string path = Path.Combine(bookDir, name);
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
JObject meta = JObject.Parse(File.ReadAllText(path, Encoding.UTF8));
|
||||
string sha = meta["content_sha"]?.ToString()?.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(sha))
|
||||
{
|
||||
return sha;
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
string jsonl = Path.Combine(bookDir, "search.jsonl");
|
||||
if (File.Exists(jsonl))
|
||||
{
|
||||
return FileFingerprint(jsonl);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Reindex changed books from disk. Returns total row count.</summary>
|
||||
public int EnsureBooksIndex()
|
||||
{
|
||||
string root = BooksRoot();
|
||||
if (!Directory.Exists(root))
|
||||
{
|
||||
return BookRowCount();
|
||||
}
|
||||
int total = 0;
|
||||
foreach (string bookDir in Directory.GetDirectories(root).OrderBy(d => d, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
string bookId = Path.GetFileName(bookDir);
|
||||
if (string.IsNullOrWhiteSpace(bookId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
string jsonl = Path.Combine(bookDir, "search.jsonl");
|
||||
if (!File.Exists(jsonl))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
string fp = ReadBookContentSha(bookDir) ?? FileFingerprint(jsonl);
|
||||
lock (_lock)
|
||||
{
|
||||
EnsureOpen();
|
||||
EnsureBooksSchema();
|
||||
string metaKey = BooksMetaPrefix + bookId;
|
||||
if (string.Equals(GetMeta(metaKey), fp, StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
int n = IndexBookJsonl(bookId, jsonl, fp);
|
||||
total += n;
|
||||
}
|
||||
if (total > 0)
|
||||
{
|
||||
Logs.Info($"AssistentMemory: indexed {total} book rows under {root}");
|
||||
}
|
||||
return BookRowCount();
|
||||
}
|
||||
|
||||
int IndexBookJsonl(string bookId, string jsonlPath, string fingerprint)
|
||||
{
|
||||
List<(string id, string title, string tags, string text, string body, string rating, string metaJson, string blob)> rows = [];
|
||||
foreach (string line in File.ReadLines(jsonlPath, Encoding.UTF8))
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
JObject o;
|
||||
try
|
||||
{
|
||||
o = JObject.Parse(line);
|
||||
}
|
||||
catch
|
||||
{
|
||||
continue;
|
||||
}
|
||||
string id = o["id"]?.ToString()?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
id = $"{bookId}:{rows.Count + 1}";
|
||||
}
|
||||
string title = o["title"]?.ToString() ?? "";
|
||||
string tagsJoined = "";
|
||||
if (o["tags"] is JArray tagArr)
|
||||
{
|
||||
tagsJoined = string.Join(", ", tagArr.Select(t => t?.ToString()?.Trim()).Where(t => !string.IsNullOrWhiteSpace(t)));
|
||||
}
|
||||
string text = o["text"]?.ToString() ?? "";
|
||||
string body = o["body"]?.ToString() ?? text;
|
||||
if (string.IsNullOrWhiteSpace(text) && string.IsNullOrWhiteSpace(body))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
string rating = o["rating"]?.ToString() ?? "";
|
||||
string metaJson = (o["meta"] as JObject)?.ToString(Newtonsoft.Json.Formatting.None) ?? "{}";
|
||||
string blob = $"{title}\n{tagsJoined}\n{text}\n{body}\n{rating}";
|
||||
rows.Add((id, title, tagsJoined, text, body, rating, metaJson, blob));
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
EnsureOpen();
|
||||
EnsureBooksSchema();
|
||||
using SqliteTransaction tx = _conn.BeginTransaction();
|
||||
using (SqliteCommand del = _conn.CreateCommand())
|
||||
{
|
||||
del.Transaction = tx;
|
||||
del.CommandText = "DELETE FROM books_rows WHERE book_id = $b";
|
||||
del.Parameters.AddWithValue("$b", bookId);
|
||||
del.ExecuteNonQuery();
|
||||
}
|
||||
using (SqliteCommand ins = _conn.CreateCommand())
|
||||
{
|
||||
ins.Transaction = tx;
|
||||
ins.CommandText =
|
||||
"""
|
||||
INSERT OR REPLACE INTO books_rows(
|
||||
id, book_id, title, tags, text, body, rating, meta_json, search_blob)
|
||||
VALUES($id,$b,$t,$tg,$tx,$bd,$r,$mj,$bl)
|
||||
""";
|
||||
var pid = ins.Parameters.Add("$id", SqliteType.Text);
|
||||
var pb = ins.Parameters.Add("$b", SqliteType.Text);
|
||||
var pt = ins.Parameters.Add("$t", SqliteType.Text);
|
||||
var ptg = ins.Parameters.Add("$tg", SqliteType.Text);
|
||||
var ptx = ins.Parameters.Add("$tx", SqliteType.Text);
|
||||
var pbd = ins.Parameters.Add("$bd", SqliteType.Text);
|
||||
var pr = ins.Parameters.Add("$r", SqliteType.Text);
|
||||
var pmj = ins.Parameters.Add("$mj", SqliteType.Text);
|
||||
var pbl = ins.Parameters.Add("$bl", SqliteType.Text);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
pid.Value = row.id;
|
||||
pb.Value = bookId;
|
||||
pt.Value = row.title;
|
||||
ptg.Value = row.tags;
|
||||
ptx.Value = row.text;
|
||||
pbd.Value = row.body;
|
||||
pr.Value = row.rating;
|
||||
pmj.Value = row.metaJson;
|
||||
pbl.Value = row.blob;
|
||||
ins.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
tx.Commit();
|
||||
SetMeta(BooksMetaPrefix + bookId, fingerprint);
|
||||
Logs.Info($"AssistentMemory: indexed book {bookId} ({rows.Count} rows)");
|
||||
return rows.Count;
|
||||
}
|
||||
}
|
||||
|
||||
public int BookRowCount()
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
EnsureOpen();
|
||||
if (!TableExists("books_rows"))
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
using SqliteCommand c = _conn.CreateCommand();
|
||||
c.CommandText = "SELECT COUNT(*) FROM books_rows";
|
||||
return Convert.ToInt32(c.ExecuteScalar());
|
||||
}
|
||||
}
|
||||
|
||||
public JArray ListBooksOnDisk()
|
||||
{
|
||||
JArray list = [];
|
||||
string root = BooksRoot();
|
||||
if (!Directory.Exists(root))
|
||||
{
|
||||
return list;
|
||||
}
|
||||
foreach (string bookDir in Directory.GetDirectories(root).OrderBy(d => d, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
string id = Path.GetFileName(bookDir);
|
||||
if (string.IsNullOrWhiteSpace(id))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
JObject spec = new() { ["id"] = id };
|
||||
string yaml = Path.Combine(bookDir, "book.yaml");
|
||||
if (File.Exists(yaml))
|
||||
{
|
||||
foreach (string rawLine in File.ReadAllLines(yaml, Encoding.UTF8))
|
||||
{
|
||||
string line = rawLine.Trim();
|
||||
if (string.IsNullOrWhiteSpace(line) || line.StartsWith('#'))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
int colon = line.IndexOf(':');
|
||||
if (colon <= 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
string key = line[..colon].Trim();
|
||||
string val = line[(colon + 1)..].Trim().Trim('"', '\'');
|
||||
if (key is "title" or "description" or "content_kind" or "language")
|
||||
{
|
||||
spec[key] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
string jsonl = Path.Combine(bookDir, "search.jsonl");
|
||||
spec["indexed"] = File.Exists(jsonl);
|
||||
spec["content_sha"] = ReadBookContentSha(bookDir) ?? "";
|
||||
list.Add(spec);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
/// <summary>FTS lookup over attached books (filter by book_id list when provided).</summary>
|
||||
public JArray LookupBooks(string query, int limit = 8, IEnumerable<string> bookIds = null, string rating = null)
|
||||
{
|
||||
query = (query ?? "").Trim();
|
||||
if (query.Length < 1)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
TryIndexBooks();
|
||||
int cap = Math.Clamp(limit, 1, 30);
|
||||
HashSet<string> bookFilter = null;
|
||||
if (bookIds is not null)
|
||||
{
|
||||
bookFilter = bookIds
|
||||
.Select(b => (b ?? "").Trim())
|
||||
.Where(b => !string.IsNullOrWhiteSpace(b))
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
if (bookFilter.Count == 0)
|
||||
{
|
||||
return [];
|
||||
}
|
||||
}
|
||||
string ratingFilter = string.IsNullOrWhiteSpace(rating) ? null : rating.Trim().ToLowerInvariant();
|
||||
List<JObject> hits = [];
|
||||
HashSet<string> seen = [];
|
||||
|
||||
void Add(SqliteDataReader reader)
|
||||
{
|
||||
string id = reader.GetString(0);
|
||||
if (!seen.Add(id))
|
||||
{
|
||||
return;
|
||||
}
|
||||
string bookId = reader.IsDBNull(1) ? "" : reader.GetString(1);
|
||||
if (bookFilter is not null && !bookFilter.Contains(bookId))
|
||||
{
|
||||
seen.Remove(id);
|
||||
return;
|
||||
}
|
||||
string title = reader.IsDBNull(2) ? "" : reader.GetString(2);
|
||||
string tags = reader.IsDBNull(3) ? "" : reader.GetString(3);
|
||||
string text = reader.IsDBNull(4) ? "" : reader.GetString(4);
|
||||
string body = reader.IsDBNull(5) ? "" : reader.GetString(5);
|
||||
string rowRating = reader.IsDBNull(6) ? "" : reader.GetString(6);
|
||||
JToken metaTok = new JObject();
|
||||
if (!reader.IsDBNull(7))
|
||||
{
|
||||
try { metaTok = JToken.Parse(reader.GetString(7)); } catch { metaTok = new JObject(); }
|
||||
}
|
||||
hits.Add(new JObject
|
||||
{
|
||||
["id"] = id,
|
||||
["book"] = bookId,
|
||||
["source"] = "book",
|
||||
["title"] = title,
|
||||
["tags"] = tags,
|
||||
["text"] = text.Length > 600 ? text[..600] + "…" : text,
|
||||
["body"] = body.Length > 1200 ? body[..1200] + "…" : body,
|
||||
["rating"] = rowRating,
|
||||
["meta"] = metaTok,
|
||||
["note"] = "BOOK reference — remix style/ideas, do not paste long verbatim",
|
||||
});
|
||||
}
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
EnsureOpen();
|
||||
if (!TableExists("books_rows"))
|
||||
{
|
||||
return [];
|
||||
}
|
||||
|
||||
string match = BuildFtsMatch(query);
|
||||
if (!string.IsNullOrWhiteSpace(match) && TableExists("books_fts"))
|
||||
{
|
||||
try
|
||||
{
|
||||
using SqliteCommand cmd = _conn.CreateCommand();
|
||||
string sql =
|
||||
"""
|
||||
SELECT b.id, b.book_id, b.title, b.tags, b.text, b.body, b.rating, b.meta_json
|
||||
FROM books_rows b
|
||||
WHERE b.rowid IN (SELECT rowid FROM books_fts WHERE books_fts MATCH $q)
|
||||
""";
|
||||
if (ratingFilter is not null)
|
||||
{
|
||||
sql += " AND lower(b.rating) = $r";
|
||||
}
|
||||
sql += " LIMIT $lim";
|
||||
cmd.CommandText = sql;
|
||||
cmd.Parameters.AddWithValue("$q", match);
|
||||
if (ratingFilter is not null)
|
||||
{
|
||||
cmd.Parameters.AddWithValue("$r", ratingFilter);
|
||||
}
|
||||
cmd.Parameters.AddWithValue("$lim", cap);
|
||||
using SqliteDataReader reader = cmd.ExecuteReader();
|
||||
while (reader.Read() && hits.Count < cap)
|
||||
{
|
||||
Add(reader);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logs.Debug($"AssistentMemory books FTS: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
if (hits.Count < cap)
|
||||
{
|
||||
try
|
||||
{
|
||||
using SqliteCommand cmd = _conn.CreateCommand();
|
||||
string sql =
|
||||
"""
|
||||
SELECT id, book_id, title, tags, text, body, rating, meta_json
|
||||
FROM books_rows
|
||||
WHERE (title LIKE $p ESCAPE '\' OR tags LIKE $p ESCAPE '\' OR text LIKE $p ESCAPE '\' OR body LIKE $p ESCAPE '\' OR search_blob LIKE $p ESCAPE '\')
|
||||
""";
|
||||
if (ratingFilter is not null)
|
||||
{
|
||||
sql += " AND lower(rating) = $r";
|
||||
}
|
||||
sql += " LIMIT $lim";
|
||||
cmd.CommandText = sql;
|
||||
cmd.Parameters.AddWithValue("$p", "%" + EscapeLike(query) + "%");
|
||||
if (ratingFilter is not null)
|
||||
{
|
||||
cmd.Parameters.AddWithValue("$r", ratingFilter);
|
||||
}
|
||||
cmd.Parameters.AddWithValue("$lim", cap);
|
||||
using SqliteDataReader reader = cmd.ExecuteReader();
|
||||
while (reader.Read() && hits.Count < cap)
|
||||
{
|
||||
Add(reader);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logs.Debug($"AssistentMemory books LIKE: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new JArray(hits.Take(cap));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user