mrleo1nid/swarm-assistent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix HF dataset import internal error and training SQLite readiness.

Browse Source 
Wrap AssistentImportHfDataset in try/catch, batch inserts in a transaction, clearer HF rows errors for gated datasets, EnsureTrainingReady with actionable message, and friendlier training UI errors instead of generic internal error.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 20:11:18 +03:00
co-authored by Cursor
parent b0f2736f65
commit 63538fe738
9 changed files with 323 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AssistentMemory.Training.cs
+75 -45
View File
@@ -68,7 +68,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
int take = Math.Clamp(limit, 1, 2000);
List<string> where = [];
if (!string.IsNullOrWhiteSpace(status) && !string.Equals(status, "all", StringComparison.OrdinalIgnoreCase))
@@ -167,57 +167,87 @@ public sealed partial class AssistentMemory
}
}
public int ImportTrainSamplesBatch(IEnumerable<JObject> samples)
{
lock (_lock)
{
EnsureTrainingReady();
int imported = 0;
using SqliteTransaction tx = _conn.BeginTransaction();
foreach (JObject sample in samples)
{
if (sample is null)
{
continue;
}
UpsertTrainSampleCore(sample, tx);
imported++;
}
tx.Commit();
return imported;
}
}
public JObject UpsertTrainSample(JObject sample)
{
lock (_lock)
{
EnsureOpen();
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
string id = sample["id"]?.ToString()?.Trim();
if (string.IsNullOrWhiteSpace(id))
{
id = $"ts_{now}_{Guid.NewGuid():N}"[..24];
}
string datasetId = sample["dataset_id"]?.ToString()?.Trim() ?? "default";
JArray messages = sample["messages"] as JArray ?? [];
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText =
"""
INSERT INTO train_samples(id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at)
VALUES($id, $ds, $src, $chat, $persona, $pack, $hf, $msg, $status, $c, $u)
ON CONFLICT(id) DO UPDATE SET
dataset_id = excluded.dataset_id,
source = excluded.source,
chat_id = excluded.chat_id,
persona = excluded.persona,
pack = excluded.pack,
hf_repo = excluded.hf_repo,
messages_json = excluded.messages_json,
status = excluded.status,
updated_at = excluded.updated_at
""";
cmd.Parameters.AddWithValue("$id", id);
cmd.Parameters.AddWithValue("$ds", datasetId);
cmd.Parameters.AddWithValue("$src", sample["source"]?.ToString() ?? "manual");
cmd.Parameters.AddWithValue("$chat", (object)sample["chat_id"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$persona", (object)sample["persona"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$pack", (object)sample["pack"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$hf", (object)sample["hf_repo"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$msg", messages.ToString(Newtonsoft.Json.Formatting.None));
cmd.Parameters.AddWithValue("$status", sample["status"]?.ToString() ?? "draft");
long created = sample["createdAt"]?.Value<long?>() ?? now;
cmd.Parameters.AddWithValue("$c", created);
cmd.Parameters.AddWithValue("$u", now);
cmd.ExecuteNonQuery();
return new JObject { ["id"] = id, ["updatedAt"] = now };
EnsureTrainingReady();
return UpsertTrainSampleCore(sample, null);
}
}
JObject UpsertTrainSampleCore(JObject sample, SqliteTransaction tx)
{
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
string id = sample["id"]?.ToString()?.Trim();
if (string.IsNullOrWhiteSpace(id))
{
id = $"ts_{now}_{Guid.NewGuid():N}"[..24];
}
string datasetId = sample["dataset_id"]?.ToString()?.Trim() ?? "default";
JArray messages = sample["messages"] as JArray ?? [];
using SqliteCommand cmd = _conn.CreateCommand();
if (tx is not null)
{
cmd.Transaction = tx;
}
cmd.CommandText =
"""
INSERT INTO train_samples(id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at)
VALUES($id, $ds, $src, $chat, $persona, $pack, $hf, $msg, $status, $c, $u)
ON CONFLICT(id) DO UPDATE SET
dataset_id = excluded.dataset_id,
source = excluded.source,
chat_id = excluded.chat_id,
persona = excluded.persona,
pack = excluded.pack,
hf_repo = excluded.hf_repo,
messages_json = excluded.messages_json,
status = excluded.status,
updated_at = excluded.updated_at
""";
cmd.Parameters.AddWithValue("$id", id);
cmd.Parameters.AddWithValue("$ds", datasetId);
cmd.Parameters.AddWithValue("$src", sample["source"]?.ToString() ?? "manual");
cmd.Parameters.AddWithValue("$chat", (object)sample["chat_id"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$persona", (object)sample["persona"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$pack", (object)sample["pack"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$hf", (object)sample["hf_repo"]?.ToString() ?? DBNull.Value);
cmd.Parameters.AddWithValue("$msg", messages.ToString(Newtonsoft.Json.Formatting.None));
cmd.Parameters.AddWithValue("$status", sample["status"]?.ToString() ?? "draft");
long created = sample["createdAt"]?.Value<long?>() ?? now;
cmd.Parameters.AddWithValue("$c", created);
cmd.Parameters.AddWithValue("$u", now);
cmd.ExecuteNonQuery();
return new JObject { ["id"] = id, ["updatedAt"] = now };
}
public bool DeleteTrainSample(string id)
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "DELETE FROM train_samples WHERE id = $id";
cmd.Parameters.AddWithValue("$id", id ?? "");
@@ -229,7 +259,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
if (string.IsNullOrWhiteSpace(status) || string.Equals(status, "all", StringComparison.OrdinalIgnoreCase))
{
@@ -288,7 +318,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at FROM train_jobs WHERE id = $id";
cmd.Parameters.AddWithValue("$id", id ?? "");
@@ -305,7 +335,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText =
"SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at "
@@ -341,7 +371,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at FROM train_jobs WHERE status IN ('pending','running') ORDER BY updated_at DESC LIMIT 1";
using SqliteDataReader r = cmd.ExecuteReader();