mrleo1nid/swarm-assistent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix HF dataset import internal error and training SQLite readiness.

Browse Source 
Wrap AssistentImportHfDataset in try/catch, batch inserts in a transaction, clearer HF rows errors for gated datasets, EnsureTrainingReady with actionable message, and friendlier training UI errors instead of generic internal error.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
2026-08-23 20:11:18 +03:00
co-authored by Cursor
parent b0f2736f65
commit 63538fe738
9 changed files with 323 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Assets/assistent.bundle.js
+25 -12
View File
@@ -10708,7 +10708,7 @@ ${HELP_TEXT}`);
setAgentHeardStats(state.agentLinked);
} catch (e) {
const msg = String(e.message || e);
setTrainStatus(isSqliteError(msg) ? `${msg} \u2014 ${sqliteHint()}` : msg);
setTrainStatus(formatTrainError(msg));
console.warn("loadAgentHeardSettings", e);
}
}
@@ -10723,7 +10723,7 @@ ${HELP_TEXT}`);
state.agentSettings = data?.settings || settings;
setTrainStatus("\u041D\u0430\u0441\u0442\u0440\u043E\u0439\u043A\u0438 \xAB\u0443\u0441\u043B\u044B\u0448\u0430\u043D\u043D\u043E\u0433\u043E\xBB \u0441\u043E\u0445\u0440\u0430\u043D\u0435\u043D\u044B");
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
async function syncAllToAgent() {
@@ -10736,7 +10736,7 @@ ${HELP_TEXT}`);
setTrainStatus(`\u041F\u043E\u0434\u043A\u043B\u044E\u0447\u0435\u043D\u043E: +${data?.linked_now ?? 0}, \u0432\u0441\u0435\u0433\u043E ${data?.total_linked ?? "\u2014"}`);
await refreshSamples();
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
function setTrainStatus(msg) {
@@ -10745,7 +10745,20 @@ ${HELP_TEXT}`);
}
function isSqliteError(msg) {
const s = String(msg || "").toLowerCase();
return s.includes("sqlite") || s.includes("sqlconnection");
return s.includes("sqlite") || s.includes("sqlconnection") || s.includes("train_samples") || s.includes("training database unavailable");
}
function isGenericServerError(msg) {
return /internal error occurred/i.test(String(msg || ""));
}
function formatTrainError(msg) {
const s = String(msg || "");
if (isSqliteError(s)) {
return `${s} \u2014 ${sqliteHint()}`;
}
if (isGenericServerError(s)) {
return `${s} \u2014 \u0447\u0430\u0441\u0442\u043E SQLite/\u0438\u043C\u043F\u043E\u0440\u0442 HF: gpu-rent seed-extensions, restart SwarmUI, HF token \u0434\u043B\u044F gated.`;
}
return s;
}
function sqliteHint() {
return "\u0411\u0430\u0437\u0430 Assistent (SQLite) \u043D\u0435\u0434\u043E\u0441\u0442\u0443\u043F\u043D\u0430 \u2014 gpu-rent seed-extensions + restart SwarmUI (\u22650.15.6).";
@@ -10816,7 +10829,7 @@ ${HELP_TEXT}`);
const data = await SA2.request("AssistentListTrainSamples", { status, persona, limit: 300 });
if (data?.error) {
const msg = data.error;
setTrainStatus(isSqliteError(msg) ? `${msg} \u2014 ${sqliteHint()}` : msg);
setTrainStatus(formatTrainError(msg));
const stats2 = $("sa_train_stats");
if (stats2) stats2.textContent = "\u0414\u0430\u0442\u0430\u0441\u0435\u0442 \u043D\u0435\u0434\u043E\u0441\u0442\u0443\u043F\u0435\u043D (SQLite)";
state.samples = [];
@@ -10845,7 +10858,7 @@ ${HELP_TEXT}`);
}
renderSamples();
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
function renderSamples() {
@@ -11073,7 +11086,7 @@ ${HELP_TEXT}`);
renderHfList();
setTrainStatus(`\u041D\u0430\u0439\u0434\u0435\u043D\u043E: ${state.hfResults.length}`);
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
async function checkHfLink() {
@@ -11148,7 +11161,7 @@ ${HELP_TEXT}`);
$("sa_train_samples")?.scrollIntoView({ behavior: "smooth", block: "nearest" });
} catch (e) {
const err = String(e.message || e);
const show = isSqliteError(err) ? `${err} \u2014 ${sqliteHint()}` : err;
const show = formatTrainError(err);
setTrainStatus(show);
if (hfSt) hfSt.textContent = show;
} finally {
@@ -11203,7 +11216,7 @@ ${HELP_TEXT}`);
setTrainStatus(`\u0413\u043E\u0442\u043E\u0432\u043E: ${data.name}`);
SA2.app?.refreshModels?.();
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
} finally {
btn?.removeAttribute("disabled");
}
@@ -11328,7 +11341,7 @@ ${HELP_TEXT}`);
setTrainingTab("train");
setTrainStatus("QLoRA \u0437\u0430\u043F\u0443\u0449\u0435\u043D\u0430 \u2014 \u043F\u0440\u043E\u0433\u0440\u0435\u0441\u0441 \u043D\u0438\u0436\u0435");
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
async function cancelQlora() {
@@ -11337,7 +11350,7 @@ ${HELP_TEXT}`);
setTrainingLock(false);
setTrainStatus("\u041E\u0442\u043C\u0435\u043D\u0435\u043D\u043E");
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
async function refreshTrainModels() {
@@ -11371,7 +11384,7 @@ ${HELP_TEXT}`);
});
setTrainStatus("\u0420\u0430\u043D\u043D\u0435\u0440 \u0441\u043E\u0445\u0440\u0430\u043D\u0451\u043D");
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
async function loadRunner() {
AssistentHuggingFace.cs
+55 -6
View File
@@ -425,6 +425,8 @@ public partial class SwarmAssistentExtension
}
public async Task<JObject> AssistentImportHfDataset(Session session, JObject raw)
{
try
{
string dataset = raw?["dataset"]?.ToString();
int limit = raw?["limit"]?.Value<int?>() ?? 200;
@@ -458,6 +460,14 @@ public partial class SwarmAssistentExtension
string rowsBody = await rowsResp.Content.ReadAsStringAsync();
if (!rowsResp.IsSuccessStatusCode)
{
if (rows.Count == 0)
{
return new JObject
{
["error"] = $"HF rows HTTP {(int)rowsResp.StatusCode}: {Clip(rowsBody, 240)}. "
+ "Для gated/NSFW добавь huggingface_api в User Settings.",
};
}
break;
}
JObject parsed = JObject.Parse(rowsBody);
@@ -480,7 +490,7 @@ public partial class SwarmAssistentExtension
{
rows = check["sample_rows"] as JArray ?? [];
}
int imported = 0;
List<JObject> toSave = [];
foreach (JToken rowTok in rows.Take(take))
{
if (rowTok is not JObject row)
@@ -493,19 +503,30 @@ public partial class SwarmAssistentExtension
{
continue;
}
Memory.UpsertTrainSample(new JObject
toSave.Add(new JObject
{
["source"] = "hf",
["hf_repo"] = id,
["messages"] = messages,
["status"] = "draft",
});
imported++;
}
int imported = toSave.Count > 0 ? Memory.ImportTrainSamplesBatch(toSave) : 0;
if (imported == 0 && check["runner_only"]?.Value<bool?>() == true)
{
return new JObject { ["success"] = true, ["imported"] = 0, ["runner_only"] = true, ["id"] = id, ["note"] = "Большой набор — используй HF id в QLoRA-раннере" };
}
if (imported == 0)
{
return new JObject
{
["error"] = rows.Count == 0
? "HF не отдал строки — проверь token (gated/NSFW) и маппинг колонок"
: "0 строк после маппинга — проверь колонки user/assistant",
["rows_fetched"] = rows.Count,
["id"] = id,
};
}
return new JObject
{
["success"] = true,
@@ -515,6 +536,34 @@ public partial class SwarmAssistentExtension
["status"] = "draft",
};
}
catch (Exception ex)
{
Logs.Error($"AssistentImportHfDataset: {ex}");
return new JObject { ["error"] = ex.Message };
}
}
static string HfCellString(JToken tok)
{
if (tok is null || tok.Type == JTokenType.Null)
{
return "";
}
if (tok is JArray arr)
{
List<string> parts = [];
foreach (JToken t in arr)
{
string s = t?.Type == JTokenType.String ? t.ToString() : t?.ToString(Newtonsoft.Json.Formatting.None);
if (!string.IsNullOrWhiteSpace(s))
{
parts.Add(s.Trim());
}
}
return string.Join(", ", parts);
}
return tok.ToString().Trim();
}
static JArray ConvertHfRowToMessages(JObject row, JObject schema, JObject mapping)
{
@@ -570,14 +619,14 @@ public partial class SwarmAssistentExtension
}
if (kind == "fiction_tags_text")
{
string text = row["text"]?.ToString() ?? "";
string text = HfCellString(row["text"]);
if (string.IsNullOrWhiteSpace(text))
{
return null;
}
List<string> userParts = [];
string title = row["title"]?.ToString()?.Trim();
string tags = row["tags"]?.ToString()?.Trim();
string title = HfCellString(row["title"]);
string tags = HfCellString(row["tags"]);
if (!string.IsNullOrWhiteSpace(title))
{
userParts.Add($"Title: {title}");
AssistentMemory.Heard.cs
+9 -2
View File
@@ -50,7 +50,7 @@ public sealed partial class AssistentMemory
}
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
if (!HasColumn("train_samples", "agent_linked"))
{
return;
@@ -68,7 +68,14 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
try
{
EnsureTrainingReady();
}
catch
{
return 0;
}
if (!HasColumn("train_samples", "agent_linked"))
{
return 0;
AssistentMemory.Training.cs
+38 -8
View File
@@ -68,7 +68,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
int take = Math.Clamp(limit, 1, 2000);
List<string> where = [];
if (!string.IsNullOrWhiteSpace(status) && !string.Equals(status, "all", StringComparison.OrdinalIgnoreCase))
@@ -167,11 +167,38 @@ public sealed partial class AssistentMemory
}
}
public int ImportTrainSamplesBatch(IEnumerable<JObject> samples)
{
lock (_lock)
{
EnsureTrainingReady();
int imported = 0;
using SqliteTransaction tx = _conn.BeginTransaction();
foreach (JObject sample in samples)
{
if (sample is null)
{
continue;
}
UpsertTrainSampleCore(sample, tx);
imported++;
}
tx.Commit();
return imported;
}
}
public JObject UpsertTrainSample(JObject sample)
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
return UpsertTrainSampleCore(sample, null);
}
}
JObject UpsertTrainSampleCore(JObject sample, SqliteTransaction tx)
{
long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
string id = sample["id"]?.ToString()?.Trim();
if (string.IsNullOrWhiteSpace(id))
@@ -181,6 +208,10 @@ public sealed partial class AssistentMemory
string datasetId = sample["dataset_id"]?.ToString()?.Trim() ?? "default";
JArray messages = sample["messages"] as JArray ?? [];
using SqliteCommand cmd = _conn.CreateCommand();
if (tx is not null)
{
cmd.Transaction = tx;
}
cmd.CommandText =
"""
INSERT INTO train_samples(id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at)
@@ -211,13 +242,12 @@ public sealed partial class AssistentMemory
cmd.ExecuteNonQuery();
return new JObject { ["id"] = id, ["updatedAt"] = now };
}
}
public bool DeleteTrainSample(string id)
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "DELETE FROM train_samples WHERE id = $id";
cmd.Parameters.AddWithValue("$id", id ?? "");
@@ -229,7 +259,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
if (string.IsNullOrWhiteSpace(status) || string.Equals(status, "all", StringComparison.OrdinalIgnoreCase))
{
@@ -288,7 +318,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at FROM train_jobs WHERE id = $id";
cmd.Parameters.AddWithValue("$id", id ?? "");
@@ -305,7 +335,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText =
"SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at "
@@ -341,7 +371,7 @@ public sealed partial class AssistentMemory
{
lock (_lock)
{
EnsureOpen();
EnsureTrainingReady();
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at FROM train_jobs WHERE status IN ('pending','running') ORDER BY updated_at DESC LIMIT 1";
using SqliteDataReader r = cmd.ExecuteReader();
AssistentMemory.cs
+28 -1
View File
@@ -149,7 +149,7 @@ public sealed partial class AssistentMemory : IDisposable
}
catch (Exception ex)
{
Logs.Debug($"AssistentMemory training schema: {ex.Message}");
Logs.Error($"AssistentMemory training schema failed: {ex.Message}");
}
try
{
@@ -166,6 +166,10 @@ public sealed partial class AssistentMemory : IDisposable
bool HasColumn(string table, string column)
{
if (!HasTable(table))
{
return false;
}
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = $"PRAGMA table_info({table})";
using SqliteDataReader reader = cmd.ExecuteReader();
@@ -179,6 +183,29 @@ public sealed partial class AssistentMemory : IDisposable
return false;
}
bool HasTable(string table)
{
using SqliteCommand cmd = _conn.CreateCommand();
cmd.CommandText = "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = $n LIMIT 1";
cmd.Parameters.AddWithValue("$n", table ?? "");
return cmd.ExecuteScalar() != null;
}
internal void EnsureTrainingReady()
{
EnsureOpen();
if (!HasTable("train_samples"))
{
EnsureTrainingSchema();
}
if (!HasTable("train_samples"))
{
throw new InvalidOperationException(
"Assistent training database unavailable (train_samples). "
+ "Run gpu-rent seed-extensions and restart SwarmUI (≥0.15.6).");
}
}
void MigratePersonaColumn()
{
if (HasColumn("memories", "persona"))
AssistentTraining.Agent.cs
+7
View File
@@ -25,6 +25,8 @@ public partial class SwarmAssistentExtension
public async Task<JObject> AssistentGetDatasetAgentSettings(Session session)
{
await Task.CompletedTask;
try
{
JObject settings = Config.LoadTrainingAgent();
return new JObject
{
@@ -34,6 +36,11 @@ public partial class SwarmAssistentExtension
["approved"] = Memory.CountTrainSamples("approved"),
};
}
catch (Exception ex)
{
return new JObject { ["error"] = ex.Message };
}
}
public async Task<JObject> AssistentSaveDatasetAgentSettings(Session session, JObject settings)
{
SwarmAssistentExtension.cs
+1 -1
View File
@@ -33,7 +33,7 @@ public partial class SwarmAssistentExtension : Extension
ExtensionAuthor = "mrleo1nid";
Description = "Collaborative Krea 2 assistant: Ollama chat, persona presets, vector memory, model cards, Generate loop.";
License = "MIT";
Version = "0.15.11";
Version = "0.15.12";
Tags = ["tabs", "ui", "llm", "ollama", "krea", "inpaint", "memory", "training", "heard", "qlora"];
}
scripts/_forbid014.json
+17
View File
@@ -0,0 +1,17 @@
[
"catalog_card",
"fromCards",
"AssistentGetCard",
"AssistentSaveCard",
"AssistentSearchCivitai",
"AssistentListWanted",
"AssistentEnqueueWanted",
"buildCivitaiCards",
"maybeWriteCardAfterDownload",
"sa_tab_cards",
"modelCards",
"extractCardJson",
"userImpliesGenerate",
"fromEmptyPatchRetry",
"synthesizePatchAfterEmptyFence"
]
src/training.js
+33 -15
View File
@@ -50,7 +50,7 @@ export function attachTraining(SA) {
setAgentHeardStats(state.agentLinked);
} catch (e) {
const msg = String(e.message || e);
setTrainStatus(isSqliteError(msg) ? `${msg} — ${sqliteHint()}` : msg);
setTrainStatus(formatTrainError(msg));
console.warn('loadAgentHeardSettings', e);
}
}
@@ -66,7 +66,7 @@ export function attachTraining(SA) {
state.agentSettings = data?.settings || settings;
setTrainStatus('Настройки «услышанного» сохранены');
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
@@ -80,7 +80,7 @@ export function attachTraining(SA) {
setTrainStatus(`Подключено: +${data?.linked_now ?? 0}, всего ${data?.total_linked ?? '—'}`);
await refreshSamples();
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
@@ -91,7 +91,25 @@ export function attachTraining(SA) {
function isSqliteError(msg) {
const s = String(msg || '').toLowerCase();
return s.includes('sqlite') || s.includes('sqlconnection');
return s.includes('sqlite')
|| s.includes('sqlconnection')
|| s.includes('train_samples')
|| s.includes('training database unavailable');
}
function isGenericServerError(msg) {
return /internal error occurred/i.test(String(msg || ''));
}
function formatTrainError(msg) {
const s = String(msg || '');
if (isSqliteError(s)) {
return `${s} — ${sqliteHint()}`;
}
if (isGenericServerError(s)) {
return `${s} — часто SQLite/импорт HF: gpu-rent seed-extensions, restart SwarmUI, HF token для gated.`;
}
return s;
}
function sqliteHint() {
@@ -167,7 +185,7 @@ export function attachTraining(SA) {
const data = await SA.request('AssistentListTrainSamples', { status, persona, limit: 300 });
if (data?.error) {
const msg = data.error;
setTrainStatus(isSqliteError(msg) ? `${msg} — ${sqliteHint()}` : msg);
setTrainStatus(formatTrainError(msg));
const stats = $('sa_train_stats');
if (stats) stats.textContent = 'Датасет недоступен (SQLite)';
state.samples = [];
@@ -196,7 +214,7 @@ export function attachTraining(SA) {
}
renderSamples();
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
@@ -441,7 +459,7 @@ export function attachTraining(SA) {
renderHfList();
setTrainStatus(`Найдено: ${state.hfResults.length}`);
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
@@ -522,7 +540,7 @@ export function attachTraining(SA) {
$('sa_train_samples')?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
} catch (e) {
const err = String(e.message || e);
const show = isSqliteError(err) ? `${err} — ${sqliteHint()}` : err;
const show = formatTrainError(err);
setTrainStatus(show);
if (hfSt) hfSt.textContent = show;
} finally {
@@ -578,7 +596,7 @@ export function attachTraining(SA) {
setTrainStatus(`Готово: ${data.name}`);
SA.app?.refreshModels?.();
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
} finally {
btn?.removeAttribute('disabled');
}
@@ -712,7 +730,7 @@ export function attachTraining(SA) {
setTrainingTab('train');
setTrainStatus('QLoRA запущена — прогресс ниже');
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
@@ -722,7 +740,7 @@ export function attachTraining(SA) {
setTrainingLock(false);
setTrainStatus('Отменено');
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
@@ -760,7 +778,7 @@ export function attachTraining(SA) {
});
setTrainStatus('Раннер сохранён');
} catch (e) {
setTrainStatus(String(e.message || e));
setTrainStatus(formatTrainError(e.message || e));
}
}
@@ -805,7 +823,7 @@ export function attachTraining(SA) {
const data = await SA.request('AssistentImportDataset', { format: 'auto', content: text });
setTrainStatus(`Импорт: ${data.imported}`);
await refreshSamples();
} catch (err) { setTrainStatus(String(err.message || err)); }
} catch (err) { setTrainStatus(formatTrainError(err.message || err)); }
e.target.value = '';
});
$('sa_btn_train_export')?.addEventListener('click', async () => {
@@ -842,7 +860,7 @@ export function attachTraining(SA) {
setAgentHeardStats(state.agentLinked);
setTrainStatus('Пример подключён к агенту');
await refreshSamples();
} catch (err) { setTrainStatus(String(err.message || err)); }
} catch (err) { setTrainStatus(formatTrainError(err.message || err)); }
} else if (e.target.closest('[data-unlink]')) {
try {
const data = await SA.request('AssistentUnlinkTrainSampleFromAgent', { id });
@@ -850,7 +868,7 @@ export function attachTraining(SA) {
setAgentHeardStats(state.agentLinked);
setTrainStatus('Пример отключён от агента');
await refreshSamples();
} catch (err) { setTrainStatus(String(err.message || err)); }
} catch (err) { setTrainStatus(formatTrainError(err.message || err)); }
} else if (e.target.closest('[data-del]')) {
if (window.confirm('Удалить пример?')) {
await SA.request('AssistentDeleteTrainSample', { id });