Fix HF dataset import internal error and training SQLite readiness.
Wrap AssistentImportHfDataset in try/catch, batch inserts in a transaction, clearer HF rows errors for gated datasets, EnsureTrainingReady with actionable message, and friendlier training UI errors instead of generic internal error. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Leonid Pershin
Cursor
+121
-72
@@ -426,94 +426,143 @@ public partial class SwarmAssistentExtension
|
||||
|
||||
public async Task<JObject> AssistentImportHfDataset(Session session, JObject raw)
|
||||
{
|
||||
string dataset = raw?["dataset"]?.ToString();
|
||||
int limit = raw?["limit"]?.Value<int?>() ?? 200;
|
||||
JObject mapping = raw?["mapping"] as JObject;
|
||||
string id = NormalizeHfDatasetId(dataset);
|
||||
if (id is null)
|
||||
try
|
||||
{
|
||||
return new JObject { ["error"] = "invalid dataset id" };
|
||||
}
|
||||
JObject check = await CheckHfDatasetInternal(session, id, useCache: true);
|
||||
string gate = check["gate"]?.ToString();
|
||||
if (gate == "rejected")
|
||||
{
|
||||
return new JObject { ["error"] = check["reason"]?.ToString() ?? "rejected" };
|
||||
}
|
||||
if (MappingRequired(check, mapping))
|
||||
{
|
||||
return new JObject { ["error"] = "Нужен маппинг колонок", ["check"] = check };
|
||||
}
|
||||
mapping = ResolveHfMapping(check, mapping);
|
||||
int take = Math.Clamp(limit, 1, 5000);
|
||||
JArray rows = [];
|
||||
string config = check["config"]?.ToString() ?? "default";
|
||||
string split = check["split"]?.ToString() ?? "train";
|
||||
int offset = 0;
|
||||
while (rows.Count < take)
|
||||
{
|
||||
int chunk = Math.Min(100, take - rows.Count);
|
||||
using HttpRequestMessage rowsReq = HfRequest($"{HfDatasetsServer}/rows?dataset={Uri.EscapeDataString(id)}&config={Uri.EscapeDataString(config)}&split={Uri.EscapeDataString(split)}&offset={offset}&length={chunk}", session);
|
||||
using HttpResponseMessage rowsResp = await HttpClient.SendAsync(rowsReq);
|
||||
string rowsBody = await rowsResp.Content.ReadAsStringAsync();
|
||||
if (!rowsResp.IsSuccessStatusCode)
|
||||
string dataset = raw?["dataset"]?.ToString();
|
||||
int limit = raw?["limit"]?.Value<int?>() ?? 200;
|
||||
JObject mapping = raw?["mapping"] as JObject;
|
||||
string id = NormalizeHfDatasetId(dataset);
|
||||
if (id is null)
|
||||
{
|
||||
break;
|
||||
return new JObject { ["error"] = "invalid dataset id" };
|
||||
}
|
||||
JObject parsed = JObject.Parse(rowsBody);
|
||||
JArray batch = parsed["rows"] as JArray ?? [];
|
||||
if (batch.Count == 0)
|
||||
JObject check = await CheckHfDatasetInternal(session, id, useCache: true);
|
||||
string gate = check["gate"]?.ToString();
|
||||
if (gate == "rejected")
|
||||
{
|
||||
break;
|
||||
return new JObject { ["error"] = check["reason"]?.ToString() ?? "rejected" };
|
||||
}
|
||||
foreach (JToken t in batch)
|
||||
if (MappingRequired(check, mapping))
|
||||
{
|
||||
rows.Add(t);
|
||||
return new JObject { ["error"] = "Нужен маппинг колонок", ["check"] = check };
|
||||
}
|
||||
offset += batch.Count;
|
||||
if (batch.Count < chunk)
|
||||
mapping = ResolveHfMapping(check, mapping);
|
||||
int take = Math.Clamp(limit, 1, 5000);
|
||||
JArray rows = [];
|
||||
string config = check["config"]?.ToString() ?? "default";
|
||||
string split = check["split"]?.ToString() ?? "train";
|
||||
int offset = 0;
|
||||
while (rows.Count < take)
|
||||
{
|
||||
break;
|
||||
int chunk = Math.Min(100, take - rows.Count);
|
||||
using HttpRequestMessage rowsReq = HfRequest($"{HfDatasetsServer}/rows?dataset={Uri.EscapeDataString(id)}&config={Uri.EscapeDataString(config)}&split={Uri.EscapeDataString(split)}&offset={offset}&length={chunk}", session);
|
||||
using HttpResponseMessage rowsResp = await HttpClient.SendAsync(rowsReq);
|
||||
string rowsBody = await rowsResp.Content.ReadAsStringAsync();
|
||||
if (!rowsResp.IsSuccessStatusCode)
|
||||
{
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
return new JObject
|
||||
{
|
||||
["error"] = $"HF rows HTTP {(int)rowsResp.StatusCode}: {Clip(rowsBody, 240)}. "
|
||||
+ "Для gated/NSFW добавь huggingface_api в User Settings.",
|
||||
};
|
||||
}
|
||||
break;
|
||||
}
|
||||
JObject parsed = JObject.Parse(rowsBody);
|
||||
JArray batch = parsed["rows"] as JArray ?? [];
|
||||
if (batch.Count == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
foreach (JToken t in batch)
|
||||
{
|
||||
rows.Add(t);
|
||||
}
|
||||
offset += batch.Count;
|
||||
if (batch.Count < chunk)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
rows = check["sample_rows"] as JArray ?? [];
|
||||
}
|
||||
int imported = 0;
|
||||
foreach (JToken rowTok in rows.Take(take))
|
||||
{
|
||||
if (rowTok is not JObject row)
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
continue;
|
||||
rows = check["sample_rows"] as JArray ?? [];
|
||||
}
|
||||
JObject rowData = row["row"] as JObject ?? row;
|
||||
JArray messages = ConvertHfRowToMessages(rowData, check["schema"] as JObject, mapping);
|
||||
if (messages is null || messages.Count == 0)
|
||||
List<JObject> toSave = [];
|
||||
foreach (JToken rowTok in rows.Take(take))
|
||||
{
|
||||
continue;
|
||||
if (rowTok is not JObject row)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
JObject rowData = row["row"] as JObject ?? row;
|
||||
JArray messages = ConvertHfRowToMessages(rowData, check["schema"] as JObject, mapping);
|
||||
if (messages is null || messages.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
toSave.Add(new JObject
|
||||
{
|
||||
["source"] = "hf",
|
||||
["hf_repo"] = id,
|
||||
["messages"] = messages,
|
||||
["status"] = "draft",
|
||||
});
|
||||
}
|
||||
Memory.UpsertTrainSample(new JObject
|
||||
int imported = toSave.Count > 0 ? Memory.ImportTrainSamplesBatch(toSave) : 0;
|
||||
if (imported == 0 && check["runner_only"]?.Value<bool?>() == true)
|
||||
{
|
||||
["source"] = "hf",
|
||||
["hf_repo"] = id,
|
||||
["messages"] = messages,
|
||||
return new JObject { ["success"] = true, ["imported"] = 0, ["runner_only"] = true, ["id"] = id, ["note"] = "Большой набор — используй HF id в QLoRA-раннере" };
|
||||
}
|
||||
if (imported == 0)
|
||||
{
|
||||
return new JObject
|
||||
{
|
||||
["error"] = rows.Count == 0
|
||||
? "HF не отдал строки — проверь token (gated/NSFW) и маппинг колонок"
|
||||
: "0 строк после маппинга — проверь колонки user/assistant",
|
||||
["rows_fetched"] = rows.Count,
|
||||
["id"] = id,
|
||||
};
|
||||
}
|
||||
return new JObject
|
||||
{
|
||||
["success"] = true,
|
||||
["imported"] = imported,
|
||||
["id"] = id,
|
||||
["rows_fetched"] = rows.Count,
|
||||
["status"] = "draft",
|
||||
});
|
||||
imported++;
|
||||
};
|
||||
}
|
||||
if (imported == 0 && check["runner_only"]?.Value<bool?>() == true)
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new JObject { ["success"] = true, ["imported"] = 0, ["runner_only"] = true, ["id"] = id, ["note"] = "Большой набор — используй HF id в QLoRA-раннере" };
|
||||
Logs.Error($"AssistentImportHfDataset: {ex}");
|
||||
return new JObject { ["error"] = ex.Message };
|
||||
}
|
||||
return new JObject
|
||||
}
|
||||
|
||||
static string HfCellString(JToken tok)
|
||||
{
|
||||
if (tok is null || tok.Type == JTokenType.Null)
|
||||
{
|
||||
["success"] = true,
|
||||
["imported"] = imported,
|
||||
["id"] = id,
|
||||
["rows_fetched"] = rows.Count,
|
||||
["status"] = "draft",
|
||||
};
|
||||
return "";
|
||||
}
|
||||
if (tok is JArray arr)
|
||||
{
|
||||
List<string> parts = [];
|
||||
foreach (JToken t in arr)
|
||||
{
|
||||
string s = t?.Type == JTokenType.String ? t.ToString() : t?.ToString(Newtonsoft.Json.Formatting.None);
|
||||
if (!string.IsNullOrWhiteSpace(s))
|
||||
{
|
||||
parts.Add(s.Trim());
|
||||
}
|
||||
}
|
||||
return string.Join(", ", parts);
|
||||
}
|
||||
return tok.ToString().Trim();
|
||||
}
|
||||
|
||||
static JArray ConvertHfRowToMessages(JObject row, JObject schema, JObject mapping)
|
||||
@@ -570,14 +619,14 @@ public partial class SwarmAssistentExtension
|
||||
}
|
||||
if (kind == "fiction_tags_text")
|
||||
{
|
||||
string text = row["text"]?.ToString() ?? "";
|
||||
string text = HfCellString(row["text"]);
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
List<string> userParts = [];
|
||||
string title = row["title"]?.ToString()?.Trim();
|
||||
string tags = row["tags"]?.ToString()?.Trim();
|
||||
string title = HfCellString(row["title"]);
|
||||
string tags = HfCellString(row["tags"]);
|
||||
if (!string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
userParts.Add($"Title: {title}");
|
||||
|
||||
Reference in New Issue
Block a user