Ship Assistent 0.13.0: real QLoRA pipeline and GGUF Ollama register.
Replace fake train loop with TRL SFTTrainer, HF column mapping with fiction preset, safetensors to GGUF conversion, and ollama create using ollama_base plus ADAPTER. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+149
-4
@@ -188,8 +188,14 @@ public partial class SwarmAssistentExtension
|
||||
return CacheHfCheck(cacheKey, result);
|
||||
}
|
||||
JObject rowsData = JObject.Parse(rowsBody);
|
||||
JObject features = rowsData["features"] as JObject;
|
||||
JObject features = FeaturesToObject(rowsData["features"]);
|
||||
(string gate, string reason, JObject schema) = ClassifyHfFeatures(features);
|
||||
if (gate == "mapping" && TryFictionTagsTextPreset(features, out JObject presetSchema))
|
||||
{
|
||||
gate = "ok";
|
||||
reason = "Fiction preset: title/tags → user, text → assistant";
|
||||
schema = presetSchema;
|
||||
}
|
||||
result["gate"] = gate;
|
||||
result["reason"] = reason;
|
||||
result["schema"] = schema;
|
||||
@@ -197,7 +203,7 @@ public partial class SwarmAssistentExtension
|
||||
result["split"] = split;
|
||||
result["features"] = features;
|
||||
result["sample_rows"] = rowsData["rows"];
|
||||
result["runner_only"] = HasHugeSizeTag(datasetId);
|
||||
result["runner_only"] = await HasHugeSizeTagAsync(session, datasetId);
|
||||
return CacheHfCheck(cacheKey, result);
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -207,7 +213,89 @@ public partial class SwarmAssistentExtension
|
||||
}
|
||||
}
|
||||
|
||||
static bool HasHugeSizeTag(string datasetId) => false;
|
||||
static bool TryFictionTagsTextPreset(JObject features, out JObject schema)
|
||||
{
|
||||
schema = null;
|
||||
if (features is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
HashSet<string> names = new(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (JProperty p in features.Properties())
|
||||
{
|
||||
names.Add(p.Name);
|
||||
}
|
||||
if (!names.Contains("text") || (!names.Contains("tags") && !names.Contains("title")))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
schema = new JObject
|
||||
{
|
||||
["kind"] = "fiction_tags_text",
|
||||
["assistant_col"] = "text",
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
async Task<bool> HasHugeSizeTagAsync(Session session, string datasetId)
|
||||
{
|
||||
try
|
||||
{
|
||||
using HttpRequestMessage req = HfRequest($"{HfHubApi}/{Uri.EscapeDataString(datasetId)}", session);
|
||||
using HttpResponseMessage resp = await HttpClient.SendAsync(req);
|
||||
if (!resp.IsSuccessStatusCode)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
JObject meta = JObject.Parse(await resp.Content.ReadAsStringAsync());
|
||||
foreach (JToken t in meta["tags"] as JArray ?? [])
|
||||
{
|
||||
string tag = t?.ToString() ?? "";
|
||||
if (tag.Contains("100K<", StringComparison.OrdinalIgnoreCase)
|
||||
|| tag.Contains("1M<", StringComparison.OrdinalIgnoreCase)
|
||||
|| tag.Contains("10M<", StringComparison.OrdinalIgnoreCase)
|
||||
|| tag.Contains("100M<", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logs.Debug($"HasHugeSizeTag {datasetId}: {ex.Message}");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static JObject ResolveHfMapping(JObject check, JObject mapping)
|
||||
{
|
||||
JObject schema = check?["schema"] as JObject;
|
||||
string kind = schema?["kind"]?.ToString();
|
||||
if (string.Equals(kind, "fiction_tags_text", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return new JObject { ["kind"] = "fiction_tags_text", ["preset"] = "fiction_tags_text" };
|
||||
}
|
||||
if (mapping is not null && mapping.Count > 0)
|
||||
{
|
||||
return mapping;
|
||||
}
|
||||
if (string.Equals(mapping?["preset"]?.ToString(), "fiction_tags_text", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return new JObject { ["kind"] = "fiction_tags_text" };
|
||||
}
|
||||
return mapping;
|
||||
}
|
||||
|
||||
static bool MappingRequired(JObject check, JObject mapping)
|
||||
{
|
||||
string gate = check?["gate"]?.ToString();
|
||||
if (gate != "mapping")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
JObject resolved = ResolveHfMapping(check, mapping);
|
||||
return resolved is null || !resolved.Properties().Any();
|
||||
}
|
||||
|
||||
JObject CacheHfCheck(string cacheKey, JObject result)
|
||||
{
|
||||
@@ -225,6 +313,33 @@ public partial class SwarmAssistentExtension
|
||||
return result;
|
||||
}
|
||||
|
||||
static JObject FeaturesToObject(JToken tok)
|
||||
{
|
||||
if (tok is JObject obj)
|
||||
{
|
||||
return obj;
|
||||
}
|
||||
if (tok is JArray arr)
|
||||
{
|
||||
JObject map = new();
|
||||
foreach (JToken t in arr)
|
||||
{
|
||||
if (t is not JObject row)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
string name = row["name"]?.ToString();
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
map[name] = row["type"] ?? row;
|
||||
}
|
||||
return map.Count > 0 ? map : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static (string gate, string reason, JObject schema) ClassifyHfFeatures(JObject features)
|
||||
{
|
||||
if (features is null || !features.Properties().Any())
|
||||
@@ -325,10 +440,11 @@ public partial class SwarmAssistentExtension
|
||||
{
|
||||
return new JObject { ["error"] = check["reason"]?.ToString() ?? "rejected" };
|
||||
}
|
||||
if (gate == "mapping" && (mapping is null || mapping.Count == 0))
|
||||
if (MappingRequired(check, mapping))
|
||||
{
|
||||
return new JObject { ["error"] = "Нужен маппинг колонок", ["check"] = check };
|
||||
}
|
||||
mapping = ResolveHfMapping(check, mapping);
|
||||
int take = Math.Clamp(limit, 1, 5000);
|
||||
JArray rows = [];
|
||||
string config = check["config"]?.ToString() ?? "default";
|
||||
@@ -445,6 +561,35 @@ public partial class SwarmAssistentExtension
|
||||
new JObject { ["role"] = "assistant", ["content"] = row["answer"]?.ToString() ?? "" },
|
||||
};
|
||||
}
|
||||
if (kind == "fiction_tags_text")
|
||||
{
|
||||
string text = row["text"]?.ToString() ?? "";
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
List<string> userParts = [];
|
||||
string title = row["title"]?.ToString()?.Trim();
|
||||
string tags = row["tags"]?.ToString()?.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(title))
|
||||
{
|
||||
userParts.Add($"Title: {title}");
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(tags))
|
||||
{
|
||||
userParts.Add($"Tags: {tags}");
|
||||
}
|
||||
string user = userParts.Count > 0 ? string.Join("\n", userParts) : tags ?? title ?? "";
|
||||
if (string.IsNullOrWhiteSpace(user))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
return new JArray
|
||||
{
|
||||
new JObject { ["role"] = "user", ["content"] = user },
|
||||
new JObject { ["role"] = "assistant", ["content"] = text },
|
||||
};
|
||||
}
|
||||
if (kind == "custom" && mapping is not null)
|
||||
{
|
||||
string userCol = mapping["user_col"]?.ToString();
|
||||
|
||||
Reference in New Issue
Block a user