Ship Assistent 0.13.0: real QLoRA pipeline and GGUF Ollama register.

Replace fake train loop with TRL SFTTrainer, HF column mapping with fiction preset, safetensors to GGUF conversion, and ollama create using ollama_base plus ADAPTER. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-22 14:45:07 +03:00
parent 1a03c3178f
commit 474a674a35
12 changed files with 1002 additions and 133 deletions
@@ -35,30 +35,32 @@ public partial class SwarmAssistentExtension
        {
            return new JObject { ["error"] = "base_model and output_name required" };
        }
+        string hfDataset = null;
+        JObject hfCheck = null;
+        JObject hfMapping = raw["hf_mapping"] as JObject;
        if (!string.IsNullOrWhiteSpace(raw["hf_dataset"]?.ToString()))
        {
-            string dsId = NormalizeHfDatasetId(raw["hf_dataset"]?.ToString());
-            if (dsId is null)
+            hfDataset = NormalizeHfDatasetId(raw["hf_dataset"]?.ToString());
+            if (hfDataset is null)
            {
                return new JObject { ["error"] = "invalid hf_dataset id" };
            }
-            JObject check = await CheckHfDatasetInternal(session, dsId, useCache: true);
-            if (check["gate"]?.ToString() == "rejected")
+            hfCheck = await CheckHfDatasetInternal(session, hfDataset, useCache: true);
+            if (hfCheck["gate"]?.ToString() == "rejected")
            {
-                return new JObject { ["error"] = check["reason"]?.ToString() ?? "hf dataset rejected" };
+                return new JObject { ["error"] = hfCheck["reason"]?.ToString() ?? "hf dataset rejected" };
+            }
+            hfMapping = ResolveHfMapping(hfCheck, hfMapping);
+            if (MappingRequired(hfCheck, hfMapping))
+            {
+                return new JObject { ["error"] = "Нужен маппинг колонок для HF набора", ["check"] = hfCheck };
            }
-            raw["hf_dataset"] = dsId;
        }
        JObject runner = Config.LoadTrainingRunner();
-        string python = runner["python"]?.ToString()?.Trim();
-        if (string.IsNullOrWhiteSpace(python))
-        {
-            python = "python";
-        }
        string kind = runner["kind"]?.ToString()?.Trim();
        if (string.IsNullOrWhiteSpace(kind))
        {
-            return new JObject { ["error"] = "QLoRA-раннер не настроен (Настройки → Модели)" };
+            kind = "builtin";
        }
        string baseUrl = NormalizeBaseUrl(raw["base_url"]?.ToString());
        string chatModel = raw["chat_model"]?.ToString()?.Trim();
@@ -66,11 +68,26 @@ public partial class SwarmAssistentExtension
        {
            await AssistentParkLlm(session, baseUrl, chatModel);
        }
-        JObject export = await AssistentExportDataset(session, "approved", "jsonl");
-        string datasetPath = export["path"]?.ToString();
-        if (string.IsNullOrWhiteSpace(datasetPath) || !File.Exists(datasetPath))
+        string datasetPath = null;
+        int exportCount = 0;
+        if (string.IsNullOrWhiteSpace(hfDataset))
        {
-            return new JObject { ["error"] = "Нет одобренных примеров для тренировки" };
+            JObject export = await AssistentExportDataset(session, "approved", "jsonl");
+            datasetPath = export["path"]?.ToString();
+            exportCount = export["count"]?.Value<int?>() ?? 0;
+            if (string.IsNullOrWhiteSpace(datasetPath) || !File.Exists(datasetPath) || exportCount <= 0)
+            {
+                return new JObject { ["error"] = "Нет одобренных примеров для тренировки (или укажи hf_dataset)" };
+            }
+        }
+        else
+        {
+            JObject export = await AssistentExportDataset(session, "approved", "jsonl");
+            exportCount = export["count"]?.Value<int?>() ?? 0;
+            if (exportCount > 0 && File.Exists(export["path"]?.ToString() ?? ""))
+            {
+                datasetPath = export["path"]?.ToString();
+            }
        }
        long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
        string jobId = $"tj_{now}";
@@ -78,19 +95,29 @@ public partial class SwarmAssistentExtension
        Directory.CreateDirectory(jobDir);
        string configPath = Path.Combine(jobDir, "config.json");
        string logPath = Path.Combine(jobDir, "log.txt");
+        string adapterDir = Path.Combine(TrainingRoot(), "adapters", SanitizeAdapterName(outputName));
+        Directory.CreateDirectory(adapterDir);
        JObject jobConfig = new()
        {
            ["base_model"] = hfBase,
            ["output_name"] = outputName,
+            ["ollama_base"] = raw["ollama_base"]?.ToString()?.Trim(),
+            ["gguf_base_path"] = raw["gguf_base_path"]?.ToString()?.Trim() ?? runner["gguf_base_path"]?.ToString()?.Trim(),
            ["dataset_path"] = datasetPath,
-            ["hf_dataset"] = raw["hf_dataset"],
+            ["hf_dataset"] = hfDataset,
+            ["hf_mapping"] = hfMapping,
+            ["hf_schema"] = hfCheck?["schema"],
+            ["max_samples"] = raw["max_samples"] ?? 0,
            ["rank"] = raw["rank"] ?? 16,
            ["alpha"] = raw["alpha"] ?? 32,
            ["lr"] = raw["lr"] ?? 0.0002,
            ["epochs"] = raw["epochs"] ?? 3,
            ["seq_len"] = raw["seq_len"] ?? 2048,
            ["four_bit"] = raw["four_bit"] ?? true,
-            ["adapter_dir"] = Path.Combine(TrainingRoot(), "adapters", outputName),
+            ["batch_size"] = raw["batch_size"] ?? 1,
+            ["gradient_accumulation_steps"] = raw["gradient_accumulation_steps"] ?? 4,
+            ["adapter_dir"] = adapterDir,
+            ["local_export_count"] = exportCount,
        };
        await File.WriteAllTextAsync(configPath, jobConfig.ToString(Newtonsoft.Json.Formatting.Indented), Encoding.UTF8);
        Memory.SaveTrainJob(new JObject
@@ -114,6 +141,21 @@ public partial class SwarmAssistentExtension
        return new JObject { ["success"] = true, ["job_id"] = jobId, ["log_path"] = logPath };
    }

+    static string SanitizeAdapterName(string name)
+    {
+        if (string.IsNullOrWhiteSpace(name))
+        {
+            return "adapter";
+        }
+        char[] bad = Path.GetInvalidFileNameChars();
+        StringBuilder sb = new();
+        foreach (char c in name)
+        {
+            sb.Append(Array.IndexOf(bad, c) >= 0 ? '_' : c);
+        }
+        return sb.ToString().Trim();
+    }
+
    static string BuildRunnerCommand(JObject runner, string configPath, string logPath, string workDir)
    {
        string python = runner["python"]?.ToString()?.Trim();
@@ -121,10 +163,10 @@ public partial class SwarmAssistentExtension
        {
            python = "python";
        }
-        string kind = runner["kind"]?.ToString()?.Trim() ?? "custom";
+        string kind = runner["kind"]?.ToString()?.Trim() ?? "builtin";
        string custom = runner["cmd"]?.ToString()?.Trim();
        string scriptPath = Path.Combine(FilePath, "scripts", "train_qlora.py");
-        if (kind == "custom" && !string.IsNullOrWhiteSpace(custom))
+        if (string.Equals(kind, "custom", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(custom))
        {
            return custom
                .Replace("{python}", python, StringComparison.OrdinalIgnoreCase)
@@ -175,27 +217,38 @@ public partial class SwarmAssistentExtension
        return new JObject { ["success"] = true };
    }

-    internal async Task FinishTrainJobAsync(string jobId, bool success, string logPath, Session session, string baseUrl, string chatModel, string adapterDir, string outputName, string ggufScript)
+    internal async Task FinishTrainJobAsync(
+        string jobId,
+        bool success,
+        string logPath,
+        Session session,
+        string baseUrl,
+        string chatModel,
+        JObject jobConfig,
+        JObject runner)
    {
        long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+        string adapterDir = jobConfig?["adapter_dir"]?.ToString() ?? "";
+        string outputName = jobConfig?["output_name"]?.ToString() ?? "";
+        JObject progress = TrainingJobManager.GetProgress();
+        string finalStatus = success ? "completed" : "failed";
+        if (success && Directory.Exists(adapterDir))
+        {
+            JObject reg = await RegisterAdapterPipeline(session, baseUrl, outputName, adapterDir, jobConfig, runner);
+            progress["ollama"] = reg;
+            if (reg["success"]?.Value<bool?>() != true && reg["skipped"]?.Value<bool?>() != true)
+            {
+                finalStatus = "completed_with_warnings";
+            }
+        }
+        progress["status"] = finalStatus;
        Memory.SaveTrainJob(new JObject
        {
            ["id"] = jobId,
-            ["status"] = success ? "completed" : "failed",
+            ["status"] = finalStatus,
            ["finished_at"] = now,
-            ["progress"] = TrainingJobManager.GetProgress(),
+            ["progress"] = progress,
        });
-        if (success && Directory.Exists(adapterDir))
-        {
-            try
-            {
-                await RegisterAdapterInOllama(session, baseUrl, outputName, adapterDir, ggufScript);
-            }
-            catch (Exception ex)
-            {
-                Logs.Debug($"RegisterAdapter: {ex.Message}");
-            }
-        }
        if (!string.IsNullOrWhiteSpace(chatModel))
        {
            await AssistentWarmLlm(session, baseUrl, chatModel);
@@ -203,19 +256,115 @@ public partial class SwarmAssistentExtension
        TrainingJobManager.ClearRunning();
    }

-    async Task RegisterAdapterInOllama(Session session, string baseUrl, string outputName, string adapterDir, string ggufScript)
+    async Task<JObject> RegisterAdapterPipeline(Session session, string baseUrl, string outputName, string adapterDir, JObject jobConfig, JObject runner)
    {
-        string adapterFile = Directory.GetFiles(adapterDir, "*.gguf").FirstOrDefault()
-            ?? Directory.GetFiles(adapterDir, "adapter_model.safetensors").FirstOrDefault();
-        if (string.IsNullOrWhiteSpace(adapterFile))
+        string safetensors = Directory.GetFiles(adapterDir, "adapter_model.safetensors").FirstOrDefault();
+        if (string.IsNullOrWhiteSpace(safetensors))
        {
-            return;
+            return new JObject { ["success"] = false, ["error"] = "adapter_model.safetensors not found" };
        }
+        string ggufPath = Directory.GetFiles(adapterDir, "*.gguf").FirstOrDefault();
+        string ggufScript = runner?["gguf_script"]?.ToString()?.Trim();
+        string ggufBase = jobConfig?["gguf_base_path"]?.ToString()?.Trim() ?? runner?["gguf_base_path"]?.ToString()?.Trim();
+        string python = runner?["python"]?.ToString()?.Trim();
+        if (string.IsNullOrWhiteSpace(python))
+        {
+            python = "python";
+        }
+        if (string.IsNullOrWhiteSpace(ggufPath) && !string.IsNullOrWhiteSpace(ggufScript) && File.Exists(ggufScript))
+        {
+            if (string.IsNullOrWhiteSpace(ggufBase) || !File.Exists(ggufBase))
+            {
+                return new JObject
+                {
+                    ["success"] = false,
+                    ["skipped"] = true,
+                    ["error"] = "gguf_base_path не задан или файл не найден — адаптер сохранён как safetensors",
+                    ["adapter_dir"] = adapterDir,
+                };
+            }
+            ggufPath = Path.Combine(adapterDir, "adapter.gguf");
+            string ggufCmd = runner?["gguf_cmd"]?.ToString()?.Trim();
+            if (string.IsNullOrWhiteSpace(ggufCmd))
+            {
+                ggufCmd = "\"{python}\" \"{script}\" \"{base}\" \"{lora}\" \"{out}\"";
+            }
+            string cmd = ggufCmd
+                .Replace("{python}", python, StringComparison.OrdinalIgnoreCase)
+                .Replace("{script}", ggufScript, StringComparison.OrdinalIgnoreCase)
+                .Replace("{base}", ggufBase, StringComparison.OrdinalIgnoreCase)
+                .Replace("{lora}", adapterDir, StringComparison.OrdinalIgnoreCase)
+                .Replace("{out}", ggufPath, StringComparison.OrdinalIgnoreCase);
+            int code = await RunShellCommandAsync(cmd, adapterDir);
+            if (code != 0 || !File.Exists(ggufPath))
+            {
+                return new JObject
+                {
+                    ["success"] = false,
+                    ["error"] = $"GGUF convert failed exit={code}",
+                    ["adapter_dir"] = adapterDir,
+                };
+            }
+        }
+        if (string.IsNullOrWhiteSpace(ggufPath) || !File.Exists(ggufPath))
+        {
+            return new JObject
+            {
+                ["success"] = false,
+                ["skipped"] = true,
+                ["note"] = "Настрой convert_lora_to_gguf.py и gguf_base_path для регистрации в Ollama",
+                ["adapter_dir"] = adapterDir,
+            };
+        }
+        string ollamaBase = jobConfig?["ollama_base"]?.ToString()?.Trim();
+        if (string.IsNullOrWhiteSpace(ollamaBase))
+        {
+            return new JObject
+            {
+                ["success"] = false,
+                ["skipped"] = true,
+                ["error"] = "ollama_base не задан — укажи базовую Ollama-модель на форме QLoRA",
+                ["adapter_dir"] = adapterDir,
+                ["gguf"] = ggufPath,
+            };
+        }
+        return await RegisterAdapterInOllama(baseUrl, outputName, ollamaBase, ggufPath);
+    }
+
+    static async Task<int> RunShellCommandAsync(string commandLine, string workDir)
+    {
+        try
+        {
+            ProcessStartInfo psi = new()
+            {
+                FileName = "cmd.exe",
+                Arguments = $"/c {commandLine}",
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true,
+                WorkingDirectory = workDir ?? Environment.CurrentDirectory,
+            };
+            using Process proc = Process.Start(psi);
+            if (proc is null)
+            {
+                return -1;
+            }
+            await proc.WaitForExitAsync();
+            return proc.ExitCode;
+        }
+        catch (Exception ex)
+        {
+            Logs.Debug($"RunShellCommand: {ex.Message}");
+            return -1;
+        }
+    }
+
+    async Task<JObject> RegisterAdapterInOllama(string baseUrl, string outputName, string ollamaBase, string adapterGguf)
+    {
        StringBuilder mf = new();
-        JObject job = Memory.GetTrainJob(TrainingJobManager.CurrentJobId ?? "");
-        string baseModel = job?["base_model"]?.ToString() ?? "unknown";
-        mf.AppendLine($"FROM {baseModel}");
-        mf.AppendLine($"ADAPTER {adapterFile.Replace("\\", "/")}");
+        mf.AppendLine($"FROM {ollamaBase}");
+        mf.AppendLine($"ADAPTER {adapterGguf.Replace("\\", "/")}");
        JObject payload = new()
        {
            ["name"] = outputName,
@@ -224,14 +373,31 @@ public partial class SwarmAssistentExtension
        };
        using StringContent content = new(payload.ToString(Newtonsoft.Json.Formatting.None), Encoding.UTF8, "application/json");
        using HttpResponseMessage resp = await HttpClient.PostAsync($"{NormalizeBaseUrl(baseUrl)}/api/create", content);
-        _ = await resp.Content.ReadAsStringAsync();
+        string body = await resp.Content.ReadAsStringAsync();
+        if (!resp.IsSuccessStatusCode)
+        {
+            return new JObject
+            {
+                ["success"] = false,
+                ["error"] = $"ollama create HTTP {(int)resp.StatusCode}: {Clip(body, 400)}",
+                ["modelfile"] = mf.ToString(),
+            };
+        }
+        return new JObject
+        {
+            ["success"] = true,
+            ["name"] = outputName,
+            ["ollama_base"] = ollamaBase,
+            ["adapter"] = adapterGguf,
+            ["response"] = body,
+        };
    }
 }

 sealed class TrainingJobManager
 {
    static readonly Regex LossRe = new(@"loss[:\s]+([0-9.]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
-    static readonly Regex StepRe = new(@"(\d+)\s*/\s*(\d+)", RegexOptions.Compiled);
+    static readonly Regex StepRe = new(@"step\s+(\d+)\s*/\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);

    Process _process;
    readonly object _lock = new();
@@ -242,6 +408,8 @@ sealed class TrainingJobManager
    string _jobId;
    string _baseUrl;
    string _chatModel;
+    long _lastProgressSaveMs;
+    int _lastSavedStep = -1;

    public bool IsRunning { get; private set; }
    public string CurrentJobId => _jobId;
@@ -260,6 +428,8 @@ sealed class TrainingJobManager
            _logPath = logPath;
            _baseUrl = baseUrl;
            _chatModel = chatModel;
+            _lastProgressSaveMs = 0;
+            _lastSavedStep = -1;
            _progress = new JObject { ["status"] = "running", ["step"] = 0, ["loss"] = null, ["log"] = "" };
            try
            {
@@ -276,6 +446,7 @@ sealed class TrainingJobManager
                if (!string.IsNullOrWhiteSpace(hfToken))
                {
                    psi.Environment["HF_TOKEN"] = hfToken;
+                    psi.Environment["HUGGING_FACE_HUB_TOKEN"] = hfToken;
                }
                _process = new Process { StartInfo = psi, EnableRaisingEvents = true };
                _process.OutputDataReceived += (_, e) => AppendLog(e.Data);
@@ -313,7 +484,7 @@ sealed class TrainingJobManager
                // ignore
            }
            string prev = _progress["log"]?.ToString() ?? "";
-            string combined = (prev + line + "\n");
+            string combined = prev + line + "\n";
            if (combined.Length > 12000)
            {
                combined = combined[^12000..];
@@ -327,33 +498,48 @@ sealed class TrainingJobManager
            Match stepM = StepRe.Match(line);
            if (stepM.Success)
            {
-                _progress["step"] = int.Parse(stepM.Groups[1].Value);
-                _progress["total_steps"] = int.Parse(stepM.Groups[2].Value);
-                int total = int.Parse(stepM.Groups[2].Value);
                int step = int.Parse(stepM.Groups[1].Value);
+                int total = int.Parse(stepM.Groups[2].Value);
+                _progress["step"] = step;
+                _progress["total_steps"] = total;
                _progress["percent"] = total > 0 ? (int)(100.0 * step / total) : 0;
            }
-            try
+            MaybeSaveProgress(stepM.Success ? int.Parse(stepM.Groups[1].Value) : -1);
+        }
+    }
+
+    void MaybeSaveProgress(int step)
+    {
+        long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+        bool stepChanged = step >= 0 && step != _lastSavedStep;
+        if (!stepChanged && now - _lastProgressSaveMs < 2500)
+        {
+            return;
+        }
+        _lastProgressSaveMs = now;
+        if (step >= 0)
+        {
+            _lastSavedStep = step;
+        }
+        try
+        {
+            _ext?.Memory?.SaveTrainJob(new JObject
            {
-                _ext?.Memory?.SaveTrainJob(new JObject
-                {
-                    ["id"] = _jobId,
-                    ["status"] = "running",
-                    ["progress"] = _progress,
-                });
-            }
-            catch
-            {
-                // ignore
-            }
+                ["id"] = _jobId,
+                ["status"] = "running",
+                ["progress"] = _progress,
+            });
+        }
+        catch
+        {
+            // ignore
        }
    }

    async Task OnExited()
    {
        bool ok = false;
-        string adapterDir = "";
-        string outputName = "";
+        JObject jobConfig = new();
        lock (_lock)
        {
            ok = _process?.ExitCode == 0;
@@ -366,16 +552,18 @@ sealed class TrainingJobManager
            JObject job = _ext.Memory.GetTrainJob(_jobId);
            try
            {
-                JObject cfg = JObject.Parse(job?["config_json"]?.ToString() ?? "{}");
-                adapterDir = cfg["adapter_dir"]?.ToString() ?? "";
-                outputName = cfg["output_name"]?.ToString() ?? job?["output_name"]?.ToString() ?? "";
+                string cfgRaw = job?["config_json"]?.ToString();
+                if (!string.IsNullOrWhiteSpace(cfgRaw))
+                {
+                    jobConfig = JObject.Parse(cfgRaw);
+                }
            }
            catch
            {
                // ignore
            }
            JObject runner = _ext.Config.LoadTrainingRunner();
-            await _ext.FinishTrainJobAsync(_jobId, ok, _logPath, _session, _baseUrl, _chatModel, adapterDir, outputName, runner["gguf_script"]?.ToString());
+            await _ext.FinishTrainJobAsync(_jobId, ok, _logPath, _session, _baseUrl, _chatModel, jobConfig, runner);
        }
    }