From e1b5743795b39ab48187ef0abe783bd9ce29e856 Mon Sep 17 00:00:00 2001 From: Leonid Pershin Date: Sun, 23 Aug 2026 23:15:27 +0300 Subject: [PATCH] Add Knowledge Hub books FTS and remove training UI (0.16.0). Index Assistent/books search.jsonl, expose knowledge hops/catalog in chat, and drop QLoRA/HF training stack. Co-authored-by: Cursor --- Assets/assistent.bundle.js | 1839 ++++++++-------------------- AssistentChatPipeline.cs | 50 +- AssistentConfig.cs | 174 ++- AssistentHuggingFace.cs | 684 ----------- AssistentKnowledge.cs | 214 ++++ AssistentKnowledgeApi.cs | 129 ++ AssistentMemory.Books.cs | 458 +++++++ AssistentMemory.Heard.cs | 41 +- AssistentMemory.Training.cs | 385 ------ AssistentMemory.cs | 21 +- AssistentOllama.cs | 6 +- AssistentPatch.cs | 4 + AssistentTraining.Agent.cs | 181 --- AssistentTraining.cs | 456 ------- AssistentTrainingJobs.cs | 607 --------- Config/_base/patch-keys.json | 2 +- Config/_base/skills/knowledge.json | 6 + Config/_base/skills/knowledge.md | 22 + Config/_base/skills/memory.md | 2 +- Config/_base/training-qlora.json | 49 - SwarmAssistentExtension.cs | 32 +- Tabs/Text2Image/Assistent.html | 161 +-- src/app.js | 167 +-- src/main.js | 3 - src/training.js | 942 -------------- 25 files changed, 1611 insertions(+), 5024 deletions(-) delete mode 100644 AssistentHuggingFace.cs create mode 100644 AssistentKnowledge.cs create mode 100644 AssistentKnowledgeApi.cs create mode 100644 AssistentMemory.Books.cs delete mode 100644 AssistentMemory.Training.cs delete mode 100644 AssistentTraining.Agent.cs delete mode 100644 AssistentTraining.cs delete mode 100644 AssistentTrainingJobs.cs create mode 100644 Config/_base/skills/knowledge.json create mode 100644 Config/_base/skills/knowledge.md delete mode 100644 Config/_base/training-qlora.json delete mode 100644 src/training.js diff --git a/Assets/assistent.bundle.js b/Assets/assistent.bundle.js index ccab9cd..66a9c95 100644 --- a/Assets/assistent.bundle.js +++ b/Assets/assistent.bundle.js @@ -1792,7 +1792,7 @@ } if (window.SA && typeof SA.createActivityController === "function") { state.activity = SA.createActivityController({ - getMessagesEl: () => $2("sa_messages"), + getMessagesEl: () => $("sa_messages"), scrollToBottom: () => scrollMessagesToBottom(), hideEmpty: () => hideChatEmpty() }); @@ -1863,7 +1863,7 @@ function diskPersist() { return window.SA && window.SA.persist || null; } - function $2(id) { + function $(id) { return document.getElementById(id); } function modelShort(name) { @@ -1879,21 +1879,21 @@ return `${Math.floor(s / 60)}m ${String(s % 60).padStart(2, "0")}s`; } function hideChatEmpty() { - const empty = $2("sa_chat_empty"); + const empty = $("sa_chat_empty"); if (empty) { empty.hidden = true; } } let scrollMessagesRaf = 0; function messagesNearBottom(thresholdPx = 96) { - const box = $2("sa_messages"); + const box = $("sa_messages"); if (!box) { return true; } return box.scrollHeight - box.scrollTop - box.clientHeight <= thresholdPx; } function scrollMessagesToBottom({ force = false } = {}) { - const box = $2("sa_messages"); + const box = $("sa_messages"); if (!box) { return; } @@ -1905,15 +1905,15 @@ } scrollMessagesRaf = requestAnimationFrame(() => { scrollMessagesRaf = 0; - const el = $2("sa_messages"); + const el = $("sa_messages"); if (el && (force || messagesNearBottom(120))) { el.scrollTop = el.scrollHeight; } }); } function showChatEmptyIfIdle() { - const box = $2("sa_messages"); - const empty = $2("sa_chat_empty"); + const box = $("sa_messages"); + const empty = $("sa_chat_empty"); if (!box || !empty) { return; } @@ -1934,7 +1934,7 @@ if (!state.gotDelta && (state.busyPhase === "thinking" || state.busyPhase === "waiting") && elapsed > 1600) { state.busyPhase = state.llmParked || state.expectColdLoad ? "loading" : "waiting"; } - const model = modelShort($2("sa_model")?.value); + const model = modelShort($("sa_model")?.value); const labels = { encoding: "Encoding image\u2026", waiting: "\u0416\u0434\u0443 Ollama / \u043F\u0435\u0440\u0432\u044B\u0439 \u0442\u043E\u043A\u0435\u043D\u2026", @@ -1983,15 +1983,15 @@ } } } - const barText = $2("sa_livebar_text"); + const barText = $("sa_livebar_text"); if (barText) { barText.textContent = text; } - const elapsedEl = $2("sa_elapsed"); + const elapsedEl = $("sa_elapsed"); if (elapsedEl) { elapsedEl.textContent = fmtElapsed(elapsed); } - const status = $2("sa_status"); + const status = $("sa_status"); if (status) { status.textContent = text; status.classList.add("sa-status-busy"); @@ -2001,24 +2001,24 @@ state.busyStarted = Date.now(); state.gotDelta = false; state.busyPhase = phase || "thinking"; - $2("swarm_assistent_root")?.classList.add("sa-is-busy"); - $2("sa_composer")?.classList.add("sa-composer-busy"); + $("swarm_assistent_root")?.classList.add("sa-is-busy"); + $("sa_composer")?.classList.add("sa-composer-busy"); state.lastBusyPhaseShown = ""; - const send = $2("sa_btn_send"); + const send = $("sa_btn_send"); if (send) { send.disabled = false; } - const input = $2("sa_input"); + const input = $("sa_input"); if (input) { input.readOnly = false; input.disabled = false; input.classList.add("sa-input-busy"); } - const bar = $2("sa_livebar"); + const bar = $("sa_livebar"); if (bar) { bar.hidden = false; } - const dot = $2("sa_live_dot"); + const dot = $("sa_live_dot"); if (dot) { dot.hidden = false; } @@ -2040,25 +2040,25 @@ state.busyPhase = "idle"; state.lastBusyPhaseShown = ""; activityFinish(finalStatus || "\u0413\u043E\u0442\u043E\u0432\u043E"); - $2("swarm_assistent_root")?.classList.remove("sa-is-busy"); - $2("sa_composer")?.classList.remove("sa-composer-busy"); - const send = $2("sa_btn_send"); + $("swarm_assistent_root")?.classList.remove("sa-is-busy"); + $("sa_composer")?.classList.remove("sa-composer-busy"); + const send = $("sa_btn_send"); if (send) { send.disabled = false; } - const input = $2("sa_input"); + const input = $("sa_input"); if (input) { input.classList.remove("sa-input-busy"); } - const bar = $2("sa_livebar"); + const bar = $("sa_livebar"); if (bar) { bar.hidden = true; } - const dot = $2("sa_live_dot"); + const dot = $("sa_live_dot"); if (dot) { dot.hidden = true; } - const status = $2("sa_status"); + const status = $("sa_status"); if (status) { status.classList.remove("sa-status-busy"); } @@ -2070,13 +2070,13 @@ syncGenerateBusy(); } function setStatus(text) { - const el = $2("sa_status"); + const el = $("sa_status"); if (el) { el.textContent = text || ""; } } function setInterruptVisible(on) { - const btn = $2("sa_btn_interrupt"); + const btn = $("sa_btn_interrupt"); if (btn) { btn.hidden = !on; btn.classList.toggle("sa-interrupt-active", !!on); @@ -2157,8 +2157,8 @@ let lastExactForceCkpt = null; function updateGate() { const ok = isKreaSelected(); - const gate = $2("sa_gate"); - const layout = $2("sa_layout"); + const gate = $("sa_gate"); + const layout = $("sa_layout"); if (gate) { gate.hidden = ok; if (!ok) { @@ -2166,7 +2166,7 @@ const seen = [m.architecture, m.compat_class, m.name].filter(Boolean).join(" \xB7 "); const p = gate.querySelector("p"); if (p) { - p.innerHTML = seen ? `Swarm Assistent is for Krea 2 models only. Current: ${escapeHtml2( + p.innerHTML = seen ? `Swarm Assistent is for Krea 2 models only. Current: ${escapeHtml( seen )} \u2014 pick a checkpoint with architecture krea-2.` : "Swarm Assistent is for Krea 2 models only. Select a Krea 2 checkpoint on Generate to enable the chat."; } @@ -2190,7 +2190,7 @@ } return ok; } - function escapeHtml2(s) { + function escapeHtml(s) { return String(s).replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); } const PROSE_SECTION_TITLES = { @@ -2271,7 +2271,7 @@ return; } parts.push( - `
    ${listItems.map((li) => `
  • ${formatProseInline(escapeHtml2(li))}
    • `).join("")}
` + `
    ${listItems.map((li) => `
  • ${formatProseInline(escapeHtml(li))}
    • `).join("")}
` ); listItems = []; }; @@ -2285,7 +2285,7 @@ } const level = Math.min((line.match(/^#+/) || ["###"])[0].length, 3); parts.push( - `
${escapeHtml2(title)}
` + `
${escapeHtml(title)}
` ); continue; } @@ -2299,7 +2299,7 @@ parts.push(''); continue; } - parts.push(`

${formatProseInline(escapeHtml2(line))}

`); + parts.push(`

${formatProseInline(escapeHtml(line))}

`); } flushList(); return parts.join(""); @@ -2759,7 +2759,7 @@ ${patch.prompt}`; } } function syncBuildGenButton() { - const btn = $2("sa_btn_build_gen"); + const btn = $("sa_btn_build_gen"); if (!btn) { return; } @@ -2773,11 +2773,11 @@ ${patch.prompt}`; } } function defaultPackId() { - return state.config?.assistant?.default_pack || $2("sa_pack")?.querySelector("option")?.value || "ordinary"; + return state.config?.assistant?.default_pack || $("sa_pack")?.querySelector("option")?.value || "ordinary"; } function syncModeBadge() { - const badge = $2("sa_mode_badge"); - const pack = $2("sa_pack")?.value || defaultPackId(); + const badge = $("sa_mode_badge"); + const pack = $("sa_pack")?.value || defaultPackId(); if (!badge) { return; } @@ -2867,7 +2867,7 @@ ${patch.prompt}`; function paramDisplayTag(label, displayValue, isCustom, defaultHint) { const cls = isCustom ? "sa-param-tag sa-param-custom" : "sa-param-tag"; const hint = isCustom && defaultHint != null ? ` title="Exact: ${defaultHint}"` : ""; - return `${escapeHtml2(label)} ${escapeHtml2(String(displayValue))}`; + return `${escapeHtml(label)} ${escapeHtml(String(displayValue))}`; } function buildLiveParamsHtml() { const ckptProfile = detectKreaProfileName2(); @@ -2978,8 +2978,8 @@ ${patch.prompt}`; } function syncLiveParamsBar() { const html = buildLiveParamsHtml(); - const boardEl = $2("sa_live_params"); - const composerEl = $2("sa_composer_params"); + const boardEl = $("sa_live_params"); + const composerEl = $("sa_composer_params"); if (boardEl) { boardEl.innerHTML = html; } @@ -3272,7 +3272,7 @@ ${patch.prompt}`; const tab = document.getElementById(TAB_BUTTON_ID); if (tab) { tab.click(); - setTimeout(() => $2("sa_input")?.focus(), 50); + setTimeout(() => $("sa_input")?.focus(), 50); return true; } const pane = document.getElementById("assistent"); @@ -3282,7 +3282,7 @@ ${patch.prompt}`; } catch (e) { } } - setTimeout(() => $2("sa_input")?.focus(), 50); + setTimeout(() => $("sa_input")?.focus(), 50); return !!tab; } function historyMessageLimit() { @@ -3331,7 +3331,7 @@ ${patch.prompt}`; const mem = getContextMemory(); const memChars = mem.summary ? mem.summary.length : 0; const hist = modelMessages || assembleOutgoingMessages(); - const numCtx = Number($2("sa_num_ctx")?.value) || state.config?.assistant?.num_ctx || 16384; + const numCtx = Number($("sa_num_ctx")?.value) || state.config?.assistant?.num_ctx || 16384; const numPredict = Number(state.config?.assistant?.num_predict) || 3072; if (!C?.estimateBudget) { return { @@ -3477,7 +3477,7 @@ ${patch.prompt}`; if (!fold.length) { return false; } - const model = $2("sa_model")?.value; + const model = $("sa_model")?.value; if (!model) { setStatus("\u0412\u044B\u0431\u0435\u0440\u0438 \u043C\u043E\u0434\u0435\u043B\u044C Ollama \u0432 \u2699"); return false; @@ -3490,8 +3490,8 @@ ${patch.prompt}`; updateCtxChip(); setBusyPhase("compressing"); setStatus("\u0421\u0436\u0438\u043C\u0430\u044E \u043A\u043E\u043D\u0442\u0435\u043A\u0441\u0442\u2026"); - const persona = $2("sa_persona")?.value || "neutral"; - const baseUrl = $2("sa_base_url")?.value || "http://127.0.0.1:11434"; + const persona = $("sa_persona")?.value || "neutral"; + const baseUrl = $("sa_base_url")?.value || "http://127.0.0.1:11434"; const payload = { baseUrl, model, @@ -3505,7 +3505,7 @@ ${patch.prompt}`; fold_count: foldCount }), skills: [], - embed_model: $2("sa_embed_model")?.value || state.preferredEmbed || "" + embed_model: $("sa_embed_model")?.value || state.preferredEmbed || "" }; try { const data = await callOllamaOnce(payload); @@ -3560,7 +3560,7 @@ ${patch.prompt}`; return `${used} / ${cap}`; } function updateCtxChip() { - const chip = $2("sa_ctx_chip"); + const chip = $("sa_ctx_chip"); if (!chip) { return; } @@ -3590,8 +3590,8 @@ ${patch.prompt}`; } } function toggleCtxPanel(force) { - const panel = $2("sa_ctx_panel"); - const chip = $2("sa_ctx_chip"); + const panel = $("sa_ctx_panel"); + const chip = $("sa_ctx_chip"); if (!panel || !chip) { return; } @@ -3604,16 +3604,16 @@ ${patch.prompt}`; } } function renderCtxPanel() { - const body = $2("sa_ctx_panel_body"); - const bar = $2("sa_ctx_bar_fill"); - const auto = $2("sa_ctx_auto"); + const body = $("sa_ctx_panel_body"); + const bar = $("sa_ctx_bar_fill"); + const auto = $("sa_ctx_auto"); if (!body) { return; } const budget = currentBudgetEstimate(); const mem = getContextMemory(); const layers = state.lastSystemLayers || {}; - const layerRows = Object.entries(layers).filter(([k]) => k !== "total").map(([k, v]) => `
${escapeHtml2(k)}${Number(v) || 0}
`).join(""); + const layerRows = Object.entries(layers).filter(([k]) => k !== "total").map(([k, v]) => `
${escapeHtml(k)}${Number(v) || 0}
`).join(""); const keep = HISTORY_KEEP_TURNS; const uncovered = Math.max(0, (state.history || []).filter((m) => m && !m.systemish).length - (mem.untilCount || 0)); body.innerHTML = ` @@ -3623,7 +3623,7 @@ ${patch.prompt}`;
memory${budget.memoryChars || 0}
${layerRows ? `
system_layers
${layerRows}` : ""}
\u041C\u043E\u0434\u0435\u043B\u044C \u0432\u0438\u0434\u0438\u0442: ${mem.summary ? `\u0441\u0430\u043C\u043C\u0430\u0440\u0438 (${mem.foldedTurns || 0} \u0445\u043E\u0434\u043E\u0432) +` : ""} \u043F\u043E\u0441\u043B\u0435\u0434\u043D\u0438\u0435 ${keep} \u0445\u043E\u0434\u043E\u0432 \xB7 \u0441\u044B\u0440\u044B\u0445 \u0432 \u043E\u043A\u043D\u0435 \u2248 ${Math.min(uncovered, historyMessageLimit())}
- ${mem.summary ? `
${escapeHtml2(mem.summary.slice(0, 800))}${mem.summary.length > 800 ? "\u2026" : ""}
` : '
\u0421\u0430\u043C\u043C\u0430\u0440\u0438 \u0435\u0449\u0451 \u043D\u0435\u0442 \u2014 \u0441\u0442\u0430\u0440\u044B\u0435 \u0445\u043E\u0434\u044B \u043F\u0440\u043E\u0441\u0442\u043E \u043E\u0442\u0431\u0440\u0430\u0441\u044B\u0432\u0430\u044E\u0442\u0441\u044F.
'} + ${mem.summary ? `
${escapeHtml(mem.summary.slice(0, 800))}${mem.summary.length > 800 ? "\u2026" : ""}
` : '
\u0421\u0430\u043C\u043C\u0430\u0440\u0438 \u0435\u0449\u0451 \u043D\u0435\u0442 \u2014 \u0441\u0442\u0430\u0440\u044B\u0435 \u0445\u043E\u0434\u044B \u043F\u0440\u043E\u0441\u0442\u043E \u043E\u0442\u0431\u0440\u0430\u0441\u044B\u0432\u0430\u044E\u0442\u0441\u044F.
'} `; if (bar) { const pct = Math.max(0, Math.min(100, budget.used / (budget.numCtx || 1) * 100)); @@ -3640,7 +3640,7 @@ ${patch.prompt}`; setStatus("\u0417\u0430\u043D\u044F\u0442\u043E \u2014 \u0434\u043E\u0436\u0434\u0438\u0441\u044C \u043A\u043E\u043D\u0446\u0430 \u043E\u0442\u0432\u0435\u0442\u0430"); return; } - const model = $2("sa_model")?.value; + const model = $("sa_model")?.value; if (!model) { setStatus("\u0412\u044B\u0431\u0435\u0440\u0438 \u043C\u043E\u0434\u0435\u043B\u044C Ollama \u0432 \u2699"); return; @@ -4118,7 +4118,7 @@ ${patch.prompt}`; await runGenerateFromPatch({ actions: ["generate"] }, { force: true, fromSession: true }); } function renderBoard() { - const board = $2("sa_board"); + const board = $("sa_board"); if (!board) { return; } @@ -4242,11 +4242,11 @@ ${patch.prompt}`; return []; } function ensureGenLightbox() { - let root = $2("sa_gen_lightbox"); + let root = $("sa_gen_lightbox"); if (root) { return root; } - const host = $2("swarm_assistent_root") || document.body; + const host = $("swarm_assistent_root") || document.body; root = document.createElement("div"); root.id = "sa_gen_lightbox"; root.className = "sa-lightbox"; @@ -4320,9 +4320,9 @@ ${patch.prompt}`; return; } root.hidden = false; - const img = $2("sa_lb_img"); - const title = $2("sa_lb_title"); - const idx = $2("sa_lb_idx"); + const img = $("sa_lb_img"); + const title = $("sa_lb_title"); + const idx = $("sa_lb_idx"); if (img) { img.src = row.src; img.alt = row.label || row.id; @@ -4350,7 +4350,7 @@ ${patch.prompt}`; } function closeGenLightbox() { state.lightboxIndex = -1; - const root = $2("sa_gen_lightbox"); + const root = $("sa_gen_lightbox"); if (root) { root.hidden = true; } @@ -4369,23 +4369,23 @@ ${patch.prompt}`; } function syncBoardChrome() { const tab = state.boardTab === "refs" ? "refs" : "generate"; - $2("sa_board_tab_gen")?.classList.toggle("sa-board-tab-active", tab === "generate"); - $2("sa_board_tab_refs")?.classList.toggle("sa-board-tab-active", tab === "refs"); - $2("sa_board_tab_gen")?.setAttribute("aria-selected", tab === "generate" ? "true" : "false"); - $2("sa_board_tab_refs")?.setAttribute("aria-selected", tab === "refs" ? "true" : "false"); - const addBtn = $2("sa_btn_add_ref"); + $("sa_board_tab_gen")?.classList.toggle("sa-board-tab-active", tab === "generate"); + $("sa_board_tab_refs")?.classList.toggle("sa-board-tab-active", tab === "refs"); + $("sa_board_tab_gen")?.setAttribute("aria-selected", tab === "generate" ? "true" : "false"); + $("sa_board_tab_refs")?.setAttribute("aria-selected", tab === "refs" ? "true" : "false"); + const addBtn = $("sa_btn_add_ref"); if (addBtn) { addBtn.hidden = tab !== "refs"; } - const maskBtn = $2("sa_btn_as_mask"); - const clearSlotBtn = $2("sa_btn_clear_image"); + const maskBtn = $("sa_btn_as_mask"); + const clearSlotBtn = $("sa_btn_clear_image"); if (maskBtn) { maskBtn.hidden = tab !== "refs"; } if (clearSlotBtn) { clearSlotBtn.hidden = tab !== "refs"; } - const badge = $2("sa_refs_badge"); + const badge = $("sa_refs_badge"); if (badge) { const refs = refSlots(); const withImg = refs.filter((s) => s.src).length; @@ -4397,9 +4397,9 @@ ${patch.prompt}`; badge.hidden = true; } } - let genBadge = $2("sa_gen_badge"); + let genBadge = $("sa_gen_badge"); if (!genBadge) { - const genTab = $2("sa_board_tab_gen"); + const genTab = $("sa_board_tab_gen"); if (genTab) { genBadge = document.createElement("span"); genBadge.id = "sa_gen_badge"; @@ -4567,11 +4567,11 @@ ${patch.prompt}`; if (localStorage.getItem(LS_WELCOMED) === "1") { return; } - if (!$2("sa_messages")) { + if (!$("sa_messages")) { return; } localStorage.setItem(LS_WELCOMED, "1"); - const box = $2("sa_messages"); + const box = $("sa_messages"); hideChatEmpty(); const div = document.createElement("div"); div.className = "sa-msg assistant sa-welcome"; @@ -4674,8 +4674,8 @@ ${patch.prompt}`; state.chatSession = S.snapshotFromLive({ genFields: readLiveGenFields(), board: boardSnapshotForSession(), - persona: $2("sa_persona")?.value || "neutral", - pack: $2("sa_pack")?.value || "ordinary", + persona: $("sa_persona")?.value || "neutral", + pack: $("sa_pack")?.value || "ordinary", context_memory: getContextMemory() }); return state.chatSession; @@ -4752,7 +4752,7 @@ ${patch.prompt}`; function applyPersonaForChat(personaId, { quiet = false } = {}) { const id = String(personaId || "neutral").trim() || "neutral"; return new Promise((resolve) => { - const sel = $2("sa_persona"); + const sel = $("sa_persona"); if (sel && [...sel.options].some((o) => o.value === id)) { sel.value = id; } @@ -4766,7 +4766,7 @@ ${patch.prompt}`; resolve(); return; } - const packKeep = $2("sa_pack")?.value; + const packKeep = $("sa_pack")?.value; genericRequest( "AssistentGetConfig", { persona: id }, @@ -4901,7 +4901,7 @@ ${patch.prompt}`; persistChatsStore(); } function resetMessagesUi(emptyHint) { - const box = $2("sa_messages"); + const box = $("sa_messages"); if (!box) { return; } @@ -4913,7 +4913,7 @@ ${patch.prompt}`; box.appendChild(empty); } function renderHistoryIntoUi(messages) { - const box = $2("sa_messages"); + const box = $("sa_messages"); if (!box) { return; } @@ -4965,7 +4965,7 @@ ${patch.prompt}`; updateCtxChip(); } function updateSessionLabel() { - const el = $2("sa_session_label"); + const el = $("sa_session_label"); if (!el) { return; } @@ -4987,8 +4987,8 @@ ${patch.prompt}`; return !chat || !chatHasTranscript(chat); } function syncHistoryBadge() { - const btn = $2("sa_btn_chats"); - const countEl = $2("sa_chats_count"); + const btn = $("sa_btn_chats"); + const countEl = $("sa_chats_count"); const n = (state.chats || []).filter(chatHasTranscript).length; const open = !!state.chatsPanelOpen; if (btn) { @@ -5061,7 +5061,7 @@ ${patch.prompt}`; return bits.join(" \xB7 "); } function renderChatsList() { - const root = $2("sa_chats_list"); + const root = $("sa_chats_list"); if (!root) { return; } @@ -5086,19 +5086,19 @@ ${patch.prompt}`; row.dataset.id = c.id; row.setAttribute("role", "listitem"); const when = formatChatWhen(c.updatedAt); - const title = escapeHtml2(c.title || "\u041D\u043E\u0432\u044B\u0439 \u0447\u0430\u0442"); + const title = escapeHtml(c.title || "\u041D\u043E\u0432\u044B\u0439 \u0447\u0430\u0442"); const tipBits = [chatListParamsBits(c.params)].filter(Boolean); - const tip = tipBits.length ? ` title="${escapeHtml2(tipBits.join(" \xB7 "))}"` : ""; - row.innerHTML = ``; + const tip = tipBits.length ? ` title="${escapeHtml(tipBits.join(" \xB7 "))}"` : ""; + row.innerHTML = ``; root.appendChild(row); } } function setChatsPanelOpen(open) { state.chatsPanelOpen = !!open; state.chatsDrawerOpen = state.chatsPanelOpen; - const panel = $2("sa_chats_panel"); - const btn = $2("sa_btn_chats"); - const root = $2("swarm_assistent_root"); + const panel = $("sa_chats_panel"); + const btn = $("sa_btn_chats"); + const root = $("swarm_assistent_root"); if (panel) { panel.hidden = !state.chatsPanelOpen; } @@ -5109,7 +5109,7 @@ ${patch.prompt}`; syncHistoryBadge(); if (state.chatsPanelOpen) { saveActiveChatToStore(); - const search = $2("sa_chats_search"); + const search = $("sa_chats_search"); if (search) { search.value = state.chatsQuery || ""; search.focus(); @@ -5124,7 +5124,7 @@ ${patch.prompt}`; if (openDrawer) { setChatsPanelOpen(true); } - $2("sa_input")?.focus(); + $("sa_input")?.focus(); return; } if (state.busy || state.generating) { @@ -5174,7 +5174,7 @@ ${patch.prompt}`; setChatsPanelOpen(true); } if (openDrawer || !force) { - $2("sa_input")?.focus(); + $("sa_input")?.focus(); } } async function switchToChat(id) { @@ -5323,7 +5323,7 @@ ${patch.prompt}`; updateCtxChip(); } function hideSlashMenu() { - const menu = $2("sa_slash_menu"); + const menu = $("sa_slash_menu"); if (menu) { menu.hidden = true; menu.innerHTML = ""; @@ -5339,7 +5339,7 @@ ${patch.prompt}`; return SLASH_COMMANDS.filter((c) => c.cmd.toLowerCase().startsWith(q) || q === "/" || c.cmd.toLowerCase().includes(q.slice(1))); } function renderSlashMenu(items) { - const menu = $2("sa_slash_menu"); + const menu = $("sa_slash_menu"); if (!menu) { return; } @@ -5355,7 +5355,7 @@ ${patch.prompt}`; btn.type = "button"; btn.className = "sa-slash-item" + (i === state.slashIndex ? " sa-slash-active" : ""); btn.setAttribute("role", "option"); - btn.innerHTML = `${escapeHtml2(item.cmd.trim())} \u2014 ${escapeHtml2(item.hint)}`; + btn.innerHTML = `${escapeHtml(item.cmd.trim())} \u2014 ${escapeHtml(item.hint)}`; btn.addEventListener("mousedown", (e) => { e.preventDefault(); applySlashPick(item); @@ -5364,7 +5364,7 @@ ${patch.prompt}`; }); } function applySlashPick(item) { - const input = $2("sa_input"); + const input = $("sa_input"); if (!input || !item) { return; } @@ -5375,7 +5375,7 @@ ${patch.prompt}`; input.setSelectionRange(pos, pos); } function updateSlashMenuFromInput() { - const text = $2("sa_input")?.value || ""; + const text = $("sa_input")?.value || ""; if (!text.startsWith("/") || text.includes("\n") || /\s/.test(text.trim().slice(1)) && !text.endsWith(" ")) { const token2 = text.split(/\s/)[0] || ""; if (!token2.startsWith("/") || text.includes(" ") && !SLASH_COMMANDS.some((c) => c.cmd.startsWith(token2))) { @@ -5393,7 +5393,7 @@ ${patch.prompt}`; renderSlashMenu(slashMatches(token)); } function onPersonaChanged() { - const id = $2("sa_persona")?.value || "neutral"; + const id = $("sa_persona")?.value || "neutral"; state.sessionExact = {}; state.lastUserParamIntent = false; saveSettings(); @@ -5404,10 +5404,10 @@ ${patch.prompt}`; } appendSystemNote(`\u0422\u043E\u043D \u2192 ${title}`); state.pendingPersonaNote = `Persona is now ${id} (${title}). Adopt this voice from now on.`; - if (data?.assistant?.default_pack && $2("sa_pack") && !state.packUserTouched) { + if (data?.assistant?.default_pack && $("sa_pack") && !state.packUserTouched) { const packId = data.assistant.default_pack; - if ([...$2("sa_pack").options || []].some((o) => o.value === packId)) { - $2("sa_pack").value = packId; + if ([...$("sa_pack").options || []].some((o) => o.value === packId)) { + $("sa_pack").value = packId; } } fillEmptyParamsFromExact(); @@ -5616,7 +5616,7 @@ ${patch.prompt}`; has_vision_image: typeof visionReadySlots === "function" ? visionReadySlots().length > 0 : false, image_slots: typeof slotCatalog === "function" ? slotCatalog() : [], attached_slot_ids: typeof attachableSlots === "function" ? attachableSlots().map((s) => s.id) : [], - auto_apply: !!$2("sa_auto_apply")?.checked, + auto_apply: !!$("sa_auto_apply")?.checked, auto_generate: true, krea_profile: kreaProfile, recommended_params: { @@ -5833,8 +5833,8 @@ ${patch.prompt}`; setStatus("\u041F\u0430\u0442\u0447\u0438 \u0443\u0431\u0440\u0430\u043D\u044B \u0438\u0437 \u0447\u0430\u0442\u0430"); } function toggleMoreMenu(menuId, btnId) { - const menu = $2(menuId); - const btn = $2(btnId); + const menu = $(menuId); + const btn = $(btnId); if (!menu) { return; } @@ -5855,7 +5855,7 @@ ${patch.prompt}`; document.querySelectorAll("#sa_btn_board_more, #sa_btn_clear_more").forEach((b) => b.setAttribute("aria-expanded", "false")); } function setPackValue(packName, { flash, user } = {}) { - const pack = $2("sa_pack"); + const pack = $("sa_pack"); if (!pack || !packName) { return false; } @@ -5881,7 +5881,7 @@ ${patch.prompt}`; if (state.packUserTouched) { return null; } - const cur = $2("sa_pack")?.value || defaultPackId(); + const cur = $("sa_pack")?.value || defaultPackId(); if (cur === "ordinary") { return null; } @@ -5916,7 +5916,7 @@ ${patch.prompt}`; if (state.packUserTouched) { return; } - const cur = $2("sa_pack")?.value || ""; + const cur = $("sa_pack")?.value || ""; if (cur === "critique_image" || cur === "describe_ref") { setPackValue(defaultPackId(), { flash: true }); } @@ -6244,15 +6244,15 @@ ${patch.prompt}`; accent: p.accent, source: p.source })); - renderPersonaOptions(state.personas, preferId || $2("sa_persona")?.value); + renderPersonaOptions(state.personas, preferId || $("sa_persona")?.value); } - if (preferId && $2("sa_persona")) { - if ([...$2("sa_persona").options].some((o) => o.value === preferId)) { - $2("sa_persona").value = preferId; + if (preferId && $("sa_persona")) { + if ([...$("sa_persona").options].some((o) => o.value === preferId)) { + $("sa_persona").value = preferId; await applyPersonaForChat(preferId, { quiet: true }); } } else { - loadConfig($2("sa_persona")?.value, () => resolve()); + loadConfig($("sa_persona")?.value, () => resolve()); return; } resolve(); @@ -6279,16 +6279,16 @@ ${patch.prompt}`; return false; } function shouldParkLlmBeforeGen() { - return !!$2("sa_park_llm")?.checked; + return !!$("sa_park_llm")?.checked; } function parkLlm() { return new Promise((resolve) => { - const model = $2("sa_model")?.value; + const model = $("sa_model")?.value; if (!shouldParkLlmBeforeGen() || !model || state.llmParked || typeof genericRequest !== "function") { resolve(false); return; } - const baseUrl = $2("sa_base_url")?.value || "http://127.0.0.1:11434"; + const baseUrl = $("sa_base_url")?.value || "http://127.0.0.1:11434"; let settled = false; const finish = (ok) => { if (settled) { @@ -6307,7 +6307,7 @@ ${patch.prompt}`; } function warmLlm({ force = false } = {}) { return new Promise((resolve) => { - const model = $2("sa_model")?.value; + const model = $("sa_model")?.value; if (!model || typeof genericRequest !== "function") { resolve({ ok: false, alreadyResident: false }); return; @@ -6316,7 +6316,7 @@ ${patch.prompt}`; resolve({ ok: false, alreadyResident: false }); return; } - const baseUrl = $2("sa_base_url")?.value || "http://127.0.0.1:11434"; + const baseUrl = $("sa_base_url")?.value || "http://127.0.0.1:11434"; let settled = false; const finish = (ok, alreadyResident = false) => { if (settled) { @@ -6387,8 +6387,8 @@ ${patch.prompt}`; if (!text) { return; } - const persona = $2("sa_persona")?.value || localStorage.getItem(LS_PERSONA) || "neutral"; - const pack = $2("sa_pack")?.value || defaultPackId(); + const persona = $("sa_persona")?.value || localStorage.getItem(LS_PERSONA) || "neutral"; + const pack = $("sa_pack")?.value || defaultPackId(); const prose = visibleAssistantProse(text); if (state.streamEl) { finalizeStreamMessage(text, []); @@ -6751,7 +6751,7 @@ ${patch.prompt}`; } const paneVisible = !!document.getElementById("swarm_assistent_root")?.offsetParent; const multiDone = !!(jobs && finishedGenResultCount() > 1); - const willAutoCritique = !multiDone && !!$2("sa_auto_critique")?.checked; + const willAutoCritique = !multiDone && !!$("sa_auto_critique")?.checked; if (state.view === "chat" && paneVisible && !willAutoCritique && epoch === state.chatEpoch) { if (parkedBeforeWarm || state.expectColdLoad) { startBusyUi("warming"); @@ -6808,7 +6808,7 @@ ${patch.prompt}`; return src && !looksLikeModelPreview(src) ? src : null; } async function maybeAutoCritique(imageSrc) { - if (!$2("sa_auto_critique")?.checked || turnHopUsed("critique") || isMultiGenResults()) { + if (!$("sa_auto_critique")?.checked || turnHopUsed("critique") || isMultiGenResults()) { return; } const src = await resolveFinishedGenerateSrc(imageSrc); @@ -6820,8 +6820,8 @@ ${patch.prompt}`; return; } setPackValue("critique_image", { flash: true }); - if ($2("sa_input")) { - $2("sa_input").value = "Strict QC of this Generate frame vs the live prompt: list concrete defects and artifacts first, then one fenced JSON patch with an improved prompt and any param/LoRA fixes. No empty praise."; + if ($("sa_input")) { + $("sa_input").value = "Strict QC of this Generate frame vs the live prompt: list concrete defects and artifacts first, then one fenced JSON patch with an improved prompt and any param/LoRA fixes. No empty praise."; } const gen = generateSlot(); if (gen) { @@ -6834,7 +6834,7 @@ ${patch.prompt}`; restoreDefaultPackAfterHop(); } async function maybeAutoVisionLook(imageSrc) { - if (!wantsAutoVision() || $2("sa_auto_critique")?.checked || turnHopUsed("vision") || state.busy || isMultiGenResults()) { + if (!wantsAutoVision() || $("sa_auto_critique")?.checked || turnHopUsed("vision") || state.busy || isMultiGenResults()) { return; } const src = await resolveFinishedGenerateSrc(imageSrc); @@ -6851,8 +6851,8 @@ ${patch.prompt}`; return; } setPackValue("critique_image", { flash: true }); - if ($2("sa_input")) { - $2("sa_input").value = "Strict review of the Generate frame vs the prompt: what failed, what artifacts you see, what to change in prompt and params next. Skip hollow compliments."; + if ($("sa_input")) { + $("sa_input").value = "Strict review of the Generate frame vs the prompt: what failed, what artifacts you see, what to change in prompt and params next. Skip hollow compliments."; } setStatus("Auto look_at\u2026"); await sendChat({ fromVisionHop: true, forceSlotIds: [GEN_ID], skipAutoPack: true }); @@ -6883,13 +6883,13 @@ ${patch.prompt}`; setView("chat"); const label = (state.genResults || []).find((r) => r.id === state.selectedGenResultId)?.label; setPackValue("critique_image", { flash: true }); - if ($2("sa_input")) { - $2("sa_input").value = label ? `\u041A\u0440\u0438\u0442\u0438\u0447\u0435\u0441\u043A\u0438 \u0440\u0430\u0437\u0431\u0435\u0440\u0438 \xAB${label}\xBB: \u0447\u0442\u043E \u043D\u0435 \u0441\u0445\u043E\u0434\u0438\u0442\u0441\u044F \u0441 \u043F\u0440\u043E\u043C\u043F\u0442\u043E\u043C, \u0430\u0440\u0442\u0435\u0444\u0430\u043A\u0442\u044B, \u043A\u043E\u043C\u043F\u043E\u0437\u0438\u0446\u0438\u044F/\u0441\u0432\u0435\u0442 \u2014 \u0438 \u043A\u0430\u043A \u043F\u043E\u043F\u0440\u0430\u0432\u0438\u0442\u044C prompt \u0438 params \u0434\u043B\u044F \u0441\u043B\u0435\u0434\u0443\u044E\u0449\u0435\u0433\u043E \u043A\u0430\u0434\u0440\u0430. \u0411\u0435\u0437 \u043E\u0431\u0449\u0438\u0445 \u043F\u043E\u0445\u0432\u0430\u043B.` : "\u041A\u0440\u0438\u0442\u0438\u0447\u0435\u0441\u043A\u0438 \u0440\u0430\u0437\u0431\u0435\u0440\u0438 \u0440\u0435\u0437\u0443\u043B\u044C\u0442\u0430\u0442: \u0447\u0442\u043E \u043D\u0435 \u0441\u0445\u043E\u0434\u0438\u0442\u0441\u044F \u0441 \u043F\u0440\u043E\u043C\u043F\u0442\u043E\u043C, \u0430\u0440\u0442\u0435\u0444\u0430\u043A\u0442\u044B, \u043A\u043E\u043C\u043F\u043E\u0437\u0438\u0446\u0438\u044F/\u0441\u0432\u0435\u0442 \u2014 \u0438 \u043A\u0430\u043A \u043F\u043E\u043F\u0440\u0430\u0432\u0438\u0442\u044C prompt \u0438 params \u0434\u043B\u044F \u0441\u043B\u0435\u0434\u0443\u044E\u0449\u0435\u0433\u043E \u043A\u0430\u0434\u0440\u0430. \u0411\u0435\u0437 \u043E\u0431\u0449\u0438\u0445 \u043F\u043E\u0445\u0432\u0430\u043B."; + if ($("sa_input")) { + $("sa_input").value = label ? `\u041A\u0440\u0438\u0442\u0438\u0447\u0435\u0441\u043A\u0438 \u0440\u0430\u0437\u0431\u0435\u0440\u0438 \xAB${label}\xBB: \u0447\u0442\u043E \u043D\u0435 \u0441\u0445\u043E\u0434\u0438\u0442\u0441\u044F \u0441 \u043F\u0440\u043E\u043C\u043F\u0442\u043E\u043C, \u0430\u0440\u0442\u0435\u0444\u0430\u043A\u0442\u044B, \u043A\u043E\u043C\u043F\u043E\u0437\u0438\u0446\u0438\u044F/\u0441\u0432\u0435\u0442 \u2014 \u0438 \u043A\u0430\u043A \u043F\u043E\u043F\u0440\u0430\u0432\u0438\u0442\u044C prompt \u0438 params \u0434\u043B\u044F \u0441\u043B\u0435\u0434\u0443\u044E\u0449\u0435\u0433\u043E \u043A\u0430\u0434\u0440\u0430. \u0411\u0435\u0437 \u043E\u0431\u0449\u0438\u0445 \u043F\u043E\u0445\u0432\u0430\u043B.` : "\u041A\u0440\u0438\u0442\u0438\u0447\u0435\u0441\u043A\u0438 \u0440\u0430\u0437\u0431\u0435\u0440\u0438 \u0440\u0435\u0437\u0443\u043B\u044C\u0442\u0430\u0442: \u0447\u0442\u043E \u043D\u0435 \u0441\u0445\u043E\u0434\u0438\u0442\u0441\u044F \u0441 \u043F\u0440\u043E\u043C\u043F\u0442\u043E\u043C, \u0430\u0440\u0442\u0435\u0444\u0430\u043A\u0442\u044B, \u043A\u043E\u043C\u043F\u043E\u0437\u0438\u0446\u0438\u044F/\u0441\u0432\u0435\u0442 \u2014 \u0438 \u043A\u0430\u043A \u043F\u043E\u043F\u0440\u0430\u0432\u0438\u0442\u044C prompt \u0438 params \u0434\u043B\u044F \u0441\u043B\u0435\u0434\u0443\u044E\u0449\u0435\u0433\u043E \u043A\u0430\u0434\u0440\u0430. \u0411\u0435\u0437 \u043E\u0431\u0449\u0438\u0445 \u043F\u043E\u0445\u0432\u0430\u043B."; } await sendChat({ forceSlotIds: [GEN_ID], skipAutoPack: true }); } function currentPersonaInfo() { - const id = ($2("sa_persona")?.value || localStorage.getItem(LS_PERSONA) || "neutral").trim() || "neutral"; + const id = ($("sa_persona")?.value || localStorage.getItem(LS_PERSONA) || "neutral").trim() || "neutral"; const known = (state.personas || []).find((p) => p && p.id === id); return { id, @@ -6905,7 +6905,7 @@ ${patch.prompt}`; return; } const persona = meta.persona || currentPersonaInfo(); - const pack = meta.pack || $2("sa_pack")?.value || ""; + const pack = meta.pack || $("sa_pack")?.value || ""; div.dataset.persona = persona.id || "neutral"; if (pack) { div.dataset.pack = pack; @@ -6927,7 +6927,7 @@ ${patch.prompt}`; div.insertBefore(row, div.firstChild); } function appendMessage(role, text, patch, civitaiResults, meta) { - const box = $2("sa_messages"); + const box = $("sa_messages"); if (!box) { return null; } @@ -6959,7 +6959,7 @@ ${patch.prompt}`; return div; } function beginStreamMessage(meta) { - const box = $2("sa_messages"); + const box = $("sa_messages"); if (!box) { return null; } @@ -7091,71 +7091,16 @@ ${patch.prompt}`; } scrollMessagesToBottom(); } - function mountCurateButtons(msgEl, meta) { - if (!msgEl || msgEl.querySelector(".sa-msg-curate")) { - return; - } - const wrap = document.createElement("div"); - wrap.className = "sa-msg-curate"; - const ok = document.createElement("button"); - ok.type = "button"; - ok.className = "basic-button"; - ok.title = "\u0412 \u0434\u0430\u0442\u0430\u0441\u0435\u0442 (\u043E\u0434\u043E\u0431\u0440\u0438\u0442\u044C)"; - ok.textContent = "+ \u0434\u0430\u0442\u0430\u0441\u0435\u0442"; - ok.addEventListener("click", () => curateAssistantMessage(msgEl, "approved")); - const bad = document.createElement("button"); - bad.type = "button"; - bad.className = "basic-button"; - bad.title = "\u041E\u0442\u043A\u043B\u043E\u043D\u0438\u0442\u044C \u0434\u043B\u044F \u0434\u0430\u0442\u0430\u0441\u0435\u0442\u0430"; - bad.textContent = "\u0431\u0440\u0430\u043A"; - bad.addEventListener("click", () => curateAssistantMessage(msgEl, "rejected")); - wrap.appendChild(ok); - wrap.appendChild(bad); - msgEl.appendChild(wrap); + function mountCurateButtons() { } - function curateAssistantMessage(msgEl, status) { - const hist = state.history || []; - let asstText = msgEl.querySelector(".sa-msg-body")?.textContent?.trim() || msgEl.textContent?.trim() || ""; - let userText = ""; - for (let i = hist.length - 1; i >= 0; i--) { - if (hist[i]?.role === "assistant" && (hist[i].content || "").trim() === asstText.trim()) { - for (let j = i - 1; j >= 0; j--) { - if (hist[j]?.role === "user") { - userText = hist[j].content || ""; - break; - } - } - break; - } - } - if (!userText) { - for (let i = hist.length - 1; i >= 0; i--) { - if (hist[i]?.role === "user") { - userText = hist[i].content || ""; - break; - } - } - } - const messages = [ - { role: "user", content: userText }, - { role: "assistant", content: asstText } - ]; - window.SA?.training?.curateFromChat?.(messages, { - chatId: state.activeChatId, - persona: $2("sa_persona")?.value || "neutral", - pack: $2("sa_pack")?.value || defaultPackId(), - status - })?.then?.((ok) => { - if (ok) { - setStatus(status === "approved" ? "\u041F\u0440\u0438\u043C\u0435\u0440 \u0434\u043E\u0431\u0430\u0432\u043B\u0435\u043D \u0432 \u0434\u0430\u0442\u0430\u0441\u0435\u0442" : "\u041F\u0440\u0438\u043C\u0435\u0440 \u043E\u0442\u043C\u0435\u0447\u0435\u043D \u043A\u0430\u043A \u0431\u0440\u0430\u043A"); - } - }); + function curateAssistantMessage() { + setStatus("\u041E\u0431\u0443\u0447\u0435\u043D\u0438\u0435 \u0441\u043D\u044F\u0442\u043E \u2014 \u0438\u0441\u043F\u043E\u043B\u044C\u0437\u0443\u0439 books/knowledge"); } function isTrainingLocked() { - return !!state.trainingLock || document.getElementById("swarm_assistent_root")?.classList.contains("sa-root-training-lock"); + return false; } function wantsAutoVision() { - return !!$2("sa_auto_vision")?.checked; + return !!$("sa_auto_vision")?.checked; } function looksLikeModelPreview(src) { const s = String(src || "").toLowerCase(); @@ -7342,32 +7287,32 @@ ${patch.prompt}`; const autoDl = localStorage.getItem(LS_AUTO_DOWNLOAD); const parkLlm2 = localStorage.getItem(LS_PARK_LLM); const paneW = localStorage.getItem(LS_PANE_WIDTH); - if (base && $2("sa_base_url")) { - $2("sa_base_url").value = base; + if (base && $("sa_base_url")) { + $("sa_base_url").value = base; } - if (pack && $2("sa_pack")) { - $2("sa_pack").value = pack; + if (pack && $("sa_pack")) { + $("sa_pack").value = pack; } - if (persona && $2("sa_persona")) { - $2("sa_persona").value = persona; + if (persona && $("sa_persona")) { + $("sa_persona").value = persona; } - if (auto != null && $2("sa_auto_vision")) { - $2("sa_auto_vision").checked = auto === "1"; + if (auto != null && $("sa_auto_vision")) { + $("sa_auto_vision").checked = auto === "1"; } - if ($2("sa_auto_apply")) { - $2("sa_auto_apply").checked = autoApply == null ? true : autoApply === "1"; + if ($("sa_auto_apply")) { + $("sa_auto_apply").checked = autoApply == null ? true : autoApply === "1"; } - if ($2("sa_auto_generate")) { - $2("sa_auto_generate").checked = autoGen == null ? true : autoGen === "1"; + if ($("sa_auto_generate")) { + $("sa_auto_generate").checked = autoGen == null ? true : autoGen === "1"; } - if ($2("sa_auto_critique") && autoCrit != null) { - $2("sa_auto_critique").checked = autoCrit === "1"; + if ($("sa_auto_critique") && autoCrit != null) { + $("sa_auto_critique").checked = autoCrit === "1"; } - if ($2("sa_auto_download") && autoDl != null) { - $2("sa_auto_download").checked = autoDl === "1"; + if ($("sa_auto_download") && autoDl != null) { + $("sa_auto_download").checked = autoDl === "1"; } - if ($2("sa_park_llm")) { - $2("sa_park_llm").checked = parkLlm2 === "1"; + if ($("sa_park_llm")) { + $("sa_park_llm").checked = parkLlm2 === "1"; } if (model) { state.preferredModel = model; @@ -7382,7 +7327,7 @@ ${patch.prompt}`; if (view === "cards") { view = "chat"; } - if (view === "chat" || view === "settings" || view === "train") { + if (view === "chat" || view === "settings") { state.view = view; } const drawer = localStorage.getItem(LS_CHATS_DRAWER); @@ -7397,18 +7342,18 @@ ${patch.prompt}`; } function collectUiState() { return { - pack: $2("sa_pack")?.value || defaultPackId(), - persona: $2("sa_persona")?.value || "neutral", - auto_vision: !!$2("sa_auto_vision")?.checked, - auto_apply: !!$2("sa_auto_apply")?.checked, + pack: $("sa_pack")?.value || defaultPackId(), + persona: $("sa_persona")?.value || "neutral", + auto_vision: !!$("sa_auto_vision")?.checked, + auto_apply: !!$("sa_auto_apply")?.checked, auto_generate: true, - auto_critique: !!$2("sa_auto_critique")?.checked, - auto_download: !!$2("sa_auto_download")?.checked, - park_llm: !!$2("sa_park_llm")?.checked, + auto_critique: !!$("sa_auto_critique")?.checked, + auto_download: !!$("sa_auto_download")?.checked, + park_llm: !!$("sa_park_llm")?.checked, pane_width: localStorage.getItem(LS_PANE_WIDTH) || "", - embed_model: $2("sa_embed_model")?.value || state.preferredEmbed || "", - base_url: $2("sa_base_url")?.value || "", - model: $2("sa_model")?.value || "", + embed_model: $("sa_embed_model")?.value || state.preferredEmbed || "", + base_url: $("sa_base_url")?.value || "", + model: $("sa_model")?.value || "", view: state.view || "chat", board_tab: state.boardTab || "generate", chats_drawer: state.chatsDrawerOpen ? "1" : "0" @@ -7436,8 +7381,8 @@ ${patch.prompt}`; apply?.(String(value)); }; fill(LS_BASE, ui.base_url, (v) => { - if ($2("sa_base_url")) { - $2("sa_base_url").value = v; + if ($("sa_base_url")) { + $("sa_base_url").value = v; } }); fill(LS_MODEL, ui.model, (v) => { @@ -7447,20 +7392,20 @@ ${patch.prompt}`; state.preferredEmbed = v; }); fill(LS_PACK, ui.pack, (v) => { - if ($2("sa_pack")) { - $2("sa_pack").value = v; + if ($("sa_pack")) { + $("sa_pack").value = v; } }); fill(LS_PERSONA, ui.persona, (v) => { - if ($2("sa_persona")) { - $2("sa_persona").value = v; + if ($("sa_persona")) { + $("sa_persona").value = v; } }); fill(LS_PANE_WIDTH, ui.pane_width, (v) => document.documentElement.style.setProperty("--sa-image-width", v)); if (ui.view === "cards") { ui.view = "chat"; } - if (ui.view === "chat" || ui.view === "settings" || ui.view === "train") { + if (ui.view === "chat" || ui.view === "settings") { fill(LS_VIEW, ui.view, (v) => { state.view = v; }); @@ -7492,7 +7437,7 @@ ${patch.prompt}`; continue; } localStorage.setItem(lsKey, on ? "1" : "0"); - const el = $2(id); + const el = $(id); if (el) { el.checked = on; } @@ -7502,18 +7447,18 @@ ${patch.prompt}`; diskPersist()?.saveUiState(collectUiState()); } function saveSettings() { - localStorage.setItem(LS_BASE, $2("sa_base_url")?.value || ""); - localStorage.setItem(LS_MODEL, $2("sa_model")?.value || ""); - localStorage.setItem(LS_EMBED, $2("sa_embed_model")?.value || state.preferredEmbed || ""); - localStorage.setItem(LS_PACK, $2("sa_pack")?.value || defaultPackId()); - localStorage.setItem(LS_PERSONA, $2("sa_persona")?.value || "neutral"); + localStorage.setItem(LS_BASE, $("sa_base_url")?.value || ""); + localStorage.setItem(LS_MODEL, $("sa_model")?.value || ""); + localStorage.setItem(LS_EMBED, $("sa_embed_model")?.value || state.preferredEmbed || ""); + localStorage.setItem(LS_PACK, $("sa_pack")?.value || defaultPackId()); + localStorage.setItem(LS_PERSONA, $("sa_persona")?.value || "neutral"); localStorage.setItem(LS_VIEW, state.view || "chat"); - localStorage.setItem(LS_AUTO_VISION, $2("sa_auto_vision")?.checked ? "1" : "0"); - localStorage.setItem(LS_AUTO_APPLY, $2("sa_auto_apply")?.checked ? "1" : "0"); - localStorage.setItem(LS_AUTO_GENERATE, $2("sa_auto_generate")?.checked ? "1" : "0"); - localStorage.setItem(LS_AUTO_CRITIQUE, $2("sa_auto_critique")?.checked ? "1" : "0"); - localStorage.setItem(LS_AUTO_DOWNLOAD, $2("sa_auto_download")?.checked ? "1" : "0"); - localStorage.setItem(LS_PARK_LLM, $2("sa_park_llm")?.checked ? "1" : "0"); + localStorage.setItem(LS_AUTO_VISION, $("sa_auto_vision")?.checked ? "1" : "0"); + localStorage.setItem(LS_AUTO_APPLY, $("sa_auto_apply")?.checked ? "1" : "0"); + localStorage.setItem(LS_AUTO_GENERATE, $("sa_auto_generate")?.checked ? "1" : "0"); + localStorage.setItem(LS_AUTO_CRITIQUE, $("sa_auto_critique")?.checked ? "1" : "0"); + localStorage.setItem(LS_AUTO_DOWNLOAD, $("sa_auto_download")?.checked ? "1" : "0"); + localStorage.setItem(LS_PARK_LLM, $("sa_park_llm")?.checked ? "1" : "0"); persistServerSettings(); saveUiStateToDisk(); } @@ -7525,10 +7470,10 @@ ${patch.prompt}`; document.querySelectorAll("#sa_skills_box input[data-skill]")?.forEach((el) => { skills[el.getAttribute("data-skill")] = !!el.checked; }); - const persona = $2("sa_persona")?.value || "neutral"; + const persona = $("sa_persona")?.value || "neutral"; const settings = { - embed_model: $2("sa_embed_model")?.value || state.preferredEmbed || "", - base_url: $2("sa_base_url")?.value || "", + embed_model: $("sa_embed_model")?.value || state.preferredEmbed || "", + base_url: $("sa_base_url")?.value || "", [persona]: { skills } }; genericRequest("AssistentSaveSettings", { settings }, () => { @@ -7539,12 +7484,9 @@ ${patch.prompt}`; if (!data || data.error) { return; } - const prevPersona = state.config?.persona || $2("sa_persona")?.value || ""; + const prevPersona = state.config?.persona || $("sa_persona")?.value || ""; const prevControls = state.config?.control_values && typeof state.config.control_values === "object" ? { ...state.config.control_values } : null; state.config = data; - if (window.SA?.training?.onConfig) { - window.SA.training.onConfig(data); - } if (window.SA?.applyConfigPatchKeys) { window.SA.applyConfigPatchKeys(data); } @@ -7601,8 +7543,8 @@ ${data.ui.help_extra}`.trim(); renderPackOptions(data.packs || [], applyDefaults ? data.assistant?.default_pack : null); renderChips(data.ui?.chips || []); renderSkillChecks(data.skills || [], state.enabledSkills); - if (applyDefaults && data.assistant?.default_pack && $2("sa_pack") && !localStorage.getItem(LS_PACK)) { - $2("sa_pack").value = data.assistant.default_pack; + if (applyDefaults && data.assistant?.default_pack && $("sa_pack") && !localStorage.getItem(LS_PACK)) { + $("sa_pack").value = data.assistant.default_pack; } if (data.assistant?.embed_model && !state.preferredEmbed) { state.preferredEmbed = data.assistant.embed_model; @@ -7645,7 +7587,7 @@ ${data.ui.help_extra}`.trim(); lastExactForceCkpt = resolveCurrentCheckpoint()?.name || null; } } - const nextPersona = data.persona || $2("sa_persona")?.value || ""; + const nextPersona = data.persona || $("sa_persona")?.value || ""; let controlValues = data.control_values || data.exact?.controls || {}; if (!applyDefaults && prevControls && nextPersona === prevPersona) { controlValues = { ...controlValues, ...prevControls }; @@ -7655,14 +7597,14 @@ ${data.ui.help_extra}`.trim(); } } renderPersonaControls(data.controls || {}, controlValues); - syncPersonaDeleteButton(data.persona_source || data.personas?.find((p) => p.id === (data.persona || $2("sa_persona")?.value))?.source); + syncPersonaDeleteButton(data.persona_source || data.personas?.find((p) => p.id === (data.persona || $("sa_persona")?.value))?.source); } function isDeletablePersonaSource(source) { const src = String(source || ""); return src === "overlay" || src === "overlay+bundled" || src === "overlay+pack"; } function syncPersonaDeleteButton(source) { - const btn = $2("sa_persona_delete"); + const btn = $("sa_persona_delete"); if (!btn) { return; } @@ -7674,7 +7616,7 @@ ${data.ui.help_extra}`.trim(); let controlsPointerDown = false; let pendingControlsRender = null; function renderPersonaControls(schema, values) { - const box = $2("sa_persona_controls"); + const box = $("sa_persona_controls"); if (!box) { return; } @@ -7829,7 +7771,7 @@ ${data.ui.help_extra}`.trim(); setStatus("/horny-game\u2026"); } function savePersonaControls(partial) { - const persona = $2("sa_persona")?.value || "neutral"; + const persona = $("sa_persona")?.value || "neutral"; if (typeof genericRequest !== "function") { return; } @@ -7864,7 +7806,7 @@ ${data.ui.help_extra}`.trim(); ); } function syncPersonaControlInputs(values) { - const box = $2("sa_persona_controls"); + const box = $("sa_persona_controls"); if (!box || !values || typeof values !== "object") { return; } @@ -7887,7 +7829,7 @@ ${data.ui.help_extra}`.trim(); }); } async function deleteCurrentOverlayPersona() { - const id = $2("sa_persona")?.value; + const id = $("sa_persona")?.value; if (!id) { return; } @@ -7917,8 +7859,8 @@ ${data.ui.help_extra}`.trim(); state.personas = data.personas; } renderPersonaOptions(state.personas || [], next); - if ($2("sa_persona")) { - $2("sa_persona").value = next; + if ($("sa_persona")) { + $("sa_persona").value = next; } await applyPersonaForChat(next, { quiet: false }); setStatus(`\u0423\u0434\u0430\u043B\u0435\u043D\u043E: ${id}`); @@ -7933,7 +7875,7 @@ ${data.ui.help_extra}`.trim(); }); } function renderPersonaOptions(personas, selected) { - const sel = $2("sa_persona"); + const sel = $("sa_persona"); if (!sel) { return; } @@ -7955,7 +7897,7 @@ ${data.ui.help_extra}`.trim(); syncPersonaDeleteButton(meta?.source || state.config?.persona_source); } function renderPackOptions(packs, preferred) { - const sel = $2("sa_pack"); + const sel = $("sa_pack"); if (!sel) { return; } @@ -7973,7 +7915,7 @@ ${data.ui.help_extra}`.trim(); } } function renderChips(chips) { - const box = $2("sa_chips"); + const box = $("sa_chips"); if (!box || !Array.isArray(chips) || !chips.length) { return; } @@ -8012,7 +7954,7 @@ ${data.ui.help_extra}`.trim(); } } function renderSkillChecks(skills, enabled) { - const box = $2("sa_skills_box"); + const box = $("sa_skills_box"); if (!box) { return; } @@ -8042,7 +7984,7 @@ ${data.ui.help_extra}`.trim(); } genericRequest( "AssistentGetConfig", - { persona: persona || $2("sa_persona")?.value || "neutral" }, + { persona: persona || $("sa_persona")?.value || "neutral" }, (data) => { applyConfigPayload(data, { applyDefaults: true }); done?.(data); @@ -8089,8 +8031,8 @@ ${data.ui.help_extra}`.trim(); return preferred || list[0]; } function setModelOptions(models, { error, preferred } = {}) { - const sel = $2("sa_model"); - const sel2 = $2("sa_settings_chat_model"); + const sel = $("sa_model"); + const sel2 = $("sa_settings_chat_model"); const apply = (target) => { if (!target) { return; @@ -8129,7 +8071,7 @@ ${data.ui.help_extra}`.trim(); apply(sel2); } function setEmbedModelOptions(models) { - const sel = $2("sa_embed_model"); + const sel = $("sa_embed_model"); if (!sel) { return; } @@ -8160,7 +8102,7 @@ ${data.ui.help_extra}`.trim(); } } function refreshModels() { - const baseUrl = $2("sa_base_url")?.value || "http://127.0.0.1:11434"; + const baseUrl = $("sa_base_url")?.value || "http://127.0.0.1:11434"; setStatus("Loading models\u2026"); if (typeof genericRequest !== "function") { setStatus("SwarmUI API not ready"); @@ -8177,10 +8119,10 @@ ${data.ui.help_extra}`.trim(); setModelOptions(models, { preferred }); setEmbedModelOptions(memoryModels); const pick = resolveChatModel(models, preferred); - if (pick && $2("sa_model")) { - $2("sa_model").value = pick; - if ($2("sa_settings_chat_model")) { - $2("sa_settings_chat_model").value = pick; + if (pick && $("sa_model")) { + $("sa_model").value = pick; + if ($("sa_settings_chat_model")) { + $("sa_settings_chat_model").value = pick; } state.preferredModel = pick; localStorage.setItem(LS_MODEL, pick); @@ -8244,16 +8186,16 @@ ${data.ui.help_extra}`.trim(); return new Promise((resolve) => refreshInventory(resolve, opts)); } function memoryKindFilter() { - return $2("sa_mem_kind")?.value || "all"; + return $("sa_mem_kind")?.value || "all"; } function memoryScopeFilter() { - return $2("sa_mem_scope")?.value || "all"; + return $("sa_mem_scope")?.value || "all"; } function memorySearchFilter() { - return ($2("sa_mem_search")?.value || "").trim().toLowerCase(); + return ($("sa_mem_search")?.value || "").trim().toLowerCase(); } function renderMemoryKinds(kinds) { - const sel = $2("sa_mem_kind"); + const sel = $("sa_mem_kind"); if (!sel) { return; } @@ -8277,7 +8219,7 @@ ${data.ui.help_extra}`.trim(); const filter = memoryKindFilter(); const scope = memoryScopeFilter(); const q = memorySearchFilter(); - const persona = $2("sa_persona")?.value || "neutral"; + const persona = $("sa_persona")?.value || "neutral"; return (state.memoryRows || []).filter((m) => { if (filter !== "all" && m.kind !== filter) { return false; @@ -8298,7 +8240,7 @@ ${data.ui.help_extra}`.trim(); }); } function renderMemoryList() { - const root = $2("sa_mem_list"); + const root = $("sa_mem_list"); if (!root) { return; } @@ -8314,7 +8256,7 @@ ${data.ui.help_extra}`.trim(); const bundled = row.source === "bundled"; const when = row.updated ? formatChatWhen(row.updated * 1e3) : ""; const scope = row.scope === "personal" ? `\u043F\u0435\u0440\u0441\u043E\u043D\u0430 ${row.persona || "\u2014"}` : "\u043E\u0431\u0449\u0430\u044F"; - el.innerHTML = `
${escapeHtml2(row.kind || "note")}${escapeHtml2(row.key || "")}
${escapeHtml2(clipDebug(row.text, 220))}
${escapeHtml2([scope, row.source || "user", when].filter(Boolean).join(" \xB7 "))}
`; + el.innerHTML = `
${escapeHtml(row.kind || "note")}${escapeHtml(row.key || "")}
${escapeHtml(clipDebug(row.text, 220))}
${escapeHtml([scope, row.source || "user", when].filter(Boolean).join(" \xB7 "))}
`; const forget = document.createElement("button"); forget.type = "button"; forget.className = "basic-button sa-mem-forget"; @@ -8334,7 +8276,7 @@ ${data.ui.help_extra}`.trim(); if (typeof genericRequest !== "function") { return; } - const list = $2("sa_mem_list"); + const list = $("sa_mem_list"); if (list && !state.memoryRows.length) { list.innerHTML = '
\u0427\u0438\u0442\u0430\u044E \u043F\u0430\u043C\u044F\u0442\u044C\u2026
'; } @@ -8345,7 +8287,7 @@ ${data.ui.help_extra}`.trim(); state.memoryRows = Array.isArray(data?.memories) ? data.memories : []; renderMemoryKinds(data?.kinds || []); renderMemoryList(); - const foot = $2("sa_mem_total"); + const foot = $("sa_mem_total"); if (foot) { foot.textContent = `\u0412\u0441\u0435\u0433\u043E: ${data?.total ?? state.memoryRows.length} \xB7 ${data?.embed_model || "\u2014"}`; } @@ -8353,7 +8295,7 @@ ${data.ui.help_extra}`.trim(); 0, (err) => { if (list) { - list.innerHTML = `
\u041F\u0430\u043C\u044F\u0442\u044C \u043D\u0435\u0434\u043E\u0441\u0442\u0443\u043F\u043D\u0430: ${escapeHtml2(String(err || "\u043E\u0448\u0438\u0431\u043A\u0430"))}
`; + list.innerHTML = `
\u041F\u0430\u043C\u044F\u0442\u044C \u043D\u0435\u0434\u043E\u0441\u0442\u0443\u043F\u043D\u0430: ${escapeHtml(String(err || "\u043E\u0448\u0438\u0431\u043A\u0430"))}
`; } } ); @@ -8425,9 +8367,9 @@ ${data.ui.help_extra}`.trim(); } if (state.settingsTab === "models") { syncSettingsHealthLine(); - const m = $2("sa_model")?.value; - if (m && $2("sa_settings_chat_model")) { - $2("sa_settings_chat_model").value = m; + const m = $("sa_model")?.value; + if (m && $("sa_settings_chat_model")) { + $("sa_settings_chat_model").value = m; } } if (state.settingsTab === "more") { @@ -8438,7 +8380,7 @@ ${data.ui.help_extra}`.trim(); const asst = data?.assistant || state.config?.assistant || {}; const exact = data?.exact || state.config?.exact || state.exact || {}; const setNum = (id, v) => { - const el = $2(id); + const el = $(id); if (el && v != null && Number.isFinite(Number(v))) { el.value = String(v); } @@ -8447,16 +8389,16 @@ ${data.ui.help_extra}`.trim(); setNum("sa_history_keep", asst.history_keep_turns); setNum("sa_compress_at", asst.compress_at != null ? asst.compress_at : COMPRESS_AT); setNum("sa_chars_per_token", asst.chars_per_token != null ? asst.chars_per_token : CHARS_PER_TOKEN); - const autoEl = $2("sa_compress_auto"); + const autoEl = $("sa_compress_auto"); if (autoEl) { autoEl.checked = asst.compress_auto != null ? !!asst.compress_auto : COMPRESS_AUTO; } setNum("sa_memory_top_k", asst.memory_top_k); const w = asst.user_prefs_weight != null ? Number(asst.user_prefs_weight) : 1; - const weightEl = $2("sa_user_prefs_weight"); + const weightEl = $("sa_user_prefs_weight"); if (weightEl) { weightEl.value = String(Math.max(0, Math.min(1.5, w))); - const lab = $2("sa_user_prefs_weight_val"); + const lab = $("sa_user_prefs_weight_val"); if (lab) { lab.textContent = Number(weightEl.value).toFixed(1); } @@ -8475,7 +8417,7 @@ ${data.ui.help_extra}`.trim(); return; } const num = (id) => { - const v = parseFloat($2(id)?.value); + const v = parseFloat($(id)?.value); return Number.isFinite(v) ? v : null; }; const assistant = { @@ -8483,7 +8425,7 @@ ${data.ui.help_extra}`.trim(); history_keep_turns: num("sa_history_keep"), compress_at: num("sa_compress_at"), chars_per_token: num("sa_chars_per_token"), - compress_auto: $2("sa_compress_auto") ? !!$2("sa_compress_auto").checked : null, + compress_auto: $("sa_compress_auto") ? !!$("sa_compress_auto").checked : null, memory_top_k: num("sa_memory_top_k"), user_prefs_weight: num("sa_user_prefs_weight") }; @@ -8534,8 +8476,8 @@ ${data.ui.help_extra}`.trim(); ); } function syncSettingsHealthLine() { - const line = $2("sa_settings_health_line"); - const badge = $2("sa_ollama_health"); + const line = $("sa_settings_health_line"); + const badge = $("sa_ollama_health"); if (line && badge) { line.textContent = badge.textContent || "Ollama \xB7 \u2026"; line.className = "sa-settings-health " + (badge.className || "").replace("sa-health", "").trim(); @@ -8557,12 +8499,12 @@ ${data.ui.help_extra}`.trim(); return "\u0432\u0441\u0442\u0440\u043E\u0435\u043D\u043E"; } function renderPersonaSettingsList() { - const root = $2("sa_persona_list"); + const root = $("sa_persona_list"); if (!root) { return; } const list = state.personas || state.config?.personas || []; - const cur = state.settingsPersonaId || $2("sa_persona")?.value || list[0]?.id; + const cur = state.settingsPersonaId || $("sa_persona")?.value || list[0]?.id; state.settingsPersonaId = cur; root.innerHTML = ""; for (const p of list) { @@ -8570,7 +8512,7 @@ ${data.ui.help_extra}`.trim(); btn.type = "button"; btn.className = "sa-persona-item" + (p.id === cur ? " sa-persona-item-active" : ""); const accent = p.accent || "currentColor"; - btn.innerHTML = `
${escapeHtml2(p.title || p.id)}
${escapeHtml2(personaSourceLabel(p.source))}
`; + btn.innerHTML = `
${escapeHtml(p.title || p.id)}
${escapeHtml(personaSourceLabel(p.source))}
`; btn.addEventListener("click", () => { state.settingsPersonaId = p.id; renderPersonaSettingsList(); @@ -8587,17 +8529,21 @@ ${data.ui.help_extra}`.trim(); const id = state.settingsPersonaId; const p = (state.personas || []).find((x) => x.id === id); const canDelete = p && isDeletablePersonaSource(p.source); - const del = $2("sa_btn_persona_delete_panel"); + const del = $("sa_btn_persona_delete_panel"); if (del) { del.disabled = !canDelete; } } function loadPersonaPreview(id) { - const box = $2("sa_persona_preview"); + const box = $("sa_persona_preview"); + const knowPanel = $("sa_knowledge_panel"); if (!box || typeof genericRequest !== "function") { return; } box.innerHTML = '
\u0417\u0430\u0433\u0440\u0443\u0437\u043A\u0430\u2026
'; + if (knowPanel) { + knowPanel.hidden = true; + } genericRequest( "AssistentGetPersonaShelves", { persona: id }, @@ -8608,15 +8554,82 @@ ${data.ui.help_extra}`.trim(); ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; syncPersonaPanelActions(); + loadPersonaKnowledge(id); }, 0, (err) => { - box.innerHTML = `
${escapeHtml2(String(err || "\u043E\u0448\u0438\u0431\u043A\u0430"))}
`; + box.innerHTML = `
${escapeHtml(String(err || "\u043E\u0448\u0438\u0431\u043A\u0430"))}
`; + } + ); + } + function loadPersonaKnowledge(personaId) { + const panel = $("sa_knowledge_panel"); + const list = $("sa_knowledge_list"); + const saveBtn = $("sa_btn_knowledge_save"); + const hint = $("sa_knowledge_hint"); + if (!panel || !list || typeof genericRequest !== "function") { + return; + } + list.innerHTML = '
\u0417\u0430\u0433\u0440\u0443\u0437\u043A\u0430 \u043A\u043D\u0438\u0433\u2026
'; + panel.hidden = false; + genericRequest( + "AssistentListKnowledgeCatalog", + { persona: personaId }, + (data) => { + const cat = data?.knowledge || {}; + const attach = new Set((cat.attach || []).map(String)); + const books = cat.books || []; + const editable = !!data?.editable; + if (hint) { + hint.textContent = editable ? "\u041E\u0442\u043C\u0435\u0442\u044C \u043A\u043D\u0438\u0433\u0438 \u0434\u043B\u044F FTS-\u043F\u043E\u0438\u0441\u043A\u0430 (ask:knowledge). \u0421\u043E\u0445\u0440\u0430\u043D\u044F\u0435\u0442\u0441\u044F \u0432 overlay knowledge.json." : "Bundled/pack \u2014 attach \u0437\u0430\u0434\u0430\u043D \u043F\u043E \u0443\u043C\u043E\u043B\u0447\u0430\u043D\u0438\u044E \u0438\u043B\u0438 assistent-pack.yaml. \u041A\u043B\u043E\u043D\u0438\u0440\u0443\u0439 \u0432 overlay \u0434\u043B\u044F \u043F\u0440\u0430\u0432\u043E\u043A."; + } + if (saveBtn) { + saveBtn.hidden = !editable; + } + list.innerHTML = ""; + if (!books.length) { + list.innerHTML = '
\u041A\u043D\u0438\u0433\u0438 \u043D\u0435 \u043D\u0430\u0439\u0434\u0435\u043D\u044B \u043D\u0430 \u0434\u0438\u0441\u043A\u0435. gpu-rent seed-books + up.
'; + return; + } + for (const b of books) { + const row = document.createElement("label"); + row.className = "sa-check sa-knowledge-row"; + const cb = document.createElement("input"); + cb.type = "checkbox"; + cb.value = b.id; + cb.checked = attach.has(String(b.id)); + cb.disabled = !editable; + row.appendChild(cb); + const title = b.title || b.id; + const kind = b.content_kind ? ` \xB7 ${b.content_kind}` : ""; + const idx = b.indexed ? "" : " \xB7 \u043D\u0435\u0442 search.jsonl"; + row.appendChild(document.createTextNode(` ${title} (${b.id})${kind}${idx}`)); + list.appendChild(row); + } + if (saveBtn && editable) { + saveBtn.onclick = () => { + const picked = [...list.querySelectorAll("input[type=checkbox]:checked")].map((el) => el.value); + genericRequest( + "AssistentSaveKnowledgeAttach", + { persona: personaId, attach: picked }, + () => { + setStatus("Knowledge attach \u0441\u043E\u0445\u0440\u0430\u043D\u0451\u043D"); + loadPersonaKnowledge(personaId); + }, + 0, + (err) => setStatus(String(err || "save failed")) + ); + }; + } + }, + 0, + (err) => { + list.innerHTML = `
${escapeHtml(String(err || "\u043E\u0448\u0438\u0431\u043A\u0430"))}
`; } ); } function exportSelectedPersona() { - const id = state.settingsPersonaId || $2("sa_persona")?.value; + const id = state.settingsPersonaId || $("sa_persona")?.value; if (!id || typeof genericRequest !== "function") { return; } @@ -8677,7 +8690,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; reader.readAsText(file); } function cloneSelectedPersona() { - const from = state.settingsPersonaId || $2("sa_persona")?.value; + const from = state.settingsPersonaId || $("sa_persona")?.value; if (!from) { return; } @@ -8731,7 +8744,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; if (typeof genericRequest !== "function") { return; } - const persona = $2("sa_persona")?.value || "neutral"; + const persona = $("sa_persona")?.value || "neutral"; genericRequest( "AssistentListUserPrefs", { persona, limit: 200 }, @@ -8744,11 +8757,11 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; ); } function renderUserPrefsLists() { - const persona = $2("sa_persona")?.value || "neutral"; + const persona = $("sa_persona")?.value || "neutral"; const global = (state.userPrefs || []).filter((p) => p.scope === "global"); const personal = (state.userPrefs || []).filter((p) => p.scope === "persona" && (p.persona_id === persona || p.persona === persona)); const fill = (rootId, rows) => { - const root = $2(rootId); + const root = $(rootId); if (!root) { return; } @@ -8761,7 +8774,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; const el = document.createElement("div"); el.className = "sa-mem-row"; const pin = row.pinned ? " \u2605" : ""; - el.innerHTML = `
${escapeHtml2(row.key || "")}${pin}
${escapeHtml2(clipDebug(row.text, 200))}
`; + el.innerHTML = `
${escapeHtml(row.key || "")}${pin}
${escapeHtml(clipDebug(row.text, 200))}
`; el.querySelector(".sa-mem-row-body")?.addEventListener("click", () => editUserPref(row)); el.querySelector(".sa-mem-row-body")?.setAttribute("title", "\u041A\u043B\u0438\u043A \u2014 \u0440\u0435\u0434\u0430\u043A\u0442\u0438\u0440\u043E\u0432\u0430\u0442\u044C"); const pinBtn = document.createElement("button"); @@ -8801,7 +8814,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; key: row.key, text: String(text).trim(), scope: row.scope || "global", - persona: row.persona_id || row.persona || $2("sa_persona")?.value || "neutral", + persona: row.persona_id || row.persona || $("sa_persona")?.value || "neutral", source: "user", pinned: !!row.pinned }, @@ -8817,7 +8830,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; key: row.key, text: row.text, scope: row.scope || "global", - persona: row.persona_id || row.persona || $2("sa_persona")?.value || "neutral", + persona: row.persona_id || row.persona || $("sa_persona")?.value || "neutral", source: row.source || "user", pinned: !row.pinned }, @@ -8841,7 +8854,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; key: key.trim(), text: text.trim(), scope, - persona: $2("sa_persona")?.value || "neutral", + persona: $("sa_persona")?.value || "neutral", source: "user", pinned: false }, @@ -8859,7 +8872,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; { key: row.key, scope: row.scope || "global", - persona: row.persona_id || row.persona || $2("sa_persona")?.value + persona: row.persona_id || row.persona || $("sa_persona")?.value }, () => refreshUserPrefs(), 0, @@ -8873,7 +8886,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; } genericRequest( "AssistentClearUserPrefs", - { scope, persona: $2("sa_persona")?.value || "neutral" }, + { scope, persona: $("sa_persona")?.value || "neutral" }, (data) => { setStatus(`\u0423\u0434\u0430\u043B\u0435\u043D\u043E: ${data?.deleted ?? 0}`); refreshUserPrefs(); @@ -8898,7 +8911,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; } function setOllamaHealth(level, text, title) { state.ollamaHealth = level; - const el = $2("sa_ollama_health"); + const el = $("sa_ollama_health"); if (!el) { return; } @@ -8913,7 +8926,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; if (typeof genericRequest !== "function") { return; } - const baseUrl = $2("sa_base_url")?.value || "http://127.0.0.1:11434"; + const baseUrl = $("sa_base_url")?.value || "http://127.0.0.1:11434"; genericRequest( "AssistentListModels", { baseUrl }, @@ -8940,36 +8953,27 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; } if (view === "settings") { state.view = "settings"; - } else if (view === "train") { - state.view = "train"; } else { state.view = "chat"; } - const chat = $2("sa_view_chat"); - const settings = $2("sa_view_settings"); - const train = $2("sa_view_train"); + const chat = $("sa_view_chat"); + const settings = $("sa_view_settings"); if (chat) { chat.hidden = state.view !== "chat"; } if (settings) { settings.hidden = state.view !== "settings"; } - if (train) { - train.hidden = state.view !== "train"; - } const tabActive = (id, on) => { - $2(id)?.classList.toggle("sa-subtab-active", on); - $2(id)?.classList.toggle("sa-app-tab-active", on); - $2(id)?.setAttribute("aria-selected", on ? "true" : "false"); + $(id)?.classList.toggle("sa-subtab-active", on); + $(id)?.classList.toggle("sa-app-tab-active", on); + $(id)?.setAttribute("aria-selected", on ? "true" : "false"); }; tabActive("sa_tab_chat", state.view === "chat"); tabActive("sa_tab_settings", state.view === "settings"); - tabActive("sa_tab_train", state.view === "train"); saveSettings(); if (state.view === "settings") { setSettingsTab(state.settingsTab || "behavior"); - } else if (state.view === "train") { - window.SA?.training?.render?.(); } else if ((state.llmParked || state.expectColdLoad) && !state.generating) { warmLlm({ force: true }); } @@ -9020,7 +9024,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; renderLoraChips(); } function renderLoraChips() { - const root = $2("sa_lora_chips"); + const root = $("sa_lora_chips"); if (!root) { return; } @@ -9113,7 +9117,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; }; filter.addEventListener("input", draw); draw(); - const composer = $2("sa_composer") || document.body; + const composer = $("sa_composer") || document.body; composer.style.position = composer.style.position || "relative"; composer.appendChild(picker); const onDoc = (ev) => { @@ -9170,7 +9174,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; pop.remove(); } }); - const composer = $2("sa_composer") || document.body; + const composer = $("sa_composer") || document.body; composer.style.position = composer.style.position || "relative"; composer.appendChild(pop); const rect = anchor.getBoundingClientRect(); @@ -9400,8 +9404,8 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; intent, generating: !!intent.generate, swarm_prompt: val("alt_prompt_textbox") || val("input_prompt") || "", - persona: $2("sa_persona")?.value || "", - pack: $2("sa_pack")?.value || "" + persona: $("sa_persona")?.value || "", + pack: $("sa_pack")?.value || "" }); } function reportDebugClientTurn(payload) { @@ -9471,7 +9475,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; syncLiveParamsBar(); } function syncChipHighlight() { - const bar = $2("sa_chips"); + const bar = $("sa_chips"); if (!bar) { return; } @@ -9516,7 +9520,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; }); } function appendSystemNote(text) { - const box = $2("sa_messages"); + const box = $("sa_messages"); if (!box) { return; } @@ -9545,10 +9549,10 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; }).join(", "); } function buildDebugSummary() { - const persona = $2("sa_persona")?.value || "neutral"; - const pack = $2("sa_pack")?.value || defaultPackId(); - const chatModel = $2("sa_model")?.value || "\u2014"; - const embed = $2("sa_embed_model")?.value || state.preferredEmbed || "\u2014"; + const persona = $("sa_persona")?.value || "neutral"; + const pack = $("sa_pack")?.value || defaultPackId(); + const chatModel = $("sa_model")?.value || "\u2014"; + const embed = $("sa_embed_model")?.value || state.preferredEmbed || "\u2014"; const profile = detectKreaProfileName2(); const defaults = mergedGenerationDefaults(profile); const session = state.sessionExact || {}; @@ -9583,7 +9587,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; `persona=${persona} \xB7 pack=${pack}`, `chat=${chatModel} \xB7 embed=${embed}`, `skills=${(state.enabledSkills || []).join(",") || "\u2014"}`, - `auto: apply=${!!$2("sa_auto_apply")?.checked} gen=${true} vision=${!!$2("sa_auto_vision")?.checked} critique=${!!$2("sa_auto_critique")?.checked}`, + `auto: apply=${!!$("sa_auto_apply")?.checked} gen=${true} vision=${!!$("sa_auto_vision")?.checked} critique=${!!$("sa_auto_critique")?.checked}`, "", "Live SwarmUI:", ` ckpt=${ctx.checkpoint?.name || "\u2014"} \xB7 krea_profile=${ctx.krea_profile || profile}`, @@ -9716,9 +9720,9 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; } slot.attach = true; renderBoard(); - if ($2("sa_input")) { + if ($("sa_input")) { const critical = id === GEN_ID || slot.type === "generate"; - $2("sa_input").value = critical ? "Strict QC of this Generate frame vs the live prompt: concrete defects first, then JSON patch with prompt/param fixes. No empty praise." : `Critically review ${id} vs the intended scene: defects and how to fix prompt/params.`; + $("sa_input").value = critical ? "Strict QC of this Generate frame vs the live prompt: concrete defects first, then JSON patch with prompt/param fixes. No empty praise." : `Critically review ${id} vs the intended scene: defects and how to fix prompt/params.`; } setPackValue("critique_image", { flash: true }); await sendChat({ forceSlotIds: [id], skipAutoPack: true }); @@ -9810,7 +9814,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; if (!setPackValue(arg, { flash: true, user: true })) { setStatus("Pack: write|ordinary|critique|compose|params|inpaint|describe|persona"); } else { - setStatus(`Pack \u2192 ${$2("sa_pack")?.value}`); + setStatus(`Pack \u2192 ${$("sa_pack")?.value}`); } return true; } @@ -9833,7 +9837,7 @@ ${summary || "(\u043F\u0443\u0441\u0442\u043E)"}`; }); return true; } - const fromId = sub === "clone" && rest ? rest.split(/\s+/)[0] : $2("sa_persona")?.value || "neutral"; + const fromId = sub === "clone" && rest ? rest.split(/\s+/)[0] : $("sa_persona")?.value || "neutral"; await sendChat({ skipAutoPack: true, forcedUserText: `\u041D\u0430\u0447\u043D\u0438 \u0438\u043D\u0442\u0435\u0440\u0432\u044C\u044E author_persona: \u043A\u043B\u043E\u043D \u0441 \u0438\u0441\u0442\u043E\u0447\u043D\u0438\u043A\u0430 \xAB${fromId}\xBB. \u0421\u043F\u0440\u0430\u0448\u0438\u0432\u0430\u0439 \u043F\u043E \u043F\u043E\u043B\u043A\u0430\u043C \u0433\u0440\u0443\u043F\u043F\u0430\u043C\u0438. \u041D\u0435 \u043F\u0438\u0448\u0438 \u043D\u0430 \u0434\u0438\u0441\u043A, \u043F\u043E\u043A\u0430 \u043C\u0430\u043B\u043E \u043E\u0442\u0432\u0435\u0442\u043E\u0432. \u041D\u0435 \u0443\u0434\u0430\u043B\u044F\u0439 \u043B\u0438\u0447\u043D\u043E\u0441\u0442\u0438.` @@ -9879,8 +9883,8 @@ ${HELP_TEXT}`); s.attach = true; } renderBoard(); - if ($2("sa_input")) { - $2("sa_input").value = `Look at board slots: ${need.map((s) => s.id).join(", ")}. Continue using these images.`; + if ($("sa_input")) { + $("sa_input").value = `Look at board slots: ${need.map((s) => s.id).join(", ")}. Continue using these images.`; } setStatus(`Vision hop \u2190 ${need.map((s) => s.label).join(", ")}`); await sendChat({ fromVisionHop: true, forceSlotIds: need.map((s) => s.id) }); @@ -9891,7 +9895,7 @@ ${HELP_TEXT}`); setStatus("\u0418\u0434\u0451\u0442 \u0442\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0430 \u2014 \u0447\u0430\u0442 \u0437\u0430\u0431\u043B\u043E\u043A\u0438\u0440\u043E\u0432\u0430\u043D"); return; } - const rawInput = ($2("sa_input")?.value || "").trim(); + const rawInput = ($("sa_input")?.value || "").trim(); const text = (opts.forcedUserText || rawInput).trim(); if (!text) { return; @@ -9911,8 +9915,8 @@ ${HELP_TEXT}`); } if (!isMachineTurn(opts) && !opts.skipSlash) { if (rawInput.startsWith("/")) { - if ($2("sa_input")) { - $2("sa_input").value = ""; + if ($("sa_input")) { + $("sa_input").value = ""; } const handled = await handleSlashCommand(rawInput); if (handled) { @@ -9923,8 +9927,8 @@ ${HELP_TEXT}`); if (!isMachineTurn(opts) && isSameButAspectRequest(text)) { const aspect = parseAspectFromUserText(text); if (aspect) { - if ($2("sa_input")) { - $2("sa_input").value = ""; + if ($("sa_input")) { + $("sa_input").value = ""; } appendMessage("user", text); state.history.push({ role: "user", content: text }); @@ -9955,9 +9959,9 @@ ${HELP_TEXT}`); if (!opts.fromDebug && false) { setPackValue("ordinary", { flash: false }); } - const pack = opts.fromDebug ? "debug_explain" : $2("sa_pack")?.value || defaultPackId(); - const persona = $2("sa_persona")?.value || localStorage.getItem(LS_PERSONA) || "neutral"; - const model = $2("sa_model")?.value; + const pack = opts.fromDebug ? "debug_explain" : $("sa_pack")?.value || defaultPackId(); + const persona = $("sa_persona")?.value || localStorage.getItem(LS_PERSONA) || "neutral"; + const model = $("sa_model")?.value; if (!model) { setStatus("\u0412\u044B\u0431\u0435\u0440\u0438 \u043C\u043E\u0434\u0435\u043B\u044C Ollama \u0432 \u2699"); refreshModels(); @@ -10033,8 +10037,8 @@ ${HELP_TEXT}`); state.pendingPersonaNote = null; } appendMessage("user", opts.historyUserText || text); - if ($2("sa_input")) { - $2("sa_input").value = ""; + if ($("sa_input")) { + $("sa_input").value = ""; } persistHistory(); } else { @@ -10097,7 +10101,7 @@ ${HELP_TEXT}`); messages[messages.length - 1].images = images; } startBusyUi("thinking"); - const baseUrl = $2("sa_base_url")?.value || "http://127.0.0.1:11434"; + const baseUrl = $("sa_base_url")?.value || "http://127.0.0.1:11434"; const payload = { baseUrl, model, @@ -10107,7 +10111,7 @@ ${HELP_TEXT}`); messages, context_json: JSON.stringify(context), skills: opts.fromDebug ? [] : state.enabledSkills || [], - embed_model: $2("sa_embed_model")?.value || state.preferredEmbed || "" + embed_model: $("sa_embed_model")?.value || state.preferredEmbed || "" }; const finishOk = async (reply, civitaiResults, meta = {}) => { if (chatEpoch !== state.chatEpoch) { @@ -10157,6 +10161,14 @@ ${HELP_TEXT}`); }); } } + if (meta.knowledge?.hops?.length) { + const hopLabels = meta.knowledge.hops.map((h) => `${h.tool || "hop"}:${h.count ?? "?"}`).slice(0, 6); + activityDone("knowledge", { + kind: "ask", + label: "knowledge hops", + detail: hopLabels.join(" \xB7 ") + }); + } captureActivityTrace(); state.history.push({ role: "assistant", @@ -10265,7 +10277,8 @@ ${HELP_TEXT}`); finishOk(reply, civitai, { system_chars: data.system_chars, system_layers: data.system_layers, - prompt_eval_count: data.prompt_eval_count ?? data.raw?.prompt_eval_count + prompt_eval_count: data.prompt_eval_count ?? data.raw?.prompt_eval_count, + knowledge: data.knowledge }); } }, @@ -10296,7 +10309,8 @@ ${HELP_TEXT}`); finishOk(reply, data.civitai_results || [], { system_chars: data.system_chars, system_layers: data.system_layers, - prompt_eval_count: data.prompt_eval_count ?? data.raw?.prompt_eval_count + prompt_eval_count: data.prompt_eval_count ?? data.raw?.prompt_eval_count, + knowledge: data.knowledge }); }, 0, @@ -10322,7 +10336,8 @@ ${HELP_TEXT}`); finishOk(reply, data.civitai_results || [], { system_chars: data.system_chars, system_layers: data.system_layers, - prompt_eval_count: data.prompt_eval_count ?? data.raw?.prompt_eval_count + prompt_eval_count: data.prompt_eval_count ?? data.raw?.prompt_eval_count, + knowledge: data.knowledge }); }, 0, @@ -10330,8 +10345,8 @@ ${HELP_TEXT}`); ); } function wireDropZone() { - const board = $2("sa_board"); - const layout = $2("sa_layout"); + const board = $("sa_board"); + const layout = $("sa_layout"); layout?.addEventListener("dragover", (e) => { if (e.dataTransfer?.types?.includes("Files") || e.dataTransfer?.types?.includes("text/uri-list")) { e.preventDefault(); @@ -10418,9 +10433,9 @@ ${HELP_TEXT}`); }); } function wireSplitter() { - const splitter = $2("sa_splitter"); - const layout = $2("sa_layout"); - const pane = $2("sa_image_pane"); + const splitter = $("sa_splitter"); + const layout = $("sa_layout"); + const pane = $("sa_image_pane"); if (!splitter || !layout || !pane) { return; } @@ -10471,7 +10486,7 @@ ${HELP_TEXT}`); note: "Image sent to Assistent", preferSelected: false }); - const pack = $2("sa_pack"); + const pack = $("sa_pack"); if (pack && (pack.value === "ordinary" || pack.value === "write_prompt")) { pack.value = "critique_image"; saveSettings(); @@ -10503,7 +10518,7 @@ ${HELP_TEXT}`); probeOllamaHealth(); } function wire() { - if (!$2("swarm_assistent_root")) { + if (!$("swarm_assistent_root")) { return; } if (typeof genericRequest !== "function") { @@ -10516,7 +10531,6 @@ ${HELP_TEXT}`); window.__swarmAssistentWired = true; loadSettings(); setView(state.view || "chat"); - void window.SA?.training?.resumePolling?.(); updateGate(); ensureBoard(); setBoardTab(state.boardTab || "generate", { persist: false }); @@ -10530,28 +10544,28 @@ ${HELP_TEXT}`); wireSplitter(); registerSendButton(); wireSlashInput(); - $2("sa_btn_new_chat")?.addEventListener("click", (e) => { + $("sa_btn_new_chat")?.addEventListener("click", (e) => { e.stopPropagation(); startNewChat({ saveCurrent: true, openDrawer: true }); }); - $2("sa_btn_new_chat_bar")?.addEventListener("click", (e) => { + $("sa_btn_new_chat_bar")?.addEventListener("click", (e) => { e.stopPropagation(); startNewChat({ saveCurrent: true, openDrawer: true }); }); - $2("sa_ctx_chip")?.addEventListener("click", (e) => { + $("sa_ctx_chip")?.addEventListener("click", (e) => { e.stopPropagation(); toggleCtxPanel(); }); - $2("sa_ctx_close")?.addEventListener("click", (e) => { + $("sa_ctx_close")?.addEventListener("click", (e) => { e.stopPropagation(); toggleCtxPanel(false); }); - $2("sa_ctx_panel")?.addEventListener("click", (e) => e.stopPropagation()); - $2("sa_ctx_compress")?.addEventListener("click", () => compressNowFromUi()); - $2("sa_ctx_reset")?.addEventListener("click", () => resetCompressionFromUi()); - $2("sa_ctx_auto")?.addEventListener("change", () => { - COMPRESS_AUTO = !!$2("sa_ctx_auto")?.checked; - const settingsAuto = $2("sa_compress_auto"); + $("sa_ctx_panel")?.addEventListener("click", (e) => e.stopPropagation()); + $("sa_ctx_compress")?.addEventListener("click", () => compressNowFromUi()); + $("sa_ctx_reset")?.addEventListener("click", () => resetCompressionFromUi()); + $("sa_ctx_auto")?.addEventListener("change", () => { + COMPRESS_AUTO = !!$("sa_ctx_auto")?.checked; + const settingsAuto = $("sa_compress_auto"); if (settingsAuto) { settingsAuto.checked = COMPRESS_AUTO; } @@ -10560,20 +10574,20 @@ ${HELP_TEXT}`); } setStatus(COMPRESS_AUTO ? "\u0410\u0432\u0442\u043E\u0441\u0436\u0430\u0442\u0438\u0435 \u0432\u043A\u043B\u044E\u0447\u0435\u043D\u043E" : "\u0410\u0432\u0442\u043E\u0441\u0436\u0430\u0442\u0438\u0435 \u0432\u044B\u043A\u043B\u044E\u0447\u0435\u043D\u043E"); }); - $2("sa_compress_auto")?.addEventListener("change", () => { - COMPRESS_AUTO = !!$2("sa_compress_auto")?.checked; - const panelAuto = $2("sa_ctx_auto"); + $("sa_compress_auto")?.addEventListener("change", () => { + COMPRESS_AUTO = !!$("sa_compress_auto")?.checked; + const panelAuto = $("sa_ctx_auto"); if (panelAuto) { panelAuto.checked = COMPRESS_AUTO; } }); - $2("sa_btn_chats")?.addEventListener("click", (e) => { + $("sa_btn_chats")?.addEventListener("click", (e) => { e.stopPropagation(); setChatsPanelOpen(!state.chatsPanelOpen); }); - $2("sa_btn_chats_close")?.addEventListener("click", () => setChatsPanelOpen(false)); - $2("sa_chats_panel")?.addEventListener("click", (e) => e.stopPropagation()); - $2("sa_chats_list")?.addEventListener("click", (e) => { + $("sa_btn_chats_close")?.addEventListener("click", () => setChatsPanelOpen(false)); + $("sa_chats_panel")?.addEventListener("click", (e) => e.stopPropagation()); + $("sa_chats_list")?.addEventListener("click", (e) => { const row = e.target.closest(".sa-chat-row"); if (!row) { return; @@ -10589,8 +10603,8 @@ ${HELP_TEXT}`); switchToChat(id); }); let chatsSearchTimer = null; - $2("sa_chats_search")?.addEventListener("input", () => { - const q = ($2("sa_chats_search")?.value || "").trim(); + $("sa_chats_search")?.addEventListener("input", () => { + const q = ($("sa_chats_search")?.value || "").trim(); state.chatsQuery = q; if (!q) { state.chatsSearchHits = null; @@ -10611,75 +10625,74 @@ ${HELP_TEXT}`); } }, 220); }); - $2("sa_tab_chat")?.addEventListener("click", () => setView("chat")); - $2("sa_tab_train")?.addEventListener("click", () => setView("train")); - $2("sa_tab_settings")?.addEventListener("click", () => openSettings(state.settingsTab || "behavior")); - $2("sa_board_tab_gen")?.addEventListener("click", () => setBoardTab("generate")); - $2("sa_board_tab_refs")?.addEventListener("click", () => setBoardTab("refs")); - $2("sa_persona")?.addEventListener("change", onPersonaChanged); - $2("sa_persona_delete")?.addEventListener("click", () => deleteCurrentOverlayPersona()); + $("sa_tab_chat")?.addEventListener("click", () => setView("chat")); + $("sa_tab_settings")?.addEventListener("click", () => openSettings(state.settingsTab || "behavior")); + $("sa_board_tab_gen")?.addEventListener("click", () => setBoardTab("generate")); + $("sa_board_tab_refs")?.addEventListener("click", () => setBoardTab("refs")); + $("sa_persona")?.addEventListener("change", onPersonaChanged); + $("sa_persona_delete")?.addEventListener("click", () => deleteCurrentOverlayPersona()); document.querySelectorAll("#sa_settings .sa-stab").forEach((btn) => { btn.addEventListener("click", () => setSettingsTab(btn.getAttribute("data-stab"))); }); - $2("sa_btn_mem_refresh")?.addEventListener("click", () => { + $("sa_btn_mem_refresh")?.addEventListener("click", () => { refreshMemoryList(); }); - $2("sa_mem_kind")?.addEventListener("change", renderMemoryList); - $2("sa_mem_scope")?.addEventListener("change", renderMemoryList); - $2("sa_mem_search")?.addEventListener("input", () => renderMemoryList()); - $2("sa_btn_mem_clear_kind")?.addEventListener("click", () => { + $("sa_mem_kind")?.addEventListener("change", renderMemoryList); + $("sa_mem_scope")?.addEventListener("change", renderMemoryList); + $("sa_mem_search")?.addEventListener("input", () => renderMemoryList()); + $("sa_btn_mem_clear_kind")?.addEventListener("click", () => { const kind = memoryKindFilter(); clearCraftMemory({ kind: kind === "all" ? "" : kind, label: kind === "all" ? "\u0432\u0435\u0441\u044C \u043A\u0440\u0430\u0444\u0442 (\u0444\u0438\u043B\u044C\u0442\u0440 \u0442\u0438\u043F\u0430)" : `\u0442\u0438\u043F ${kind}` }); }); - $2("sa_btn_mem_clear_persona")?.addEventListener("click", () => { - clearCraftMemory({ scope: "personal", persona: $2("sa_persona")?.value || "neutral", label: "\u043A\u0440\u0430\u0444\u0442 \u044D\u0442\u043E\u0439 \u043B\u0438\u0447\u043D\u043E\u0441\u0442\u0438" }); + $("sa_btn_mem_clear_persona")?.addEventListener("click", () => { + clearCraftMemory({ scope: "personal", persona: $("sa_persona")?.value || "neutral", label: "\u043A\u0440\u0430\u0444\u0442 \u044D\u0442\u043E\u0439 \u043B\u0438\u0447\u043D\u043E\u0441\u0442\u0438" }); }); - $2("sa_btn_mem_clear_shared")?.addEventListener("click", () => { + $("sa_btn_mem_clear_shared")?.addEventListener("click", () => { clearCraftMemory({ scope: "shared", label: "\u043E\u0431\u0449\u0443\u044E \u043A\u0440\u0430\u0444\u0442-\u043F\u0430\u043C\u044F\u0442\u044C" }); }); - $2("sa_btn_mem_clear_all")?.addEventListener("click", () => { + $("sa_btn_mem_clear_all")?.addEventListener("click", () => { clearCraftMemory({ label: "\u0432\u0435\u0441\u044C \u043A\u0440\u0430\u0444\u0442 (non-bundled)" }); }); - $2("sa_btn_prefs_refresh")?.addEventListener("click", () => refreshUserPrefs()); - $2("sa_btn_pref_add_global")?.addEventListener("click", () => addUserPref("global")); - $2("sa_btn_pref_add_persona")?.addEventListener("click", () => addUserPref("persona")); - $2("sa_btn_prefs_clear_global")?.addEventListener("click", () => clearUserPrefs("global")); - $2("sa_btn_prefs_clear_persona")?.addEventListener("click", () => clearUserPrefs("persona")); - $2("sa_btn_prefs_clear_all")?.addEventListener("click", () => clearUserPrefs("all")); - $2("sa_user_prefs_weight")?.addEventListener("input", () => { - const lab = $2("sa_user_prefs_weight_val"); + $("sa_btn_prefs_refresh")?.addEventListener("click", () => refreshUserPrefs()); + $("sa_btn_pref_add_global")?.addEventListener("click", () => addUserPref("global")); + $("sa_btn_pref_add_persona")?.addEventListener("click", () => addUserPref("persona")); + $("sa_btn_prefs_clear_global")?.addEventListener("click", () => clearUserPrefs("global")); + $("sa_btn_prefs_clear_persona")?.addEventListener("click", () => clearUserPrefs("persona")); + $("sa_btn_prefs_clear_all")?.addEventListener("click", () => clearUserPrefs("all")); + $("sa_user_prefs_weight")?.addEventListener("input", () => { + const lab = $("sa_user_prefs_weight_val"); if (lab) { - lab.textContent = Number($2("sa_user_prefs_weight").value).toFixed(1); + lab.textContent = Number($("sa_user_prefs_weight").value).toFixed(1); } }); - $2("sa_user_prefs_weight")?.addEventListener("change", () => saveKnobs()); - $2("sa_memory_top_k")?.addEventListener("change", () => saveKnobs()); - $2("sa_btn_knobs_save")?.addEventListener("click", () => saveKnobs()); - $2("sa_btn_reset_ui")?.addEventListener("click", () => resetUiState()); - $2("sa_btn_persona_export")?.addEventListener("click", () => exportSelectedPersona()); - $2("sa_btn_persona_import")?.addEventListener("click", () => $2("sa_persona_import_file")?.click()); - $2("sa_persona_import_file")?.addEventListener("change", (e) => { + $("sa_user_prefs_weight")?.addEventListener("change", () => saveKnobs()); + $("sa_memory_top_k")?.addEventListener("change", () => saveKnobs()); + $("sa_btn_knobs_save")?.addEventListener("click", () => saveKnobs()); + $("sa_btn_reset_ui")?.addEventListener("click", () => resetUiState()); + $("sa_btn_persona_export")?.addEventListener("click", () => exportSelectedPersona()); + $("sa_btn_persona_import")?.addEventListener("click", () => $("sa_persona_import_file")?.click()); + $("sa_persona_import_file")?.addEventListener("change", (e) => { const file = e.target?.files?.[0]; if (file) { importPersonaFile(file); } e.target.value = ""; }); - $2("sa_btn_persona_clone")?.addEventListener("click", () => cloneSelectedPersona()); - $2("sa_btn_persona_delete_panel")?.addEventListener("click", () => deleteSelectedOverlayPersona()); - $2("sa_btn_settings_health")?.addEventListener("click", () => { + $("sa_btn_persona_clone")?.addEventListener("click", () => cloneSelectedPersona()); + $("sa_btn_persona_delete_panel")?.addEventListener("click", () => deleteSelectedOverlayPersona()); + $("sa_btn_settings_health")?.addEventListener("click", () => { probeOllamaHealth(); setTimeout(syncSettingsHealthLine, 400); }); - $2("sa_settings_chat_model")?.addEventListener("change", () => { - const v = $2("sa_settings_chat_model")?.value; - if (v && $2("sa_model")) { - $2("sa_model").value = v; + $("sa_settings_chat_model")?.addEventListener("change", () => { + const v = $("sa_settings_chat_model")?.value; + if (v && $("sa_model")) { + $("sa_model").value = v; saveSettings(); } }); - $2("sa_btn_look_result")?.addEventListener("click", () => askLookAtResult()); - $2("sa_ollama_health")?.addEventListener("click", () => probeOllamaHealth()); + $("sa_btn_look_result")?.addEventListener("click", () => askLookAtResult()); + $("sa_ollama_health")?.addEventListener("click", () => probeOllamaHealth()); document.addEventListener("keydown", (e) => { if (state.lightboxIndex >= 0) { if (e.key === "Escape") { @@ -10710,7 +10723,7 @@ ${HELP_TEXT}`); setChatsPanelOpen(false); closed = true; } - const slash = $2("sa_slash_menu"); + const slash = $("sa_slash_menu"); if (slash && !slash.hidden) { slash.hidden = true; closed = true; @@ -10721,22 +10734,22 @@ ${HELP_TEXT}`); } }); document.getElementById(TAB_BUTTON_ID)?.addEventListener("click", () => { - setTimeout(() => $2("sa_input")?.focus(), 80); + setTimeout(() => $("sa_input")?.focus(), 80); }); - $2("sa_btn_refresh_models")?.addEventListener("click", () => { + $("sa_btn_refresh_models")?.addEventListener("click", () => { saveSettings(); refreshModels(); probeOllamaHealth(); }); - $2("sa_btn_refresh_inventory")?.addEventListener("click", () => refreshInventory(() => { + $("sa_btn_refresh_inventory")?.addEventListener("click", () => refreshInventory(() => { renderLoraChips(); }, { rescan: true })); - $2("sa_btn_add_ref")?.addEventListener("click", () => { + $("sa_btn_add_ref")?.addEventListener("click", () => { setBoardTab("refs"); addRefSlot({ select: true }); }); - $2("sa_btn_use_current")?.addEventListener("click", () => snapshotGenerateToRef()); - $2("sa_btn_as_init")?.addEventListener("click", async () => { + $("sa_btn_use_current")?.addEventListener("click", () => snapshotGenerateToRef()); + $("sa_btn_as_init")?.addEventListener("click", async () => { closeAllMoreMenus(); const src = selectedSrc() || findCurrentGenerateSrc(); if (!src) { @@ -10744,12 +10757,12 @@ ${HELP_TEXT}`); return; } await setInitFromSrc(src); - const pack = $2("sa_pack"); + const pack = $("sa_pack"); if (pack && (pack.value === "ordinary" || pack.value === "write_prompt")) { setPackValue("inpaint_edit", { flash: true }); } }); - $2("sa_btn_as_mask")?.addEventListener("click", async () => { + $("sa_btn_as_mask")?.addEventListener("click", async () => { closeAllMoreMenus(); const src = selectedSrc(); if (!src) { @@ -10759,38 +10772,38 @@ ${HELP_TEXT}`); await setMaskFromSrc(src); setPackValue("inpaint_edit", { flash: true }); }); - $2("sa_btn_clear_init")?.addEventListener("click", () => { + $("sa_btn_clear_init")?.addEventListener("click", () => { if (window.confirm("\u0421\u0431\u0440\u043E\u0441\u0438\u0442\u044C Init \u0438 Mask?")) { clearInitAndMask(); } closeAllMoreMenus(); }); - $2("sa_btn_clear_image")?.addEventListener("click", () => clearSlot(state.selectedSlotId)); - $2("sa_btn_board_more")?.addEventListener("click", (e) => { + $("sa_btn_clear_image")?.addEventListener("click", () => clearSlot(state.selectedSlotId)); + $("sa_btn_board_more")?.addEventListener("click", (e) => { e.stopPropagation(); toggleMoreMenu("sa_board_more_menu", "sa_btn_board_more"); }); - $2("sa_btn_send")?.addEventListener("click", () => sendChat()); - $2("sa_btn_build_gen")?.addEventListener("click", () => buildCurrentAndGenerate()); - $2("sa_btn_interrupt")?.addEventListener("click", () => { + $("sa_btn_send")?.addEventListener("click", () => sendChat()); + $("sa_btn_build_gen")?.addEventListener("click", () => buildCurrentAndGenerate()); + $("sa_btn_interrupt")?.addEventListener("click", () => { doInterruptNow(); }); - $2("sa_btn_clear")?.addEventListener("click", () => { + $("sa_btn_clear")?.addEventListener("click", () => { if (window.confirm("\u041E\u0447\u0438\u0441\u0442\u0438\u0442\u044C \u0432\u0435\u0441\u044C \u0447\u0430\u0442 Assistent?")) { clearChatHistory(); } }); - $2("sa_btn_clear_more")?.addEventListener("click", (e) => { + $("sa_btn_clear_more")?.addEventListener("click", (e) => { e.stopPropagation(); toggleMoreMenu("sa_clear_more_menu", "sa_btn_clear_more"); }); - $2("sa_btn_clear_confirm")?.addEventListener("click", () => { + $("sa_btn_clear_confirm")?.addEventListener("click", () => { closeAllMoreMenus(); if (window.confirm("\u041E\u0447\u0438\u0441\u0442\u0438\u0442\u044C \u0432\u0435\u0441\u044C \u0447\u0430\u0442 Assistent?")) { clearChatHistory(); } }); - $2("sa_btn_clear_patches")?.addEventListener("click", () => { + $("sa_btn_clear_patches")?.addEventListener("click", () => { closeAllMoreMenus(); clearPatchBlocksOnly(); }); @@ -10803,26 +10816,26 @@ ${HELP_TEXT}`); } closeAllMoreMenus(); }); - $2("sa_board_more_menu")?.addEventListener("click", (e) => e.stopPropagation()); - $2("sa_clear_more_menu")?.addEventListener("click", (e) => e.stopPropagation()); - $2("sa_base_url")?.addEventListener("change", saveSettings); - $2("sa_model")?.addEventListener("change", () => { - const v = $2("sa_model")?.value; - if (v && $2("sa_settings_chat_model")) { - $2("sa_settings_chat_model").value = v; + $("sa_board_more_menu")?.addEventListener("click", (e) => e.stopPropagation()); + $("sa_clear_more_menu")?.addEventListener("click", (e) => e.stopPropagation()); + $("sa_base_url")?.addEventListener("change", saveSettings); + $("sa_model")?.addEventListener("change", () => { + const v = $("sa_model")?.value; + if (v && $("sa_settings_chat_model")) { + $("sa_settings_chat_model").value = v; } saveSettings(); }); - $2("sa_embed_model")?.addEventListener("change", () => { - state.preferredEmbed = $2("sa_embed_model")?.value || ""; + $("sa_embed_model")?.addEventListener("change", () => { + state.preferredEmbed = $("sa_embed_model")?.value || ""; saveSettings(); }); - $2("sa_pack")?.addEventListener("change", () => { + $("sa_pack")?.addEventListener("change", () => { state.packUserTouched = true; saveSettings(); syncModeBadge(); }); - $2("sa_chips")?.addEventListener("click", async (e) => { + $("sa_chips")?.addEventListener("click", async (e) => { const btn = e.target.closest(".sa-chip"); if (!btn) { return; @@ -10866,7 +10879,7 @@ ${HELP_TEXT}`); } renderLoraChips(); }); - $2("sa_auto_vision")?.addEventListener("change", () => { + $("sa_auto_vision")?.addEventListener("change", () => { saveSettings(); const gen = generateSlot(); if (gen) { @@ -10874,11 +10887,11 @@ ${HELP_TEXT}`); renderBoard(); } }); - $2("sa_auto_apply")?.addEventListener("change", saveSettings); - $2("sa_auto_generate")?.addEventListener("change", saveSettings); - $2("sa_auto_critique")?.addEventListener("change", saveSettings); - $2("sa_auto_download")?.addEventListener("change", saveSettings); - $2("sa_park_llm")?.addEventListener("change", saveSettings); + $("sa_auto_apply")?.addEventListener("change", saveSettings); + $("sa_auto_generate")?.addEventListener("change", saveSettings); + $("sa_auto_critique")?.addEventListener("change", saveSettings); + $("sa_auto_download")?.addEventListener("change", saveSettings); + $("sa_park_llm")?.addEventListener("change", saveSettings); syncChipHighlight(); setInterval(syncChipHighlight, 2500); setInterval(renderLoraChips, 4e3); @@ -10947,14 +10960,14 @@ ${HELP_TEXT}`); }; } function wireSlashInput() { - const input = $2("sa_input"); + const input = $("sa_input"); if (!input || input.dataset.saSlashWired) { return; } input.dataset.saSlashWired = "1"; input.addEventListener("input", () => updateSlashMenuFromInput()); input.addEventListener("keydown", (e) => { - const menu = $2("sa_slash_menu"); + const menu = $("sa_slash_menu"); const open = menu && !menu.hidden; if (open) { const items = slashMatches(input.value.split(/\s/)[0] || ""); @@ -10998,897 +11011,6 @@ ${HELP_TEXT}`); } })(); - // src/training.js - var $ = (id) => document.getElementById(id); - function escapeHtml(s) { - return String(s ?? "").replace(/&/g, "&").replace(//g, ">").replace(/"/g, """); - } - var QLORA_HF_CUSTOM = "__custom__"; - function attachTraining(SA2) { - const state = { - ttab: "dataset", - samples: [], - hfResults: [], - hfSelected: null, - hfCheck: null, - hfMapping: null, - trainWs: null, - polling: null, - qloraTraining: null, - agentSettings: { enabled: true, auto_link_on_approve: true, heard_quota: 3 }, - agentLinked: 0 - }; - function setAgentHeardStats(linked) { - const el = $("sa_agent_heard_stats"); - if (el) { - el.textContent = `\u041F\u043E\u0434\u043A\u043B\u044E\u0447\u0435\u043D\u043E: ${linked ?? state.agentLinked ?? "\u2014"}`; - } - } - async function loadAgentHeardSettings() { - try { - const data = await SA2.request("AssistentGetDatasetAgentSettings", {}); - const s = data?.settings || {}; - state.agentSettings = { - enabled: s.enabled !== false, - auto_link_on_approve: s.auto_link_on_approve !== false, - heard_quota: s.heard_quota ?? 3 - }; - state.agentLinked = data?.linked ?? 0; - if ($("sa_agent_heard_enabled")) $("sa_agent_heard_enabled").checked = state.agentSettings.enabled; - if ($("sa_agent_auto_link")) $("sa_agent_auto_link").checked = state.agentSettings.auto_link_on_approve; - if ($("sa_agent_heard_quota")) $("sa_agent_heard_quota").value = String(state.agentSettings.heard_quota); - setAgentHeardStats(state.agentLinked); - } catch (e) { - const msg = String(e.message || e); - setTrainStatus(formatTrainError(msg)); - console.warn("loadAgentHeardSettings", e); - } - } - async function saveAgentHeardSettings() { - const settings = { - enabled: !!$("sa_agent_heard_enabled")?.checked, - auto_link_on_approve: !!$("sa_agent_auto_link")?.checked, - heard_quota: Math.max(0, Math.min(8, parseInt($("sa_agent_heard_quota")?.value, 10) || 3)) - }; - try { - const data = await SA2.request("AssistentSaveDatasetAgentSettings", { settings }); - state.agentSettings = data?.settings || settings; - setTrainStatus("\u041D\u0430\u0441\u0442\u0440\u043E\u0439\u043A\u0438 \xAB\u0443\u0441\u043B\u044B\u0448\u0430\u043D\u043D\u043E\u0433\u043E\xBB \u0441\u043E\u0445\u0440\u0430\u043D\u0435\u043D\u044B"); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } - } - async function syncAllToAgent() { - setTrainStatus("\u041F\u043E\u0434\u043A\u043B\u044E\u0447\u0435\u043D\u0438\u0435 \u043A \u0430\u0433\u0435\u043D\u0442\u0443\u2026"); - try { - await saveAgentHeardSettings(); - const data = await SA2.request("AssistentSyncDatasetToAgent", { approved_only: true, relink: false }); - state.agentLinked = data?.total_linked ?? state.agentLinked; - setAgentHeardStats(state.agentLinked); - setTrainStatus(`\u041F\u043E\u0434\u043A\u043B\u044E\u0447\u0435\u043D\u043E: +${data?.linked_now ?? 0}, \u0432\u0441\u0435\u0433\u043E ${data?.total_linked ?? "\u2014"}`); - await refreshSamples(); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } - } - function setTrainStatus(msg) { - const el = $("sa_train_status"); - if (el) el.textContent = msg || ""; - } - function isSqliteError(msg) { - const s = String(msg || "").toLowerCase(); - return s.includes("sqlite") || s.includes("sqlconnection") || s.includes("train_samples") || s.includes("training database unavailable"); - } - function isGenericServerError(msg) { - return /internal error occurred/i.test(String(msg || "")); - } - function formatTrainError(msg) { - const s = String(msg || ""); - if (isSqliteError(s)) { - return `${s} \u2014 ${sqliteHint()}`; - } - if (isGenericServerError(s)) { - return `${s} \u2014 \u0447\u0430\u0441\u0442\u043E SQLite/\u0438\u043C\u043F\u043E\u0440\u0442 HF: gpu-rent seed-extensions, restart SwarmUI, HF token \u0434\u043B\u044F gated.`; - } - return s; - } - function sqliteHint() { - return "\u0411\u0430\u0437\u0430 Assistent (SQLite) \u043D\u0435\u0434\u043E\u0441\u0442\u0443\u043F\u043D\u0430 \u2014 gpu-rent seed-extensions + restart SwarmUI (\u22650.15.6)."; - } - function parseJobProgress(job) { - if (!job) return null; - const raw = job.progress_json; - if (!raw) return null; - try { - return typeof raw === "string" ? JSON.parse(raw) : raw; - } catch { - return null; - } - } - function renderLastTrainJob(job) { - const box = $("sa_train_last_job"); - if (!box) return; - if (!job) { - box.hidden = true; - box.innerHTML = ""; - return; - } - const prog = parseJobProgress(job); - const status = job.status || prog?.status || "\u2014"; - const out = job.output_name || prog?.ollama?.name || "\u2014"; - const base = job.base_model || "\u2014"; - const ollama = prog?.ollama; - let ollamaLine = ""; - if (ollama?.success) { - ollamaLine = `
Ollama: ${escapeHtml(ollama.name || out)} \u2014 \u0432\u044B\u0431\u0435\u0440\u0438 \u0432 \u0448\u0430\u043F\u043A\u0435 \u0447\u0430\u0442\u0430
`; - } else if (ollama?.error) { - ollamaLine = `
Ollama: ${escapeHtml(ollama.error)}
`; - } else if (ollama?.skipped) { - ollamaLine = `
${escapeHtml(ollama.note || "\u0410\u0434\u0430\u043F\u0442\u0435\u0440 \u043D\u0430 \u0434\u0438\u0441\u043A\u0435, ollama create \u0432\u0440\u0443\u0447\u043D\u0443\u044E")}
`; - } - box.hidden = false; - box.innerHTML = ` -
\u041F\u043E\u0441\u043B\u0435\u0434\u043D\u044F\u044F \u0442\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0430 \xB7 ${escapeHtml(job.kind || "qlora")} \xB7 ${escapeHtml(status)}
-
HF base: ${escapeHtml(base)} \u2192 \u0438\u043C\u044F: ${escapeHtml(out)}
- ${ollamaLine} - ${prog?.log ? `
${escapeHtml(String(prog.log).slice(-4e3))}
` : ""}`; - } - function setTrainingTab(id) { - state.ttab = id || "dataset"; - document.querySelectorAll("#sa_training .sa-ttab").forEach((btn) => { - const on = btn.getAttribute("data-ttab") === state.ttab; - btn.classList.toggle("sa-ttab-active", on); - btn.setAttribute("aria-selected", on ? "true" : "false"); - }); - document.querySelectorAll("#sa_training .sa-tpane").forEach((pane) => { - pane.hidden = pane.getAttribute("data-tpane") !== state.ttab; - }); - if (state.ttab === "dataset") { - refreshSamples(); - loadAgentHeardSettings(); - } - if (state.ttab === "train") { - syncModelfileModels(); - syncQloraModels(); - void resumeTrainJobPolling(); - } - if (state.ttab === "models") refreshTrainModels(); - } - async function refreshSamples() { - try { - const status = $("sa_train_filter_status")?.value || "all"; - const persona = $("sa_train_filter_persona")?.value || "all"; - const data = await SA2.request("AssistentListTrainSamples", { status, persona, limit: 300 }); - if (data?.error) { - const msg = data.error; - setTrainStatus(formatTrainError(msg)); - const stats2 = $("sa_train_stats"); - if (stats2) stats2.textContent = "\u0414\u0430\u0442\u0430\u0441\u0435\u0442 \u043D\u0435\u0434\u043E\u0441\u0442\u0443\u043F\u0435\u043D (SQLite)"; - state.samples = []; - renderSamples(); - return; - } - state.samples = data?.samples || []; - const stats = $("sa_train_stats"); - if (stats) { - const appr = data?.approved ?? "\u2014"; - const draft = data?.draft ?? "\u2014"; - const total = data?.total ?? "\u2014"; - stats.textContent = `\u041E\u0434\u043E\u0431\u0440\u0435\u043D\u043E: ${appr} \xB7 \u0447\u0435\u0440\u043D\u043E\u0432\u0438\u043A\u0438: ${draft} \xB7 \u0432\u0441\u0435\u0433\u043E: ${total}`; - } - const personaSel = $("sa_train_filter_persona"); - if (personaSel && $("sa_persona")) { - const cur = personaSel.value || "all"; - personaSel.innerHTML = ''; - for (const opt of $("sa_persona").options) { - const o = document.createElement("option"); - o.value = opt.value; - o.textContent = opt.textContent; - personaSel.appendChild(o); - } - personaSel.value = cur; - } - renderSamples(); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } - } - function renderSamples() { - const root = $("sa_train_samples"); - if (!root) return; - const filter = $("sa_train_filter_status")?.value || "all"; - if (!state.samples.length) { - const hint = filter === "approved" ? "\u041F\u043E\u0434 \u0444\u0438\u043B\u044C\u0442\u0440\u043E\u043C \xAB\u041E\u0434\u043E\u0431\u0440\u0435\u043D\u043D\u044B\u0435\xBB \u043F\u0443\u0441\u0442\u043E. HF-\u0438\u043C\u043F\u043E\u0440\u0442 \u0441\u043E\u0437\u0434\u0430\u0451\u0442 \u0447\u0435\u0440\u043D\u043E\u0432\u0438\u043A\u0438 \u2014 \u043F\u0435\u0440\u0435\u043A\u043B\u044E\u0447\u0438 \u043D\u0430 \xAB\u0427\u0435\u0440\u043D\u043E\u0432\u0438\u043A\u0438\xBB \u0438\u043B\u0438 \xAB\u0412\u0441\u0435 \u0441\u0442\u0430\u0442\u0443\u0441\u044B\xBB." : filter === "draft" ? "\u041D\u0435\u0442 \u0447\u0435\u0440\u043D\u043E\u0432\u0438\u043A\u043E\u0432. \u0418\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u0443\u0439 HF \u0438\u043B\u0438 \u043E\u0442\u043C\u0435\u0442\u044C \u043F\u0440\u0438\u043C\u0435\u0440\u044B \u0432 \u0447\u0430\u0442\u0435." : "\u041D\u0435\u0442 \u043F\u0440\u0438\u043C\u0435\u0440\u043E\u0432. \u041E\u0442\u043C\u0435\u0442\u044C \u043E\u0442\u0432\u0435\u0442\u044B \u0432 \u0447\u0430\u0442\u0435 \u0438\u043B\u0438 \u0438\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u0443\u0439 \u0434\u0430\u0442\u0430\u0441\u0435\u0442 (HF / \u0444\u0430\u0439\u043B)."; - root.innerHTML = `
${hint}
`; - return; - } - root.innerHTML = ""; - for (const s of state.samples) { - const div = document.createElement("div"); - div.className = "sa-train-sample"; - div.dataset.id = s.id; - const msgs = s.messages || []; - const preview = msgs.map((m) => `${m.role}: ${(m.content || "").slice(0, 120)}`).join("\n"); - const linked = s.agent_linked ? " \xB7 \u{1F517} \u0430\u0433\u0435\u043D\u0442" : ""; - div.innerHTML = ` -
- ${escapeHtml(s.status)} - ${escapeHtml(s.source)} \xB7 ${escapeHtml(s.persona || "\u2014")} \xB7 ${escapeHtml(s.pack || "\u2014")}${linked} - - - - - -
- `; - root.appendChild(div); - } - } - async function upsertSample(patch) { - await SA2.request("AssistentUpsertTrainSample", patch); - await refreshSamples(); - } - function hfStringColumns(check) { - const cols = check?.schema?.columns; - if (Array.isArray(cols) && cols.length) return cols; - const feats = check?.features; - if (Array.isArray(feats)) { - return feats.map((f) => f?.name).filter(Boolean); - } - if (feats && typeof feats === "object") return Object.keys(feats); - return []; - } - function renderHfMappingUI(check) { - const row = $("sa_hf_mapping_row"); - if (!row) return; - const gate = check?.gate; - const schemaKind = check?.schema?.kind; - const needsMapping = gate === "mapping" || schemaKind === "fiction_tags_text"; - row.hidden = !needsMapping; - if (!needsMapping) { - state.hfMapping = null; - return; - } - const cols = hfStringColumns(check); - const userSel = $("sa_hf_user_col"); - const asstSel = $("sa_hf_asst_col"); - const presetSel = $("sa_hf_mapping_preset"); - if (userSel) { - userSel.innerHTML = cols.map((c) => ``).join(""); - if (cols.includes("tags")) userSel.value = "tags"; - else if (cols.includes("title")) userSel.value = "title"; - } - if (asstSel) { - asstSel.innerHTML = cols.map((c) => ``).join(""); - if (cols.includes("text")) asstSel.value = "text"; - else if (cols.includes("output")) asstSel.value = "output"; - } - if (schemaKind === "fiction_tags_text" && presetSel) { - presetSel.value = "fiction_tags_text"; - state.hfMapping = { kind: "fiction_tags_text", preset: "fiction_tags_text" }; - } - } - function buildHfMappingPayload() { - const preset = $("sa_hf_mapping_preset")?.value; - if (preset === "fiction_tags_text") { - return { kind: "fiction_tags_text", preset: "fiction_tags_text" }; - } - const userCol = $("sa_hf_user_col")?.value; - const asstCol = $("sa_hf_asst_col")?.value; - if (userCol && asstCol) { - return { kind: "custom", user_col: userCol, assistant_col: asstCol }; - } - return state.hfMapping; - } - function getQloraHfBase() { - const sel = $("sa_qlora_base"); - if (!sel) return ""; - if (sel.value === QLORA_HF_CUSTOM) { - return ($("sa_qlora_base_custom")?.value || "").trim(); - } - return (sel.value || "").trim(); - } - function applyQloraPresetFromSelect({ fillName = true } = {}) { - const sel = $("sa_qlora_base"); - const customRow = $("sa_qlora_base_custom_row"); - if (!sel) return; - if (sel.value === QLORA_HF_CUSTOM) { - if (customRow) customRow.hidden = false; - return; - } - if (customRow) customRow.hidden = true; - const presetJson = sel.selectedOptions[0]?.dataset?.preset; - if (!presetJson) return; - let preset; - try { - preset = JSON.parse(presetJson); - } catch { - return; - } - const nameEl = $("sa_qlora_name"); - if (fillName && nameEl && !nameEl.value.trim() && preset.default_output) { - nameEl.value = preset.default_output; - } - const ollamaSel = $("sa_qlora_ollama_base"); - if (ollamaSel && preset.ollama_hint) { - const hint = preset.ollama_hint; - if ([...ollamaSel.options].some((o) => o.value === hint)) { - ollamaSel.value = hint; - } - } - if (preset.rank != null && $("sa_qlora_rank")) { - $("sa_qlora_rank").value = preset.rank; - } - if (preset.seq_len != null && $("sa_qlora_seq")) { - $("sa_qlora_seq").value = preset.seq_len; - } - } - function populateQloraHfPresets(training) { - const sel = $("sa_qlora_base"); - if (!sel) return; - const prevBase = getQloraHfBase(); - const models = Array.isArray(training?.hf_models) ? training.hf_models : []; - sel.innerHTML = ''; - for (const m of models) { - const hfId = (m.hf_id || m.id || "").trim(); - if (!hfId) continue; - const opt = document.createElement("option"); - opt.value = hfId; - opt.textContent = m.title ? `${m.title} (${hfId})` : hfId; - opt.dataset.preset = JSON.stringify(m); - sel.appendChild(opt); - } - const customOpt = document.createElement("option"); - customOpt.value = QLORA_HF_CUSTOM; - customOpt.textContent = "\u0414\u0440\u0443\u0433\u0430\u044F (\u0432\u0432\u0435\u0441\u0442\u0438 HF id\u2026)"; - sel.appendChild(customOpt); - if (prevBase && [...sel.options].some((o) => o.value === prevBase)) { - sel.value = prevBase; - } else if (prevBase) { - sel.value = QLORA_HF_CUSTOM; - const custom = $("sa_qlora_base_custom"); - if (custom) custom.value = prevBase; - } else if (models.length) { - sel.value = (models[0].hf_id || models[0].id || "").trim(); - } - applyQloraPresetFromSelect({ fillName: !prevBase }); - } - async function ensureTrainingQloraConfig(force = false) { - if (!force && state.qloraTraining) { - return state.qloraTraining; - } - try { - const persona = $("sa_persona")?.value || ""; - const data = await SA2.request("AssistentGetConfig", { persona }); - state.qloraTraining = data?.training && typeof data.training === "object" ? data.training : { hf_models: [] }; - return state.qloraTraining; - } catch { - state.qloraTraining = state.qloraTraining || { hf_models: [] }; - return state.qloraTraining; - } - } - async function syncQloraModels() { - try { - const training = await ensureTrainingQloraConfig(); - populateQloraHfPresets(training); - const baseUrl = $("sa_base_url")?.value || localStorage.getItem("swarm_assistent_base_url") || ""; - const data = await SA2.request("AssistentListModels", { baseUrl }); - const models = data?.models || []; - const sel = $("sa_qlora_ollama_base"); - if (!sel) return; - const cur = sel.value; - sel.innerHTML = ''; - for (const m of models) { - const opt = document.createElement("option"); - opt.value = m; - opt.textContent = m; - sel.appendChild(opt); - } - if (cur) sel.value = cur; - else if ($("sa_model")?.value) sel.value = $("sa_model").value; - applyQloraPresetFromSelect({ fillName: false }); - } catch (e) { - } - } - function renderHfList() { - const root = $("sa_hf_list"); - if (!root) return; - root.innerHTML = ""; - const showAll = !!$("sa_hf_show_all")?.checked; - for (const r of state.hfResults) { - if (!showAll && r.gate === "rejected") continue; - const row = document.createElement("div"); - row.className = "sa-hf-row" + (state.hfSelected === r.id ? " sa-hf-row-active" : "") + (r.gate === "rejected" ? " sa-hf-rejected" : ""); - row.dataset.id = r.id; - const badge = r.gate === "ok" ? "ok" : r.gate === "mapping" ? "map" : "no"; - row.innerHTML = `${escapeHtml(r.gate)}${escapeHtml(r.id)}${escapeHtml(r.reason || "")}`; - root.appendChild(row); - } - } - async function searchHf() { - const q = ($("sa_hf_search")?.value || "").trim(); - setTrainStatus("\u041F\u043E\u0438\u0441\u043A\u2026"); - try { - const data = await SA2.request("AssistentSearchHfDatasets", { - q, - limit: 24, - show_all: !!$("sa_hf_show_all")?.checked - }); - state.hfResults = data?.results || []; - renderHfList(); - setTrainStatus(`\u041D\u0430\u0439\u0434\u0435\u043D\u043E: ${state.hfResults.length}`); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } - } - async function checkHfLink() { - const link = ($("sa_hf_link")?.value || "").trim(); - const status = $("sa_hf_status"); - if (!link) return; - if (status) status.textContent = "\u041F\u0440\u043E\u0432\u0435\u0440\u044F\u044E\u2026"; - try { - const data = await SA2.request("AssistentCheckHfDataset", { dataset: link }); - state.hfCheck = data; - state.hfSelected = data.id; - if (status) { - status.textContent = data.gate === "rejected" ? `\u041E\u0442\u043A\u043B\u043E\u043D\u0435\u043D\u043E: ${data.reason}` : `${data.gate}: ${data.reason || "OK"}`; - } - const preview = $("sa_hf_preview"); - if (preview) { - preview.hidden = false; - preview.textContent = JSON.stringify(data.sample_rows || data.features || data, null, 2).slice(0, 8e3); - } - const importRow = $("sa_hf_import_row"); - if (importRow) importRow.hidden = data.gate === "rejected"; - renderHfMappingUI(data); - } catch (e) { - if (status) status.textContent = String(e.message || e); - } - } - async function importHf() { - const link = ($("sa_hf_link")?.value || "").trim(); - if (!state.hfCheck?.id && link) { - await checkHfLink(); - } - if (!state.hfSelected && !state.hfCheck?.id) { - setTrainStatus("\u0421\u043D\u0430\u0447\u0430\u043B\u0430 \u043F\u0440\u043E\u0432\u0435\u0440\u044C \u043D\u0430\u0431\u043E\u0440"); - const hfSt2 = $("sa_hf_status"); - if (hfSt2) hfSt2.textContent = "\u0412\u0441\u0442\u0430\u0432\u044C \u0441\u0441\u044B\u043B\u043A\u0443 \u0438 \u043D\u0430\u0436\u043C\u0438 \xAB\u041F\u0440\u043E\u0432\u0435\u0440\u0438\u0442\u044C\xBB"; - return; - } - const id = state.hfSelected || state.hfCheck.id; - const limit = Number($("sa_hf_import_limit")?.value) || 200; - const mapping = buildHfMappingPayload(); - const btn = $("sa_btn_hf_import"); - const hfSt = $("sa_hf_status"); - const busy = limit > 400 ? `\u0418\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u0443\u044E \u0434\u043E ${limit} \u0441\u0442\u0440\u043E\u043A\u2026 (1\u20132 \u043C\u0438\u043D)` : `\u0418\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u0443\u044E \u0434\u043E ${limit}\u2026`; - setTrainStatus(busy); - if (hfSt) { - hfSt.textContent = busy; - hfSt.classList.add("sa-hf-busy"); - } - if (btn) { - btn.disabled = true; - btn.dataset.label = btn.textContent; - btn.textContent = "\u2026"; - } - try { - const data = await SA2.request("AssistentImportHfDataset", { dataset: id, limit, mapping }); - const n = Number(data?.imported) || 0; - let msg; - if (data?.runner_only) { - msg = `Runner-only: ${data.note || id} (\u0432 sqlite \u043D\u0435 \u0438\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u043E\u0432\u0430\u043B\u0438)`; - } else if (n > 0) { - const filt = $("sa_train_filter_status"); - if (filt && filt.value === "approved") { - filt.value = "draft"; - } - msg = `\u0418\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u043E\u0432\u0430\u043D\u043E: ${n} \u0447\u0435\u0440\u043D\u043E\u0432\u0438\u043A(\u043E\u0432) \u2014 \u0441\u043F\u0438\u0441\u043E\u043A \u043D\u0438\u0436\u0435 (\u0444\u0438\u043B\u044C\u0442\u0440 \u2192 \u0447\u0435\u0440\u043D\u043E\u0432\u0438\u043A\u0438)`; - } else { - msg = "\u0418\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u043E\u0432\u0430\u043D\u043E: 0 \u2014 HF token \u0432 User Settings, \u043C\u0430\u043F\u043F\u0438\u043D\u0433 \u0438\u043B\u0438 gated-\u043D\u0430\u0431\u043E\u0440"; - } - setTrainStatus(msg); - if (hfSt) hfSt.textContent = msg; - await refreshSamples(); - $("sa_train_samples")?.scrollIntoView({ behavior: "smooth", block: "nearest" }); - } catch (e) { - const err = String(e.message || e); - const show = formatTrainError(err); - setTrainStatus(show); - if (hfSt) hfSt.textContent = show; - } finally { - if (hfSt) hfSt.classList.remove("sa-hf-busy"); - if (btn) { - btn.disabled = false; - btn.textContent = btn.dataset.label || "\u0418\u043C\u043F\u043E\u0440\u0442\u0438\u0440\u043E\u0432\u0430\u0442\u044C"; - } - } - } - async function syncModelfileModels() { - try { - const baseUrl = $("sa_base_url")?.value || localStorage.getItem("swarm_assistent_base_url") || ""; - const data = await SA2.request("AssistentListModels", { baseUrl }); - const models = data?.models || []; - for (const selId of ["sa_modelfile_base"]) { - const sel = $(selId); - if (!sel) continue; - const cur = sel.value; - sel.innerHTML = ''; - for (const m of models) { - const opt = document.createElement("option"); - opt.value = m; - opt.textContent = m; - sel.appendChild(opt); - } - if (cur) sel.value = cur; - } - const personaSel = $("sa_modelfile_persona"); - if (personaSel && $("sa_persona")) { - personaSel.innerHTML = $("sa_persona").innerHTML; - personaSel.value = $("sa_persona").value || "neutral"; - } - } catch (e) { - } - } - async function createModelfile() { - const btn = $("sa_btn_modelfile_create"); - btn?.setAttribute("disabled", "disabled"); - setTrainStatus("\u0421\u043E\u0437\u0434\u0430\u044E Modelfile \u0432 Ollama\u2026"); - try { - const data = await SA2.request("AssistentCreateOllamaModel", { - base_url: $("sa_base_url")?.value, - base_model: $("sa_modelfile_base")?.value, - name: $("sa_modelfile_name")?.value, - persona: $("sa_modelfile_persona")?.value, - system: $("sa_modelfile_system")?.value, - shots: Number($("sa_modelfile_shots")?.value) || 8, - num_ctx: Number($("sa_modelfile_num_ctx")?.value) || 16384, - temperature: Number($("sa_modelfile_temp")?.value) || 0.7 - }); - setTrainStatus(`\u0413\u043E\u0442\u043E\u0432\u043E: ${data.name}`); - SA2.app?.refreshModels?.(); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } finally { - btn?.removeAttribute("disabled"); - } - } - function setTrainMode(mode) { - $("sa_train_form_modelfile").hidden = mode !== "modelfile"; - $("sa_train_form_qlora").hidden = mode !== "qlora"; - const radio = document.querySelector(`input[name="sa_train_mode"][value="${mode}"]`); - if (radio) radio.checked = true; - } - function setTrainingLock(on, text) { - const root = $("swarm_assistent_root"); - const banner = $("sa_train_banner"); - if (root) root.classList.toggle("sa-root-training-lock", !!on); - if (banner) { - banner.hidden = !on; - const t = $("sa_train_banner_text"); - if (t && text) t.textContent = text; - } - SA2.app?.setTrainingLock?.(!!on); - } - async function pollTrainJob() { - try { - const data = await SA2.request("AssistentGetTrainJob", {}); - const prog = data?.progress || (data?.job?.progress_json ? JSON.parse(data.job.progress_json) : null); - const active = data?.training_active || data?.job?.status === "running"; - const status = data?.job?.status || prog?.status; - const pct = prog?.percent; - const bannerText = active && pct != null ? `QLoRA \xB7 ${pct}%` : active ? "\u0418\u0434\u0451\u0442 QLoRA\u2026" : "\u0418\u0434\u0451\u0442 \u0442\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0430\u2026"; - setTrainingLock(active, bannerText); - const logEl = $("sa_train_log"); - const bar = $("sa_train_progress_fill"); - const box = $("sa_train_progress"); - if (prog) { - if (box) box.hidden = false; - if (bar && pct != null) bar.style.width = `${pct}%`; - if (logEl && prog.log) logEl.textContent = prog.log; - } - if (active) { - setTrainStatus(pct != null ? `QLoRA \xB7 ${pct}% \u2014 \u043F\u043E\u043B\u043D\u044B\u0439 \u043B\u043E\u0433 \u043D\u0430 \u0432\u043A\u043B\u0430\u0434\u043A\u0435 \xAB\u0422\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0430\xBB` : "QLoRA \u0437\u0430\u043F\u0443\u0449\u0435\u043D\u0430 \u2014 \u043F\u043E\u043B\u043D\u044B\u0439 \u043B\u043E\u0433 \u043D\u0430 \u0432\u043A\u043B\u0430\u0434\u043A\u0435 \xAB\u0422\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0430\xBB"); - } - if (!active) { - clearInterval(state.polling); - state.polling = null; - $("sa_btn_qlora_cancel").hidden = true; - setTrainingLock(false); - if (status === "completed" || status === "completed_with_warnings") { - const ollama = prog?.ollama; - if (ollama?.success) { - setTrainStatus(`\u0413\u043E\u0442\u043E\u0432\u043E: \u043C\u043E\u0434\u0435\u043B\u044C ${ollama.name} \u0432 Ollama \u2014 \u0432\u043A\u043B\u0430\u0434\u043A\u0430 \xAB\u041C\u043E\u0434\u0435\u043B\u0438\xBB`); - SA2.app?.refreshModels?.(); - } else if (ollama?.skipped) { - setTrainStatus(ollama.note || ollama.error || "\u0410\u0434\u0430\u043F\u0442\u0435\u0440 \u0441\u043E\u0445\u0440\u0430\u043D\u0451\u043D \u2014 \u0441\u043C. \u0432\u043A\u043B\u0430\u0434\u043A\u0443 \xAB\u041C\u043E\u0434\u0435\u043B\u0438\xBB"); - } else if (ollama?.error) { - setTrainStatus(`\u041E\u0431\u0443\u0447\u0435\u043D\u0438\u0435 OK, Ollama: ${ollama.error}`); - } else if (status === "completed_with_warnings") { - setTrainStatus("\u0417\u0430\u0432\u0435\u0440\u0448\u0435\u043D\u043E \u0441 \u043F\u0440\u0435\u0434\u0443\u043F\u0440\u0435\u0436\u0434\u0435\u043D\u0438\u044F\u043C\u0438 \u2014 \u043B\u043E\u0433 \u043D\u0430 \xAB\u041C\u043E\u0434\u0435\u043B\u0438\xBB"); - } else { - setTrainStatus("QLoRA \u0437\u0430\u0432\u0435\u0440\u0448\u0435\u043D\u043E \u2014 \u0432\u043A\u043B\u0430\u0434\u043A\u0430 \xAB\u041C\u043E\u0434\u0435\u043B\u0438\xBB"); - } - setTrainingTab("models"); - await refreshTrainModels(); - } else if (status === "failed") { - setTrainStatus(`\u041E\u0448\u0438\u0431\u043A\u0430 \u0442\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0438 (exit ${prog?.exit_code ?? "?"})`); - } - } - } catch (e) { - } - } - async function resumeTrainJobPolling() { - try { - const data = await SA2.request("AssistentGetTrainJob", {}); - const active = data?.training_active || data?.job?.status === "running"; - await pollTrainJob(); - if (!active) return; - setTrainMode("qlora"); - $("sa_btn_qlora_cancel").hidden = false; - if (state.polling) clearInterval(state.polling); - state.polling = setInterval(pollTrainJob, 1500); - } catch (e) { - } - } - async function startQlora() { - const baseModel = getQloraHfBase(); - const outputName = ($("sa_qlora_name")?.value || "").trim(); - if (!baseModel) { - setTrainStatus("\u0412\u044B\u0431\u0435\u0440\u0438\u0442\u0435 HF base model \u0438\u0437 \u0441\u043F\u0438\u0441\u043A\u0430 \u0438\u043B\u0438 \u0443\u043A\u0430\u0436\u0438\u0442\u0435 \u0441\u0432\u043E\u0439 HF id"); - $("sa_qlora_base")?.focus(); - return; - } - if (!outputName) { - setTrainStatus("\u0423\u043A\u0430\u0436\u0438\u0442\u0435 \u0438\u043C\u044F \u043C\u043E\u0434\u0435\u043B\u0438 \u0432 Ollama (\u043D\u0430\u043F\u0440\u0438\u043C\u0435\u0440 my-lora:v1)"); - $("sa_qlora_name")?.focus(); - return; - } - setTrainStatus("\u0417\u0430\u043F\u0443\u0441\u043A\u2026"); - try { - const hfDs = ($("sa_qlora_hf_dataset")?.value || "").trim(); - const mapping = hfDs ? buildHfMappingPayload() : void 0; - await SA2.request("AssistentStartTrainJob", { - base_url: $("sa_base_url")?.value, - chat_model: $("sa_model")?.value, - base_model: baseModel, - ollama_base: $("sa_qlora_ollama_base")?.value, - output_name: outputName, - rank: Number($("sa_qlora_rank")?.value) || 16, - alpha: Number($("sa_qlora_alpha")?.value) || 32, - lr: Number($("sa_qlora_lr")?.value) || 2e-4, - epochs: Number($("sa_qlora_epochs")?.value) || 3, - seq_len: Number($("sa_qlora_seq")?.value) || 2048, - max_samples: Number($("sa_qlora_max_samples")?.value) || 0, - four_bit: !!$("sa_qlora_4bit")?.checked, - hf_dataset: hfDs || void 0, - hf_mapping: mapping - }); - $("sa_btn_qlora_cancel").hidden = false; - setTrainingLock(true, "\u0418\u0434\u0451\u0442 \u0442\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0430\u2026"); - if (state.polling) clearInterval(state.polling); - state.polling = setInterval(pollTrainJob, 1500); - pollTrainJob(); - setTrainMode("qlora"); - setTrainingTab("train"); - setTrainStatus("QLoRA \u0437\u0430\u043F\u0443\u0449\u0435\u043D\u0430 \u2014 \u043F\u0440\u043E\u0433\u0440\u0435\u0441\u0441 \u043D\u0438\u0436\u0435"); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } - } - async function cancelQlora() { - try { - await SA2.request("AssistentCancelTrainJob", {}); - setTrainingLock(false); - setTrainStatus("\u041E\u0442\u043C\u0435\u043D\u0435\u043D\u043E"); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } - } - async function refreshTrainModels() { - const root = $("sa_train_models_list"); - if (!root) return; - try { - const jobData = await SA2.request("AssistentGetTrainJob", {}); - renderLastTrainJob(jobData?.last_job || jobData?.job); - const data = await SA2.request("AssistentListModels", { baseUrl: $("sa_base_url")?.value }); - const models = data?.models || []; - const lastOut = jobData?.last_job?.output_name; - root.innerHTML = models.length ? models.map((m) => { - const hit = lastOut && String(m).includes(String(lastOut).split(":")[0]); - return `
${escapeHtml(m)}${hit ? " \xB7 \u043F\u043E\u0441\u043B\u0435\u0434\u043D\u044F\u044F \u0442\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0430" : ""}
`; - }).join("") : '
\u041D\u0435\u0442 \u043C\u043E\u0434\u0435\u043B\u0435\u0439 \u0432 Ollama \u2014 \u043F\u043E\u0441\u043B\u0435 QLoRA \u043D\u0430\u0436\u043C\u0438 \xAB\u041E\u0431\u043D\u043E\u0432\u0438\u0442\u044C\xBB \u0438\u043B\u0438 \u043F\u0440\u043E\u0432\u0435\u0440\u044C \u043B\u043E\u0433 \u0442\u0440\u0435\u043D\u0438\u0440\u043E\u0432\u043A\u0438
'; - } catch (e) { - const msg = String(e.message || e); - root.innerHTML = `
${escapeHtml(isSqliteError(msg) ? sqliteHint() : msg)}
`; - } - } - async function saveRunner() { - try { - await SA2.request("AssistentSaveRunnerSettings", { - python: $("sa_runner_python")?.value, - kind: $("sa_runner_kind")?.value || "builtin", - workdir: $("sa_runner_workdir")?.value, - cmd: $("sa_runner_cmd")?.value, - gguf_script: $("sa_runner_gguf_script")?.value, - gguf_base_path: $("sa_runner_gguf_base")?.value, - gguf_cmd: $("sa_runner_gguf_cmd")?.value - }); - setTrainStatus("\u0420\u0430\u043D\u043D\u0435\u0440 \u0441\u043E\u0445\u0440\u0430\u043D\u0451\u043D"); - } catch (e) { - setTrainStatus(formatTrainError(e.message || e)); - } - } - async function loadRunner() { - try { - const data = await SA2.request("AssistentGetRunnerSettings", {}); - const s = data?.settings || {}; - if ($("sa_runner_python") && s.python) $("sa_runner_python").value = s.python; - if ($("sa_runner_kind")) $("sa_runner_kind").value = s.kind || "builtin"; - if ($("sa_runner_workdir") && s.workdir) $("sa_runner_workdir").value = s.workdir; - if ($("sa_runner_cmd") && s.cmd) $("sa_runner_cmd").value = s.cmd; - if ($("sa_runner_gguf_script") && s.gguf_script) $("sa_runner_gguf_script").value = s.gguf_script; - if ($("sa_runner_gguf_base") && s.gguf_base_path) $("sa_runner_gguf_base").value = s.gguf_base_path; - if ($("sa_runner_gguf_cmd") && s.gguf_cmd) $("sa_runner_gguf_cmd").value = s.gguf_cmd; - } catch (e) { - } - } - function wireTraining() { - if (window.__saTrainingWired) return; - window.__saTrainingWired = true; - document.querySelectorAll("#sa_training .sa-ttab").forEach((btn) => { - btn.addEventListener("click", () => setTrainingTab(btn.getAttribute("data-ttab"))); - }); - $("sa_btn_agent_sync")?.addEventListener("click", syncAllToAgent); - $("sa_agent_heard_enabled")?.addEventListener("change", saveAgentHeardSettings); - $("sa_agent_auto_link")?.addEventListener("change", saveAgentHeardSettings); - $("sa_agent_heard_quota")?.addEventListener("change", saveAgentHeardSettings); - loadAgentHeardSettings(); - $("sa_btn_train_from_chats")?.addEventListener("click", async () => { - try { - const data = await SA2.request("AssistentBuildDatasetFromChats", {}); - setTrainStatus(`\u0418\u0437 \u0447\u0430\u0442\u043E\u0432: +${data.added}`); - await refreshSamples(); - } catch (e) { - setTrainStatus(String(e.message || e)); - } - }); - $("sa_btn_train_import_file")?.addEventListener("click", () => $("sa_train_import_file")?.click()); - $("sa_train_import_file")?.addEventListener("change", async (e) => { - const file = e.target?.files?.[0]; - if (!file) return; - const text = await file.text(); - try { - const data = await SA2.request("AssistentImportDataset", { format: "auto", content: text }); - setTrainStatus(`\u0418\u043C\u043F\u043E\u0440\u0442: ${data.imported}`); - await refreshSamples(); - } catch (err) { - setTrainStatus(formatTrainError(err.message || err)); - } - e.target.value = ""; - }); - $("sa_btn_train_export")?.addEventListener("click", async () => { - try { - const data = await SA2.request("AssistentExportDataset", { status: "approved" }); - if (data.content) { - const blob = new Blob([data.content], { type: "application/jsonl" }); - const a = document.createElement("a"); - a.href = URL.createObjectURL(blob); - a.download = "assistent-dataset.jsonl"; - a.click(); - } - setTrainStatus(`\u042D\u043A\u0441\u043F\u043E\u0440\u0442: ${data.count} \u043F\u0440\u0438\u043C\u0435\u0440\u043E\u0432`); - } catch (e) { - setTrainStatus(String(e.message || e)); - } - }); - $("sa_train_filter_status")?.addEventListener("change", refreshSamples); - $("sa_train_filter_persona")?.addEventListener("change", refreshSamples); - $("sa_train_samples")?.addEventListener("click", async (e) => { - const row = e.target.closest(".sa-train-sample"); - if (!row) return; - const id = row.dataset.id; - const sample = state.samples.find((s) => s.id === id); - if (!sample) return; - if (e.target.closest("[data-approve]")) { - await upsertSample({ ...sample, status: "approved" }); - await loadAgentHeardSettings(); - } else if (e.target.closest("[data-reject]")) { - await upsertSample({ ...sample, status: "rejected" }); - await loadAgentHeardSettings(); - } else if (e.target.closest("[data-link]")) { - try { - const data = await SA2.request("AssistentLinkTrainSampleToAgent", { id }); - state.agentLinked = data?.linked ?? state.agentLinked; - setAgentHeardStats(state.agentLinked); - setTrainStatus("\u041F\u0440\u0438\u043C\u0435\u0440 \u043F\u043E\u0434\u043A\u043B\u044E\u0447\u0451\u043D \u043A \u0430\u0433\u0435\u043D\u0442\u0443"); - await refreshSamples(); - } catch (err) { - setTrainStatus(formatTrainError(err.message || err)); - } - } else if (e.target.closest("[data-unlink]")) { - try { - const data = await SA2.request("AssistentUnlinkTrainSampleFromAgent", { id }); - state.agentLinked = data?.linked ?? state.agentLinked; - setAgentHeardStats(state.agentLinked); - setTrainStatus("\u041F\u0440\u0438\u043C\u0435\u0440 \u043E\u0442\u043A\u043B\u044E\u0447\u0451\u043D \u043E\u0442 \u0430\u0433\u0435\u043D\u0442\u0430"); - await refreshSamples(); - } catch (err) { - setTrainStatus(formatTrainError(err.message || err)); - } - } else if (e.target.closest("[data-del]")) { - if (window.confirm("\u0423\u0434\u0430\u043B\u0438\u0442\u044C \u043F\u0440\u0438\u043C\u0435\u0440?")) { - await SA2.request("AssistentDeleteTrainSample", { id }); - await refreshSamples(); - } - } - }); - $("sa_btn_hf_search")?.addEventListener("click", searchHf); - $("sa_hf_show_all")?.addEventListener("change", () => { - renderHfList(); - }); - $("sa_hf_list")?.addEventListener("click", async (e) => { - const row = e.target.closest(".sa-hf-row"); - if (!row || row.classList.contains("sa-hf-rejected")) return; - state.hfSelected = row.dataset.id; - $("sa_hf_link").value = row.dataset.id; - renderHfList(); - await checkHfLink(); - }); - $("sa_btn_hf_check")?.addEventListener("click", checkHfLink); - $("sa_hf_mapping_preset")?.addEventListener("change", () => { - state.hfMapping = buildHfMappingPayload(); - }); - $("sa_btn_hf_import")?.addEventListener("click", importHf); - document.querySelectorAll('input[name="sa_train_mode"]').forEach((r) => { - r.addEventListener("change", () => setTrainMode(r.value)); - }); - $("sa_btn_modelfile_create")?.addEventListener("click", createModelfile); - $("sa_qlora_base")?.addEventListener("change", () => applyQloraPresetFromSelect()); - $("sa_btn_qlora_start")?.addEventListener("click", startQlora); - $("sa_btn_qlora_cancel")?.addEventListener("click", cancelQlora); - $("sa_btn_train_models_refresh")?.addEventListener("click", refreshTrainModels); - $("sa_btn_save_runner")?.addEventListener("click", saveRunner); - loadRunner(); - setTrainMode("modelfile"); - void resumeTrainJobPolling(); - } - SA2.training = { - render() { - wireTraining(); - setTrainingTab(state.ttab); - }, - onConfig(data) { - if (data?.training && typeof data.training === "object") { - state.qloraTraining = data.training; - if (state.ttab === "train") { - populateQloraHfPresets(state.qloraTraining); - applyQloraPresetFromSelect({ fillName: false }); - } - } - }, - resumePolling: resumeTrainJobPolling, - async curateFromChat(messages, meta) { - try { - await SA2.request("AssistentUpsertTrainSample", { - source: "chat", - chat_id: meta?.chatId, - persona: meta?.persona, - pack: meta?.pack, - status: meta?.status || "approved", - messages - }); - return true; - } catch (e) { - console.warn("curateFromChat", e); - return false; - } - }, - setTrainingLock, - pollTrainJob - }; - } - // src/main.js window.SA = window.SA || {}; window.SA.userAsksGenerate = userAsksGenerate; @@ -11908,5 +11030,4 @@ ${HELP_TEXT}`); window.SA.PATCH_KEYS = keys; } }; - attachTraining(window.SA); })(); diff --git a/AssistentChatPipeline.cs b/AssistentChatPipeline.cs index 084752e..f5d1625 100644 --- a/AssistentChatPipeline.cs +++ b/AssistentChatPipeline.cs @@ -98,6 +98,11 @@ public partial class SwarmAssistentExtension } AddLayer("skills", skillsBlock.ToString()); AddLayer("identity", Config.RenderIdentityBlock(pid)); + string knowledgeLayer = BuildKnowledgeSystemLayer(pid); + if (!string.IsNullOrWhiteSpace(knowledgeLayer)) + { + AddLayer("knowledge", knowledgeLayer); + } } if (!string.IsNullOrWhiteSpace(packName)) @@ -179,7 +184,7 @@ public partial class SwarmAssistentExtension return (ollamaMessages, layers); } - async Task<(string reply, JObject raw, JArray civitaiResults, int systemChars, JObject systemLayers)> RunChatWithHops( + async Task<(string reply, JObject raw, JArray civitaiResults, JObject knowledge, int systemChars, JObject systemLayers)> RunChatWithHops( Session session, string root, string modelName, @@ -238,6 +243,8 @@ public partial class SwarmAssistentExtension ?? 0; string reply = ""; JObject lastRaw = null; + JArray knowledgeHops = []; + JArray knowledgeResults = []; int maxHops = slimUtility ? 1 : CfgInt("max_tool_hops", MaxToolHopsFallback); HashSet hopDone = new(StringComparer.OrdinalIgnoreCase); for (int hop = 0; hop < maxHops; hop++) @@ -267,7 +274,7 @@ public partial class SwarmAssistentExtension follow = null; break; } - follow = await RunToolHop(session, pid, patch, tool, hopDone); + follow = await RunToolHop(session, pid, patch, tool, hopDone, knowledgeHops, knowledgeResults); if (follow is not null) { break; @@ -285,7 +292,9 @@ public partial class SwarmAssistentExtension messages.Add(new JObject { ["role"] = "assistant", ["content"] = assistantContent }); messages.Add(new JObject { ["role"] = "user", ["content"] = follow }); } - return (reply, lastRaw, [], systemChars, systemLayers); + JObject knowledge = BuildKnowledgeResponse(pid, knowledgeHops, knowledgeResults); + JArray civitai = CivitaiResultsShim(knowledgeResults); + return (reply, lastRaw, civitai, knowledge, systemChars, systemLayers); } static string BuildRetrieveQuery(JArray userMessages, string contextJson, string packName = null) @@ -413,7 +422,9 @@ public partial class SwarmAssistentExtension string pid, JObject patch, string tool, - HashSet hopDone) + HashSet hopDone, + JArray knowledgeHops = null, + JArray knowledgeResults = null) { if (tool == "ask_settings") { @@ -452,6 +463,35 @@ public partial class SwarmAssistentExtension + "omit ask:inventory unless you need a different query.\n```json\n" + rows.ToString(Newtonsoft.Json.Formatting.None) + "\n```"; } + if (tool == "ask_knowledge") + { + string q = patch?["knowledge_query"]?.ToString()?.Trim() + ?? patch?["example_query"]?.ToString()?.Trim() + ?? patch?["memory_query"]?.ToString()?.Trim() + ?? ""; + string rating = patch?["knowledge_rating"]?.ToString()?.Trim() + ?? patch?["example_rating"]?.ToString()?.Trim(); + string sig = "ask_knowledge:" + q.ToLowerInvariant() + ":" + (rating ?? ""); + if (!hopDone.Add(sig)) + { + return null; + } + if (string.IsNullOrWhiteSpace(q)) + { + return + "ask:knowledge needs knowledge_query (tags / scene / style). " + + "Retry with \"ask\":[\"knowledge\"], \"knowledge_query\":\"redhead stockings\"."; + } + int lim = Config.LoadAssistant(pid)["knowledge_hop_limit"]?.Value() + ?? Config.LoadAssistant(pid)["examples_hop_limit"]?.Value() ?? 6; + JArray rows = SearchKnowledge(pid, q, lim, rating); + knowledgeHops?.Add(new JObject { ["tool"] = "ask_knowledge", ["query"] = q, ["count"] = rows.Count }); + MergeKnowledgeResults(knowledgeResults, rows); + return + "ask:knowledge — FTS over attached books (read-only references). " + + "Remix ideas; do not paste long verbatim.\n```json\n" + + rows.ToString(Newtonsoft.Json.Formatting.None) + "\n```"; + } if (tool == "ask_examples") { string q = patch?["example_query"]?.ToString()?.Trim() @@ -471,6 +511,8 @@ public partial class SwarmAssistentExtension } int lim = Config.LoadAssistant(pid)["examples_hop_limit"]?.Value() ?? 5; JArray examples = Memory?.LookupExamples(q, lim, rating) ?? []; + knowledgeHops?.Add(new JObject { ["tool"] = "ask_examples", ["query"] = q, ["count"] = examples.Count }); + MergeKnowledgeResults(knowledgeResults, examples, legacyExamples: true); return "ask:examples — Civitai Krea2 prompt references (FTS, no embeddings). " + "These are EXAMPLES to remix, not copy 1:1. Prefer craft over pasting.\n```json\n" diff --git a/AssistentConfig.cs b/AssistentConfig.cs index 705067b..43a0171 100644 --- a/AssistentConfig.cs +++ b/AssistentConfig.cs @@ -239,6 +239,134 @@ public sealed class AssistentConfig return map; } + static List ParseYamlBracketList(string value) + { + value = (value ?? "").Trim(); + if (value.StartsWith('[') && value.EndsWith(']')) + { + value = value[1..^1]; + } + return value.Split(',') + .Select(s => s.Trim().Trim('"', '\'')) + .Where(s => !string.IsNullOrWhiteSpace(s)) + .ToList(); + } + + public string TryFindPackManifestPath(string personaId) + { + string shelf = PackShelfRoot(personaId); + if (string.IsNullOrWhiteSpace(shelf)) + { + return null; + } + string dir = shelf; + for (int i = 0; i < 5 && !string.IsNullOrWhiteSpace(dir); i++) + { + string yaml = Path.Combine(dir, "assistent-pack.yaml"); + if (File.Exists(yaml)) + { + return yaml; + } + dir = Directory.GetParent(dir)?.FullName; + } + return null; + } + + /// Pack manifest knowledge.attach list for a pack persona. + public List TryParsePackKnowledgeAttach(string personaId) + { + string path = TryFindPackManifestPath(personaId); + if (string.IsNullOrWhiteSpace(path)) + { + return []; + } + List items = []; + try + { + foreach (string rawLine in File.ReadAllLines(path, Encoding.UTF8)) + { + string line = rawLine.Trim(); + if (string.IsNullOrWhiteSpace(line) || line.StartsWith('#')) + { + continue; + } + if (line.StartsWith("knowledge.attach:", StringComparison.OrdinalIgnoreCase)) + { + string val = line["knowledge.attach:".Length..].Trim(); + items.AddRange(ParseYamlBracketList(val)); + break; + } + } + } + catch (Exception ex) + { + Logs.Debug($"AssistentConfig pack knowledge.attach {path}: {ex.Message}"); + } + return items + .Select(SafeId) + .Where(id => id is not null) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + public JObject LoadKnowledgeAttachOverlay(string personaId) + { + string id = SafeId(personaId); + if (id is null) + { + return null; + } + string path = Path.Combine(_overlayRoot, "personas", id, "knowledge.json"); + return File.Exists(path) ? TryReadJson(path) : null; + } + + public void SaveKnowledgeAttachOverlay(string personaId, JArray attach) + { + lock (_lock) + { + string id = SafeId(personaId) ?? throw new InvalidOperationException("invalid persona"); + string dir = Path.Combine(_overlayRoot, "personas", id); + Directory.CreateDirectory(dir); + string path = Path.Combine(dir, "knowledge.json"); + JObject doc = new() + { + ["attach"] = attach ?? new JArray(), + }; + File.WriteAllText(path, doc.ToString(Newtonsoft.Json.Formatting.Indented) + "\n", Encoding.UTF8); + } + } + + static readonly string[] DefaultBundledKnowledgeAttach = ["civitai-krea2", "ru-fictext-rplus"]; + + static readonly HashSet BundledKnowledgePersonas = new(StringComparer.OrdinalIgnoreCase) + { + "neutral", "aggressive", "dreamer", + }; + + /// Effective book attach list for a persona (overlay → pack → bundled defaults). + public List ResolveKnowledgeAttach(string personaId) + { + string pid = SafeId(personaId) ?? DefaultPersonaId(); + JObject overlay = LoadKnowledgeAttachOverlay(pid); + if (overlay?["attach"] is JArray custom && custom.Count > 0) + { + return custom.Select(t => SafeId(t?.ToString())) + .Where(id => id is not null) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + List fromPack = TryParsePackKnowledgeAttach(pid); + if (fromPack.Count > 0) + { + return fromPack; + } + if (IsBundledPersona(pid) && BundledKnowledgePersonas.Contains(pid)) + { + return DefaultBundledKnowledgeAttach.ToList(); + } + return []; + } + /// Discover persona packs under Assistent/extensions/*/. public List DiscoverPackPersonas() { @@ -477,7 +605,7 @@ public sealed class AssistentConfig static readonly HashSet ReservedConfigFiles = new(StringComparer.OrdinalIgnoreCase) { - "exact.json", "controls.json", "skills.json", "ui.json", "assistant.json", "training-qlora.json", + "exact.json", "controls.json", "skills.json", "ui.json", "assistant.json", "knowledge.json", }; static readonly HashSet ReservedConfigDirs = new(StringComparer.OrdinalIgnoreCase) @@ -780,7 +908,7 @@ public sealed class AssistentConfig { "persona.json", "bio.json", "voice.json", "humor.json", "craft.json", "appearance.json", "outfits.json", "roleplay.json", "likes.json", "dislikes.json", - "rules.json", "controls.json", "exact.json", "extra.md", + "rules.json", "controls.json", "exact.json", "knowledge.json", "extra.md", }; public static bool IsWritableShelfName(string fileName) @@ -934,8 +1062,6 @@ public sealed class AssistentConfig public JObject LoadUi(string personaId) => MergeJsonLayers("ui.json", LayerRoots(personaId)); - public JObject LoadTrainingQlora(string personaId) => MergeJsonLayers("training-qlora.json", LayerRoots(personaId)); - /// Exact (KV) memory: bundled base → persona overlays → disk overlays. Persona keys overwrite base. public JObject LoadExact(string personaId) => MergeJsonLayers("exact.json", LayerRoots(personaId)); @@ -1549,40 +1675,6 @@ public sealed class AssistentConfig } } - public JObject LoadTrainingRunner() - => TryReadJson(Path.Combine(_overlayRoot, "training-runner.json")) ?? new JObject(); - - public void SaveTrainingRunner(JObject settings) - { - lock (_lock) - { - Directory.CreateDirectory(_overlayRoot); - string path = Path.Combine(_overlayRoot, "training-runner.json"); - JObject merged = DeepMerge(LoadTrainingRunner(), settings ?? new JObject()); - File.WriteAllText(path, merged.ToString(Newtonsoft.Json.Formatting.Indented), Encoding.UTF8); - } - } - - public JObject LoadTrainingAgent() - => TryReadJson(Path.Combine(_overlayRoot, "training-agent.json")) - ?? new JObject - { - ["enabled"] = true, - ["auto_link_on_approve"] = true, - ["heard_quota"] = 3, - }; - - public void SaveTrainingAgent(JObject settings) - { - lock (_lock) - { - Directory.CreateDirectory(_overlayRoot); - string path = Path.Combine(_overlayRoot, "training-agent.json"); - JObject merged = DeepMerge(LoadTrainingAgent(), settings ?? new JObject()); - File.WriteAllText(path, merged.ToString(Newtonsoft.Json.Formatting.Indented), Encoding.UTF8); - } - } - public JObject LoadOllamaRoles() { return TryReadJson(Path.Combine(_overlayRoot, "ollama-roles.json")) @@ -1643,7 +1735,6 @@ public sealed class AssistentConfig JObject ui = LoadUi(id); JObject model = LoadModelProfile(id); JObject exact = LoadExact(id); - JObject training = LoadTrainingQlora(id); var packs = ListPacks(id); var skills = ListSkills(id); var personas = ListPersonaCatalog(); @@ -1659,7 +1750,6 @@ public sealed class AssistentConfig ["ui"] = ui, ["model"] = model, ["exact"] = exact, - ["training"] = training, ["controls"] = controlsSchema, ["control_values"] = controlValues, ["persona_source"] = PersonaSource(id), @@ -1687,6 +1777,10 @@ public sealed class AssistentConfig ["identity_summary"] = RenderIdentityBlock(id, includeAllShelves: true), ["enabled_skills"] = new JArray(ResolveEnabledSkills(id, null)), ["patch_keys"] = new JArray(LoadPatchKeys()), + ["knowledge"] = new JObject + { + ["attach"] = new JArray(ResolveKnowledgeAttach(id)), + }, }; } } diff --git a/AssistentHuggingFace.cs b/AssistentHuggingFace.cs deleted file mode 100644 index 24aff97..0000000 --- a/AssistentHuggingFace.cs +++ /dev/null @@ -1,684 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using SwarmUI.Accounts; -using SwarmUI.Utils; - -namespace Mrleo1nid.SwarmAssistent; - -/// Hugging Face datasets: search, compatibility gate, preview, import. -public partial class SwarmAssistentExtension -{ - static readonly Regex HfRepoIdRe = new(@"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}(/[A-Za-z0-9][A-Za-z0-9._-]{0,95})?$", RegexOptions.Compiled); - - const string HfDatasetsServer = "https://datasets-server.huggingface.co"; - const string HfHubApi = "https://huggingface.co/api/datasets"; - - static string GetHfToken(Session session) - => session?.User?.GetGenericData("huggingface_api", "key")?.Trim(); - - static HttpRequestMessage HfRequest(string url, Session session) - { - HttpRequestMessage req = new(HttpMethod.Get, url); - string token = GetHfToken(session); - if (!string.IsNullOrWhiteSpace(token)) - { - req.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); - } - return req; - } - - /// Normalize owner/name or HF datasets URL. Returns null when invalid. - public static string NormalizeHfDatasetId(string raw) - { - string s = (raw ?? "").Trim(); - if (string.IsNullOrWhiteSpace(s)) - { - return null; - } - if (s.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || s.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) - { - if (!Uri.TryCreate(s, UriKind.Absolute, out Uri uri)) - { - return null; - } - if (!string.Equals(uri.Host, "huggingface.co", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - string[] parts = uri.AbsolutePath.Trim('/').Split('/'); - if (parts.Length < 2 || !string.Equals(parts[0], "datasets", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - s = $"{parts[1]}/{parts[2]}"; - } - s = s.Trim().TrimEnd('/'); - return HfRepoIdRe.IsMatch(s) ? s : null; - } - - public async Task AssistentSearchHfDatasets(Session session, string q = null, int limit = 20, bool show_all = false) - { - int take = Math.Clamp(limit, 1, 50); - string search = (q ?? "").Trim(); - StringBuilder url = new($"{HfHubApi}?limit={take}&full=true"); - url.Append("&filter=task_categories:text-generation"); - url.Append("&filter=modality:text"); - if (!string.IsNullOrWhiteSpace(search)) - { - url.Append("&search=").Append(Uri.EscapeDataString(search)); - } - try - { - using HttpRequestMessage req = HfRequest(url.ToString(), session); - using HttpResponseMessage resp = await HttpClient.SendAsync(req); - string body = await resp.Content.ReadAsStringAsync(); - if (!resp.IsSuccessStatusCode) - { - return new JObject { ["error"] = $"HF search HTTP {(int)resp.StatusCode}: {Clip(body, 300)}" }; - } - JArray rawList = JArray.Parse(body); - JArray results = []; - foreach (JToken item in rawList) - { - if (item is not JObject o) - { - continue; - } - string id = o["id"]?.ToString(); - if (string.IsNullOrWhiteSpace(id)) - { - continue; - } - JObject check = await CheckHfDatasetInternal(session, id, useCache: true); - string gate = check["gate"]?.ToString() ?? "rejected"; - if (!show_all && gate == "rejected") - { - continue; - } - results.Add(new JObject - { - ["id"] = id, - ["title"] = o["id"], - ["downloads"] = o["downloads"], - ["gate"] = gate, - ["reason"] = check["reason"], - ["schema"] = check["schema"], - }); - } - return new JObject { ["success"] = true, ["results"] = results, ["has_hf_token"] = !string.IsNullOrWhiteSpace(GetHfToken(session)) }; - } - catch (Exception ex) - { - return new JObject { ["error"] = $"HF search: {ex.Message}" }; - } - } - - public async Task AssistentCheckHfDataset(Session session, string dataset) - { - string id = NormalizeHfDatasetId(dataset); - if (id is null) - { - return new JObject { ["success"] = false, ["error"] = "Нужен owner/name или ссылка huggingface.co/datasets/…" }; - } - JObject check = await CheckHfDatasetInternal(session, id, useCache: false); - check["success"] = check["gate"]?.ToString() != "rejected"; - check["id"] = id; - return check; - } - - async Task CheckHfDatasetInternal(Session session, string datasetId, bool useCache) - { - string cacheKey = $"hf:{datasetId}"; - if (useCache) - { - JObject cached = Memory.GetKvObject(AssistentMemory.KvHfDatasetCache)?[cacheKey] as JObject; - if (cached is not null && cached["checked_at"]?.Value() > DateTimeOffset.UtcNow.AddHours(-6).ToUnixTimeMilliseconds()) - { - return cached; - } - } - JObject result = new() { ["id"] = datasetId, ["gate"] = "rejected", ["reason"] = "unknown" }; - try - { - using HttpRequestMessage validReq = HfRequest($"{HfDatasetsServer}/is-valid?dataset={Uri.EscapeDataString(datasetId)}", session); - using HttpResponseMessage validResp = await HttpClient.SendAsync(validReq); - string validBody = await validResp.Content.ReadAsStringAsync(); - if (!validResp.IsSuccessStatusCode) - { - result["reason"] = $"is-valid HTTP {(int)validResp.StatusCode}"; - return CacheHfCheck(cacheKey, result); - } - JObject valid = JObject.Parse(validBody); - bool viewer = valid["viewer"]?.Value() == true; - bool preview = valid["preview"]?.Value() == true; - if (!viewer && !preview) - { - result["reason"] = "Набор не читается через datasets (viewer/preview = false)"; - return CacheHfCheck(cacheKey, result); - } - using HttpRequestMessage splitReq = HfRequest($"{HfDatasetsServer}/splits?dataset={Uri.EscapeDataString(datasetId)}", session); - using HttpResponseMessage splitResp = await HttpClient.SendAsync(splitReq); - string splitBody = await splitResp.Content.ReadAsStringAsync(); - if (!splitResp.IsSuccessStatusCode) - { - result["reason"] = $"splits HTTP {(int)splitResp.StatusCode}"; - return CacheHfCheck(cacheKey, result); - } - JArray splits = JObject.Parse(splitBody)["splits"] as JArray ?? []; - if (splits.Count == 0) - { - result["reason"] = "Нет splits"; - return CacheHfCheck(cacheKey, result); - } - JObject first = splits[0] as JObject; - string config = first?["config"]?.ToString() ?? "default"; - string split = first?["split"]?.ToString() ?? "train"; - using HttpRequestMessage rowsReq = HfRequest($"{HfDatasetsServer}/first-rows?dataset={Uri.EscapeDataString(datasetId)}&config={Uri.EscapeDataString(config)}&split={Uri.EscapeDataString(split)}", session); - using HttpResponseMessage rowsResp = await HttpClient.SendAsync(rowsReq); - string rowsBody = await rowsResp.Content.ReadAsStringAsync(); - if (!rowsResp.IsSuccessStatusCode) - { - result["reason"] = $"first-rows HTTP {(int)rowsResp.StatusCode}: {Clip(rowsBody, 200)}"; - return CacheHfCheck(cacheKey, result); - } - JObject rowsData = JObject.Parse(rowsBody); - JObject features = FeaturesToObject(rowsData["features"]); - (string gate, string reason, JObject schema) = ClassifyHfFeatures(features); - if (gate == "mapping" && TryFictionTagsTextPreset(features, out JObject presetSchema)) - { - gate = "ok"; - reason = "Fiction preset: title/tags → user, text → assistant"; - schema = presetSchema; - } - result["gate"] = gate; - result["reason"] = reason; - result["schema"] = schema; - result["config"] = config; - result["split"] = split; - result["features"] = features; - result["sample_rows"] = rowsData["rows"]; - result["runner_only"] = await HasHugeSizeTagAsync(session, datasetId); - return CacheHfCheck(cacheKey, result); - } - catch (Exception ex) - { - result["reason"] = ex.Message; - return CacheHfCheck(cacheKey, result); - } - } - - static bool TryFictionTagsTextPreset(JObject features, out JObject schema) - { - schema = null; - if (features is null) - { - return false; - } - HashSet names = new(StringComparer.OrdinalIgnoreCase); - foreach (JProperty p in features.Properties()) - { - names.Add(p.Name); - } - if (!names.Contains("text") || (!names.Contains("tags") && !names.Contains("title"))) - { - return false; - } - schema = new JObject - { - ["kind"] = "fiction_tags_text", - ["assistant_col"] = "text", - }; - return true; - } - - async Task HasHugeSizeTagAsync(Session session, string datasetId) - { - try - { - using HttpRequestMessage req = HfRequest($"{HfHubApi}/{Uri.EscapeDataString(datasetId)}", session); - using HttpResponseMessage resp = await HttpClient.SendAsync(req); - if (!resp.IsSuccessStatusCode) - { - return false; - } - JObject meta = JObject.Parse(await resp.Content.ReadAsStringAsync()); - foreach (JToken t in meta["tags"] as JArray ?? []) - { - string tag = t?.ToString() ?? ""; - if (tag.Contains("100K<", StringComparison.OrdinalIgnoreCase) - || tag.Contains("1M<", StringComparison.OrdinalIgnoreCase) - || tag.Contains("10M<", StringComparison.OrdinalIgnoreCase) - || tag.Contains("100M<", StringComparison.OrdinalIgnoreCase)) - { - return true; - } - } - } - catch (Exception ex) - { - Logs.Debug($"HasHugeSizeTag {datasetId}: {ex.Message}"); - } - return false; - } - - static JObject ResolveHfMapping(JObject check, JObject mapping) - { - JObject schema = check?["schema"] as JObject; - string kind = schema?["kind"]?.ToString(); - if (string.Equals(kind, "fiction_tags_text", StringComparison.OrdinalIgnoreCase)) - { - return new JObject { ["kind"] = "fiction_tags_text", ["preset"] = "fiction_tags_text" }; - } - if (mapping is not null && mapping.Count > 0) - { - return mapping; - } - if (string.Equals(mapping?["preset"]?.ToString(), "fiction_tags_text", StringComparison.OrdinalIgnoreCase)) - { - return new JObject { ["kind"] = "fiction_tags_text" }; - } - return mapping; - } - - static bool MappingRequired(JObject check, JObject mapping) - { - string gate = check?["gate"]?.ToString(); - if (gate != "mapping") - { - return false; - } - JObject resolved = ResolveHfMapping(check, mapping); - return resolved is null || !resolved.Properties().Any(); - } - - JObject CacheHfCheck(string cacheKey, JObject result) - { - result["checked_at"] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - try - { - JObject bag = Memory.GetKvObject(AssistentMemory.KvHfDatasetCache) ?? new JObject(); - bag[cacheKey] = result; - Memory.SetKvObject(AssistentMemory.KvHfDatasetCache, bag); - } - catch (Exception ex) - { - Logs.Debug($"CacheHfCheck: {ex.Message}"); - } - return result; - } - - static JObject FeaturesToObject(JToken tok) - { - if (tok is JObject obj) - { - return obj; - } - if (tok is JArray arr) - { - JObject map = new(); - foreach (JToken t in arr) - { - if (t is not JObject row) - { - continue; - } - string name = row["name"]?.ToString(); - if (string.IsNullOrWhiteSpace(name)) - { - continue; - } - map[name] = row["type"] ?? row; - } - return map.Count > 0 ? map : null; - } - return null; - } - - static (string gate, string reason, JObject schema) ClassifyHfFeatures(JObject features) - { - if (features is null || !features.Properties().Any()) - { - return ("rejected", "Нет колонок (features пуст)", null); - } - HashSet names = new(StringComparer.OrdinalIgnoreCase); - foreach (JProperty p in features.Properties()) - { - names.Add(p.Name); - JToken dtype = p.Value?["dtype"] ?? p.Value?["type"]; - string dt = dtype?.ToString() ?? ""; - if (dt.Contains("image", StringComparison.OrdinalIgnoreCase) - || dt.Contains("audio", StringComparison.OrdinalIgnoreCase) - || dt.Contains("video", StringComparison.OrdinalIgnoreCase)) - { - return ("rejected", $"Мультимодальная колонка {p.Name} ({dt})", null); - } - } - if (names.Contains("chosen") && names.Contains("rejected")) - { - return ("rejected", "DPO-набор (chosen/rejected) — не для SFT", null); - } - if (names.Count == 1 && names.Contains("text")) - { - return ("rejected", "Предобучение (одна колонка text), не диалоги", null); - } - if (names.Contains("messages")) - { - return ("ok", "OpenAI messages", new JObject { ["kind"] = "messages" }); - } - if (names.Contains("conversations")) - { - return ("ok", "ShareGPT conversations", new JObject { ["kind"] = "conversations" }); - } - if (names.Contains("instruction") && names.Contains("output")) - { - return ("ok", "Alpaca instruction/output", new JObject { ["kind"] = "alpaca" }); - } - if (names.Contains("prompt") && (names.Contains("response") || names.Contains("completion") || names.Contains("answer"))) - { - string respCol = names.Contains("response") ? "response" : names.Contains("completion") ? "completion" : "answer"; - return ("ok", "prompt/response", new JObject { ["kind"] = "prompt_response", ["response_col"] = respCol }); - } - if (names.Contains("question") && names.Contains("answer")) - { - return ("ok", "question/answer", new JObject { ["kind"] = "qa" }); - } - List stringCols = []; - foreach (JProperty p in features.Properties()) - { - JToken dtype = p.Value?["dtype"] ?? p.Value?["type"]; - string dt = dtype?.ToString() ?? ""; - if (dt.Contains("string", StringComparison.OrdinalIgnoreCase) || dt == "value") - { - stringCols.Add(p.Name); - } - } - if (stringCols.Count >= 2) - { - return ("mapping", "Нужен ручной маппинг колонок", new JObject - { - ["kind"] = "custom", - ["columns"] = new JArray(stringCols), - }); - } - return ("rejected", "Схема не подходит для SFT", null); - } - - public async Task AssistentPreviewHfDataset(Session session, string dataset, string config = null, string split = null) - { - string id = NormalizeHfDatasetId(dataset); - if (id is null) - { - return new JObject { ["error"] = "invalid dataset id" }; - } - JObject check = await CheckHfDatasetInternal(session, id, useCache: true); - if (check["gate"]?.ToString() == "rejected") - { - return new JObject { ["error"] = check["reason"]?.ToString() ?? "rejected", ["check"] = check }; - } - return new JObject { ["success"] = true, ["check"] = check }; - } - - public async Task AssistentImportHfDataset(Session session, JObject raw) - { - try - { - string dataset = raw?["dataset"]?.ToString(); - int limit = raw?["limit"]?.Value() ?? 200; - JObject mapping = raw?["mapping"] as JObject; - string id = NormalizeHfDatasetId(dataset); - if (id is null) - { - return new JObject { ["error"] = "invalid dataset id" }; - } - JObject check = await CheckHfDatasetInternal(session, id, useCache: true); - string gate = check["gate"]?.ToString(); - if (gate == "rejected") - { - return new JObject { ["error"] = check["reason"]?.ToString() ?? "rejected" }; - } - if (MappingRequired(check, mapping)) - { - return new JObject { ["error"] = "Нужен маппинг колонок", ["check"] = check }; - } - mapping = ResolveHfMapping(check, mapping); - int take = Math.Clamp(limit, 1, 5000); - JArray rows = []; - string config = check["config"]?.ToString() ?? "default"; - string split = check["split"]?.ToString() ?? "train"; - int offset = 0; - while (rows.Count < take) - { - int chunk = Math.Min(100, take - rows.Count); - using HttpRequestMessage rowsReq = HfRequest($"{HfDatasetsServer}/rows?dataset={Uri.EscapeDataString(id)}&config={Uri.EscapeDataString(config)}&split={Uri.EscapeDataString(split)}&offset={offset}&length={chunk}", session); - using HttpResponseMessage rowsResp = await HttpClient.SendAsync(rowsReq); - string rowsBody = await rowsResp.Content.ReadAsStringAsync(); - if (!rowsResp.IsSuccessStatusCode) - { - if (rows.Count == 0) - { - return new JObject - { - ["error"] = $"HF rows HTTP {(int)rowsResp.StatusCode}: {Clip(rowsBody, 240)}. " - + "Для gated/NSFW добавь huggingface_api в User Settings.", - }; - } - break; - } - JObject parsed = JObject.Parse(rowsBody); - JArray batch = parsed["rows"] as JArray ?? []; - if (batch.Count == 0) - { - break; - } - foreach (JToken t in batch) - { - rows.Add(t); - } - offset += batch.Count; - if (batch.Count < chunk) - { - break; - } - } - if (rows.Count == 0) - { - rows = check["sample_rows"] as JArray ?? []; - } - List toSave = []; - foreach (JToken rowTok in rows.Take(take)) - { - if (rowTok is not JObject row) - { - continue; - } - JObject rowData = row["row"] as JObject ?? row; - JArray messages = ConvertHfRowToMessages(rowData, check["schema"] as JObject, mapping); - if (messages is null || messages.Count == 0) - { - continue; - } - toSave.Add(new JObject - { - ["source"] = "hf", - ["hf_repo"] = id, - ["messages"] = messages, - ["status"] = "draft", - }); - } - int imported = toSave.Count > 0 ? Memory.ImportTrainSamplesBatch(toSave) : 0; - if (imported == 0 && check["runner_only"]?.Value() == true) - { - return new JObject { ["success"] = true, ["imported"] = 0, ["runner_only"] = true, ["id"] = id, ["note"] = "Большой набор — используй HF id в QLoRA-раннере" }; - } - if (imported == 0) - { - return new JObject - { - ["error"] = rows.Count == 0 - ? "HF не отдал строки — проверь token (gated/NSFW) и маппинг колонок" - : "0 строк после маппинга — проверь колонки user/assistant", - ["rows_fetched"] = rows.Count, - ["id"] = id, - }; - } - return new JObject - { - ["success"] = true, - ["imported"] = imported, - ["id"] = id, - ["rows_fetched"] = rows.Count, - ["status"] = "draft", - }; - } - catch (Exception ex) - { - Logs.Error($"AssistentImportHfDataset: {ex}"); - return new JObject { ["error"] = ex.Message }; - } - } - - static string HfCellString(JToken tok) - { - if (tok is null || tok.Type == JTokenType.Null) - { - return ""; - } - if (tok is JArray arr) - { - List parts = []; - foreach (JToken t in arr) - { - string s = t?.Type == JTokenType.String ? t.ToString() : t?.ToString(Newtonsoft.Json.Formatting.None); - if (!string.IsNullOrWhiteSpace(s)) - { - parts.Add(s.Trim()); - } - } - return string.Join(", ", parts); - } - return tok.ToString().Trim(); - } - - static JArray ConvertHfRowToMessages(JObject row, JObject schema, JObject mapping) - { - string kind = schema?["kind"]?.ToString() ?? mapping?["kind"]?.ToString(); - if (kind == "messages" && row["messages"] is JArray msgs) - { - return NormalizeMessagesArray(msgs); - } - if (kind == "conversations" && row["conversations"] is JArray conv) - { - JArray outArr = []; - foreach (JToken c in conv) - { - if (c is not JObject co) - { - continue; - } - string from = co["from"]?.ToString() ?? ""; - string val = co["value"]?.ToString() ?? ""; - string role = from is "human" or "user" ? "user" : from is "gpt" or "assistant" or "chatgpt" ? "assistant" : "user"; - outArr.Add(new JObject { ["role"] = role, ["content"] = val }); - } - return outArr.Count > 0 ? outArr : null; - } - if (kind == "alpaca") - { - string instr = row["instruction"]?.ToString() ?? ""; - string inp = row["input"]?.ToString() ?? ""; - string output = row["output"]?.ToString() ?? ""; - string user = string.IsNullOrWhiteSpace(inp) ? instr : $"{instr}\n{inp}"; - return new JArray - { - new JObject { ["role"] = "user", ["content"] = user }, - new JObject { ["role"] = "assistant", ["content"] = output }, - }; - } - if (kind == "prompt_response") - { - string respCol = schema?["response_col"]?.ToString() ?? "response"; - return new JArray - { - new JObject { ["role"] = "user", ["content"] = row["prompt"]?.ToString() ?? "" }, - new JObject { ["role"] = "assistant", ["content"] = row[respCol]?.ToString() ?? "" }, - }; - } - if (kind == "qa") - { - return new JArray - { - new JObject { ["role"] = "user", ["content"] = row["question"]?.ToString() ?? "" }, - new JObject { ["role"] = "assistant", ["content"] = row["answer"]?.ToString() ?? "" }, - }; - } - if (kind == "fiction_tags_text") - { - string text = HfCellString(row["text"]); - if (string.IsNullOrWhiteSpace(text)) - { - return null; - } - List userParts = []; - string title = HfCellString(row["title"]); - string tags = HfCellString(row["tags"]); - if (!string.IsNullOrWhiteSpace(title)) - { - userParts.Add($"Title: {title}"); - } - if (!string.IsNullOrWhiteSpace(tags)) - { - userParts.Add($"Tags: {tags}"); - } - string user = userParts.Count > 0 ? string.Join("\n", userParts) : tags ?? title ?? ""; - if (string.IsNullOrWhiteSpace(user)) - { - return null; - } - return new JArray - { - new JObject { ["role"] = "user", ["content"] = user }, - new JObject { ["role"] = "assistant", ["content"] = text }, - }; - } - if (kind == "custom" && mapping is not null) - { - string userCol = mapping["user_col"]?.ToString(); - string asstCol = mapping["assistant_col"]?.ToString(); - if (!string.IsNullOrWhiteSpace(userCol) && !string.IsNullOrWhiteSpace(asstCol)) - { - return new JArray - { - new JObject { ["role"] = "user", ["content"] = row[userCol]?.ToString() ?? "" }, - new JObject { ["role"] = "assistant", ["content"] = row[asstCol]?.ToString() ?? "" }, - }; - } - } - return null; - } - - static JArray NormalizeMessagesArray(JArray msgs) - { - JArray outArr = []; - foreach (JToken m in msgs) - { - if (m is not JObject mo) - { - continue; - } - string role = mo["role"]?.ToString() ?? "user"; - string content = mo["content"]?.ToString() ?? mo["text"]?.ToString() ?? ""; - if (string.IsNullOrWhiteSpace(content)) - { - continue; - } - outArr.Add(new JObject { ["role"] = role, ["content"] = content }); - } - return outArr.Count > 0 ? outArr : null; - } -} diff --git a/AssistentKnowledge.cs b/AssistentKnowledge.cs new file mode 100644 index 0000000..ed43664 --- /dev/null +++ b/AssistentKnowledge.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Newtonsoft.Json.Linq; +using SwarmUI.Utils; + +namespace Mrleo1nid.SwarmAssistent; + +/// Unified knowledge hub: books FTS + legacy examples shim + catalog for personas. +public partial class SwarmAssistentExtension +{ + public List ResolveAttachedBooks(string personaId) + => Config.ResolveKnowledgeAttach(personaId); + + public JObject BuildKnowledgeCatalog(string personaId) + { + string pid = AssistentConfig.SafeId(personaId) ?? Config.DefaultPersonaId(); + List attached = ResolveAttachedBooks(pid); + JArray books = Memory?.ListBooksOnDisk() ?? []; + HashSet attachSet = attached.ToHashSet(StringComparer.OrdinalIgnoreCase); + JArray catalog = []; + foreach (JToken t in books) + { + if (t is not JObject b) + { + continue; + } + string id = b["id"]?.ToString() ?? ""; + catalog.Add(new JObject + { + ["id"] = id, + ["title"] = b["title"] ?? id, + ["description"] = b["description"] ?? "", + ["content_kind"] = b["content_kind"] ?? "", + ["language"] = b["language"] ?? "", + ["attached"] = attachSet.Contains(id), + ["indexed"] = b["indexed"]?.Value() ?? false, + }); + } + return new JObject + { + ["persona"] = pid, + ["attach"] = new JArray(attached), + ["books"] = catalog, + ["books_indexed"] = Memory?.BookRowCount() ?? 0, + ["examples_indexed"] = Memory?.ExampleCount() ?? 0, + ["tags_indexed"] = Memory?.TagCount() ?? 0, + }; + } + + public string BuildKnowledgeSystemLayer(string personaId) + { + JObject cat = BuildKnowledgeCatalog(personaId); + JArray attach = cat["attach"] as JArray ?? []; + if (attach.Count == 0) + { + return ""; + } + StringBuilder sb = new(); + sb.AppendLine("## Knowledge books (FTS reference — read-only)"); + sb.AppendLine("Attached corpora for this persona. Search via ask:[\"knowledge\"] + knowledge_query (or legacy ask:[\"examples\"] + example_query)."); + foreach (JToken t in attach) + { + string id = t?.ToString()?.Trim(); + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + JObject meta = (cat["books"] as JArray)?.OfType() + .FirstOrDefault(b => string.Equals(b["id"]?.ToString(), id, StringComparison.OrdinalIgnoreCase)); + string title = meta?["title"]?.ToString() ?? id; + string kind = meta?["content_kind"]?.ToString() ?? ""; + string desc = meta?["description"]?.ToString() ?? ""; + sb.AppendLine($"- **{id}** ({title}){ (string.IsNullOrWhiteSpace(kind) ? "" : $" · {kind}")}"); + if (!string.IsNullOrWhiteSpace(desc)) + { + sb.AppendLine($" {desc}"); + } + } + return sb.ToString().TrimEnd(); + } + + public JArray SearchKnowledge(string personaId, string query, int limit = 8, string rating = null) + { + query = (query ?? "").Trim(); + if (query.Length < 1 || Memory is null) + { + return []; + } + List attached = ResolveAttachedBooks(personaId); + JArray hits = []; + HashSet seen = []; + if (attached.Count > 0) + { + foreach (JToken t in Memory.LookupBooks(query, limit, attached, rating)) + { + string key = t?["id"]?.ToString() ?? t?.ToString(); + if (!string.IsNullOrWhiteSpace(key) && seen.Add(key)) + { + hits.Add(t); + } + } + } + if (hits.Count < limit && attached.Any(b => string.Equals(b, "civitai-krea2", StringComparison.OrdinalIgnoreCase))) + { + int remain = limit - hits.Count; + foreach (JToken t in Memory.LookupExamples(query, remain, rating)) + { + string key = "legacy:" + (t?["id"]?.ToString() ?? ""); + if (!seen.Add(key)) + { + continue; + } + JObject row = t as JObject ?? new JObject(); + hits.Add(new JObject + { + ["id"] = row["id"], + ["book"] = "civitai-krea2", + ["source"] = "examples-legacy", + ["title"] = "", + ["tags"] = row["tags"], + ["text"] = row["prompt"], + ["body"] = row["prompt"], + ["rating"] = row["rating"], + ["params"] = row["params"], + ["loras"] = row["loras"], + ["note"] = row["note"], + }); + } + } + return hits; + } + + public static JArray CivitaiResultsShim(JArray knowledgeResults) + { + JArray outRows = []; + foreach (JToken t in knowledgeResults ?? []) + { + if (t is not JObject row) + { + continue; + } + string book = row["book"]?.ToString() ?? ""; + string source = row["source"]?.ToString() ?? ""; + if (!string.Equals(book, "civitai-krea2", StringComparison.OrdinalIgnoreCase) + && source is not "examples-legacy") + { + continue; + } + if (row["prompt"] is not null) + { + outRows.Add(row); + continue; + } + outRows.Add(new JObject + { + ["id"] = row["id"], + ["rating"] = row["rating"], + ["tags"] = row["tags"], + ["prompt"] = row["text"] ?? row["body"], + ["negative"] = row["meta"]?["negative"] ?? "", + ["params"] = row["params"] ?? row["meta"]?["params"], + ["loras"] = row["loras"] ?? row["meta"]?["loras"], + ["note"] = row["note"] ?? "EXAMPLE from Civitai — remix, do not copy 1:1", + }); + } + return outRows; + } + + JObject BuildKnowledgeResponse(string personaId, JArray hops, JArray results) + => new() + { + ["catalog"] = BuildKnowledgeCatalog(personaId), + ["hops"] = hops ?? new JArray(), + ["results"] = results ?? new JArray(), + }; + + static void MergeKnowledgeResults(JArray target, JArray rows, bool legacyExamples = false) + { + if (target is null || rows is null) + { + return; + } + HashSet seen = target + .Select(t => t?["id"]?.ToString() ?? t?.ToString()) + .Where(s => !string.IsNullOrWhiteSpace(s)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (JToken t in rows) + { + JObject row = t as JObject ?? new JObject { ["text"] = t?.ToString() }; + if (legacyExamples) + { + row = new JObject + { + ["id"] = row["id"], + ["book"] = "civitai-krea2", + ["source"] = "examples-legacy", + ["tags"] = row["tags"], + ["text"] = row["prompt"], + ["body"] = row["prompt"], + ["rating"] = row["rating"], + ["note"] = row["note"], + }; + } + string key = row["id"]?.ToString() ?? row["text"]?.ToString(); + if (string.IsNullOrWhiteSpace(key) || !seen.Add(key)) + { + continue; + } + target.Add(row); + } + } +} diff --git a/AssistentKnowledgeApi.cs b/AssistentKnowledgeApi.cs new file mode 100644 index 0000000..b49dda5 --- /dev/null +++ b/AssistentKnowledgeApi.cs @@ -0,0 +1,129 @@ +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using SwarmUI.Accounts; + +namespace Mrleo1nid.SwarmAssistent; + +public partial class SwarmAssistentExtension +{ + public async Task AssistentListKnowledgeCatalog(Session session, string persona = null) + { + await Task.CompletedTask; + if (Memory is null) + { + return new JObject { ["error"] = "memory not ready" }; + } + try + { + Memory.EnsureBooksIndex(); + string pid = AssistentConfig.SafeId(persona) ?? Config.DefaultPersonaId(); + return new JObject + { + ["success"] = true, + ["knowledge"] = BuildKnowledgeCatalog(pid), + ["editable"] = Config.IsOverlayPersona(pid) && !Config.IsProtectedPersona(pid), + }; + } + catch (System.Exception ex) + { + return new JObject { ["error"] = $"knowledge catalog: {ex.Message}" }; + } + } + + public async Task AssistentSearchKnowledge(Session session, string query, string persona = null, int limit = 8, string rating = null, string book = null) + { + await Task.CompletedTask; + if (Memory is null) + { + return new JObject { ["error"] = "memory not ready" }; + } + query = (query ?? "").Trim(); + if (query.Length < 1) + { + return new JObject { ["error"] = "query required" }; + } + try + { + string pid = AssistentConfig.SafeId(persona) ?? Config.DefaultPersonaId(); + JArray results; + if (!string.IsNullOrWhiteSpace(book)) + { + results = Memory.LookupBooks(query, limit, [book.Trim()], rating); + } + else + { + results = SearchKnowledge(pid, query, limit, rating); + } + return new JObject + { + ["success"] = true, + ["query"] = query, + ["persona"] = pid, + ["results"] = results, + ["civitai_results"] = CivitaiResultsShim(results), + }; + } + catch (System.Exception ex) + { + return new JObject { ["error"] = $"knowledge search: {ex.Message}" }; + } + } + + public async Task AssistentGetKnowledgeAttach(Session session, string persona = null) + { + await Task.CompletedTask; + string pid = AssistentConfig.SafeId(persona) ?? Config.DefaultPersonaId(); + return new JObject + { + ["success"] = true, + ["persona"] = pid, + ["attach"] = new JArray(ResolveAttachedBooks(pid)), + ["editable"] = Config.IsOverlayPersona(pid) && !Config.IsProtectedPersona(pid), + ["source"] = Config.PersonaSource(pid), + }; + } + + public async Task AssistentSaveKnowledgeAttach(Session session, string persona, JArray attach) + { + await Task.CompletedTask; + string pid = AssistentConfig.SafeId(persona); + if (pid is null) + { + return new JObject { ["error"] = "persona required" }; + } + if (Config.IsProtectedPersona(pid)) + { + return new JObject { ["error"] = "bundled/pack personas cannot edit attach here — clone to overlay first" }; + } + if (!Config.IsOverlayPersona(pid)) + { + return new JObject { ["error"] = "overlay persona required" }; + } + try + { + JArray cleaned = []; + if (attach is not null) + { + foreach (JToken t in attach) + { + string id = AssistentConfig.SafeId(t?.ToString()); + if (id is not null) + { + cleaned.Add(id); + } + } + } + Config.SaveKnowledgeAttachOverlay(pid, cleaned); + return new JObject + { + ["success"] = true, + ["persona"] = pid, + ["attach"] = new JArray(ResolveAttachedBooks(pid)), + }; + } + catch (System.Exception ex) + { + return new JObject { ["error"] = $"save knowledge attach: {ex.Message}" }; + } + } +} diff --git a/AssistentMemory.Books.cs b/AssistentMemory.Books.cs new file mode 100644 index 0000000..6e40a9a --- /dev/null +++ b/AssistentMemory.Books.cs @@ -0,0 +1,458 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Data.Sqlite; +using Newtonsoft.Json.Linq; +using SwarmUI.Utils; + +namespace Mrleo1nid.SwarmAssistent; + +/// Reference books (search.jsonl) — FTS only, seeded by gpu-rent to Assistent/books/. +public sealed partial class AssistentMemory +{ + const string BooksMetaPrefix = "books_fp:"; + + void TryIndexBooks() + { + try + { + EnsureBooksIndex(); + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory books index: {ex.Message}"); + } + } + + public string BooksRoot() + => Path.Combine(_dataRoot, "Assistent", "books"); + + void EnsureBooksSchema() + { + Exec( + """ + CREATE TABLE IF NOT EXISTS books_rows ( + id TEXT PRIMARY KEY, + book_id TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + tags TEXT NOT NULL DEFAULT '', + text TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + rating TEXT NOT NULL DEFAULT '', + meta_json TEXT NOT NULL DEFAULT '{}', + search_blob TEXT NOT NULL DEFAULT '' + ); + """); + Exec("CREATE INDEX IF NOT EXISTS idx_books_rows_book ON books_rows(book_id);"); + Exec( + """ + CREATE VIRTUAL TABLE IF NOT EXISTS books_fts USING fts5( + book_id, + title, + tags, + text, + body, + search_blob, + tokenize = 'unicode61 remove_diacritics 2' + ); + """); + Exec( + """ + CREATE TRIGGER IF NOT EXISTS books_fts_ai AFTER INSERT ON books_rows BEGIN + INSERT INTO books_fts(rowid, book_id, title, tags, text, body, search_blob) + VALUES (new.rowid, new.book_id, new.title, new.tags, new.text, new.body, new.search_blob); + END; + """); + Exec( + """ + CREATE TRIGGER IF NOT EXISTS books_fts_ad AFTER DELETE ON books_rows BEGIN + INSERT INTO books_fts(books_fts, rowid) VALUES('delete', old.rowid); + END; + """); + Exec( + """ + CREATE TRIGGER IF NOT EXISTS books_fts_au AFTER UPDATE ON books_rows BEGIN + INSERT INTO books_fts(books_fts, rowid) VALUES('delete', old.rowid); + INSERT INTO books_fts(rowid, book_id, title, tags, text, body, search_blob) + VALUES (new.rowid, new.book_id, new.title, new.tags, new.text, new.body, new.search_blob); + END; + """); + } + + static string ReadBookContentSha(string bookDir) + { + foreach (string name in new[] { ".gpu-rent-meta.json", "meta.json" }) + { + string path = Path.Combine(bookDir, name); + if (!File.Exists(path)) + { + continue; + } + try + { + JObject meta = JObject.Parse(File.ReadAllText(path, Encoding.UTF8)); + string sha = meta["content_sha"]?.ToString()?.Trim(); + if (!string.IsNullOrWhiteSpace(sha)) + { + return sha; + } + } + catch + { + // ignore + } + } + string jsonl = Path.Combine(bookDir, "search.jsonl"); + if (File.Exists(jsonl)) + { + return FileFingerprint(jsonl); + } + return null; + } + + /// Reindex changed books from disk. Returns total row count. + public int EnsureBooksIndex() + { + string root = BooksRoot(); + if (!Directory.Exists(root)) + { + return BookRowCount(); + } + int total = 0; + foreach (string bookDir in Directory.GetDirectories(root).OrderBy(d => d, StringComparer.OrdinalIgnoreCase)) + { + string bookId = Path.GetFileName(bookDir); + if (string.IsNullOrWhiteSpace(bookId)) + { + continue; + } + string jsonl = Path.Combine(bookDir, "search.jsonl"); + if (!File.Exists(jsonl)) + { + continue; + } + string fp = ReadBookContentSha(bookDir) ?? FileFingerprint(jsonl); + lock (_lock) + { + EnsureOpen(); + EnsureBooksSchema(); + string metaKey = BooksMetaPrefix + bookId; + if (string.Equals(GetMeta(metaKey), fp, StringComparison.Ordinal)) + { + continue; + } + } + int n = IndexBookJsonl(bookId, jsonl, fp); + total += n; + } + if (total > 0) + { + Logs.Info($"AssistentMemory: indexed {total} book rows under {root}"); + } + return BookRowCount(); + } + + int IndexBookJsonl(string bookId, string jsonlPath, string fingerprint) + { + List<(string id, string title, string tags, string text, string body, string rating, string metaJson, string blob)> rows = []; + foreach (string line in File.ReadLines(jsonlPath, Encoding.UTF8)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + JObject o; + try + { + o = JObject.Parse(line); + } + catch + { + continue; + } + string id = o["id"]?.ToString()?.Trim(); + if (string.IsNullOrWhiteSpace(id)) + { + id = $"{bookId}:{rows.Count + 1}"; + } + string title = o["title"]?.ToString() ?? ""; + string tagsJoined = ""; + if (o["tags"] is JArray tagArr) + { + tagsJoined = string.Join(", ", tagArr.Select(t => t?.ToString()?.Trim()).Where(t => !string.IsNullOrWhiteSpace(t))); + } + string text = o["text"]?.ToString() ?? ""; + string body = o["body"]?.ToString() ?? text; + if (string.IsNullOrWhiteSpace(text) && string.IsNullOrWhiteSpace(body)) + { + continue; + } + string rating = o["rating"]?.ToString() ?? ""; + string metaJson = (o["meta"] as JObject)?.ToString(Newtonsoft.Json.Formatting.None) ?? "{}"; + string blob = $"{title}\n{tagsJoined}\n{text}\n{body}\n{rating}"; + rows.Add((id, title, tagsJoined, text, body, rating, metaJson, blob)); + } + + lock (_lock) + { + EnsureOpen(); + EnsureBooksSchema(); + using SqliteTransaction tx = _conn.BeginTransaction(); + using (SqliteCommand del = _conn.CreateCommand()) + { + del.Transaction = tx; + del.CommandText = "DELETE FROM books_rows WHERE book_id = $b"; + del.Parameters.AddWithValue("$b", bookId); + del.ExecuteNonQuery(); + } + using (SqliteCommand ins = _conn.CreateCommand()) + { + ins.Transaction = tx; + ins.CommandText = + """ + INSERT OR REPLACE INTO books_rows( + id, book_id, title, tags, text, body, rating, meta_json, search_blob) + VALUES($id,$b,$t,$tg,$tx,$bd,$r,$mj,$bl) + """; + var pid = ins.Parameters.Add("$id", SqliteType.Text); + var pb = ins.Parameters.Add("$b", SqliteType.Text); + var pt = ins.Parameters.Add("$t", SqliteType.Text); + var ptg = ins.Parameters.Add("$tg", SqliteType.Text); + var ptx = ins.Parameters.Add("$tx", SqliteType.Text); + var pbd = ins.Parameters.Add("$bd", SqliteType.Text); + var pr = ins.Parameters.Add("$r", SqliteType.Text); + var pmj = ins.Parameters.Add("$mj", SqliteType.Text); + var pbl = ins.Parameters.Add("$bl", SqliteType.Text); + foreach (var row in rows) + { + pid.Value = row.id; + pb.Value = bookId; + pt.Value = row.title; + ptg.Value = row.tags; + ptx.Value = row.text; + pbd.Value = row.body; + pr.Value = row.rating; + pmj.Value = row.metaJson; + pbl.Value = row.blob; + ins.ExecuteNonQuery(); + } + } + tx.Commit(); + SetMeta(BooksMetaPrefix + bookId, fingerprint); + Logs.Info($"AssistentMemory: indexed book {bookId} ({rows.Count} rows)"); + return rows.Count; + } + } + + public int BookRowCount() + { + lock (_lock) + { + EnsureOpen(); + if (!TableExists("books_rows")) + { + return 0; + } + using SqliteCommand c = _conn.CreateCommand(); + c.CommandText = "SELECT COUNT(*) FROM books_rows"; + return Convert.ToInt32(c.ExecuteScalar()); + } + } + + public JArray ListBooksOnDisk() + { + JArray list = []; + string root = BooksRoot(); + if (!Directory.Exists(root)) + { + return list; + } + foreach (string bookDir in Directory.GetDirectories(root).OrderBy(d => d, StringComparer.OrdinalIgnoreCase)) + { + string id = Path.GetFileName(bookDir); + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + JObject spec = new() { ["id"] = id }; + string yaml = Path.Combine(bookDir, "book.yaml"); + if (File.Exists(yaml)) + { + foreach (string rawLine in File.ReadAllLines(yaml, Encoding.UTF8)) + { + string line = rawLine.Trim(); + if (string.IsNullOrWhiteSpace(line) || line.StartsWith('#')) + { + continue; + } + int colon = line.IndexOf(':'); + if (colon <= 0) + { + continue; + } + string key = line[..colon].Trim(); + string val = line[(colon + 1)..].Trim().Trim('"', '\''); + if (key is "title" or "description" or "content_kind" or "language") + { + spec[key] = val; + } + } + } + string jsonl = Path.Combine(bookDir, "search.jsonl"); + spec["indexed"] = File.Exists(jsonl); + spec["content_sha"] = ReadBookContentSha(bookDir) ?? ""; + list.Add(spec); + } + return list; + } + + /// FTS lookup over attached books (filter by book_id list when provided). + public JArray LookupBooks(string query, int limit = 8, IEnumerable bookIds = null, string rating = null) + { + query = (query ?? "").Trim(); + if (query.Length < 1) + { + return []; + } + TryIndexBooks(); + int cap = Math.Clamp(limit, 1, 30); + HashSet bookFilter = null; + if (bookIds is not null) + { + bookFilter = bookIds + .Select(b => (b ?? "").Trim()) + .Where(b => !string.IsNullOrWhiteSpace(b)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + if (bookFilter.Count == 0) + { + return []; + } + } + string ratingFilter = string.IsNullOrWhiteSpace(rating) ? null : rating.Trim().ToLowerInvariant(); + List hits = []; + HashSet seen = []; + + void Add(SqliteDataReader reader) + { + string id = reader.GetString(0); + if (!seen.Add(id)) + { + return; + } + string bookId = reader.IsDBNull(1) ? "" : reader.GetString(1); + if (bookFilter is not null && !bookFilter.Contains(bookId)) + { + seen.Remove(id); + return; + } + string title = reader.IsDBNull(2) ? "" : reader.GetString(2); + string tags = reader.IsDBNull(3) ? "" : reader.GetString(3); + string text = reader.IsDBNull(4) ? "" : reader.GetString(4); + string body = reader.IsDBNull(5) ? "" : reader.GetString(5); + string rowRating = reader.IsDBNull(6) ? "" : reader.GetString(6); + JToken metaTok = new JObject(); + if (!reader.IsDBNull(7)) + { + try { metaTok = JToken.Parse(reader.GetString(7)); } catch { metaTok = new JObject(); } + } + hits.Add(new JObject + { + ["id"] = id, + ["book"] = bookId, + ["source"] = "book", + ["title"] = title, + ["tags"] = tags, + ["text"] = text.Length > 600 ? text[..600] + "…" : text, + ["body"] = body.Length > 1200 ? body[..1200] + "…" : body, + ["rating"] = rowRating, + ["meta"] = metaTok, + ["note"] = "BOOK reference — remix style/ideas, do not paste long verbatim", + }); + } + + lock (_lock) + { + EnsureOpen(); + if (!TableExists("books_rows")) + { + return []; + } + + string match = BuildFtsMatch(query); + if (!string.IsNullOrWhiteSpace(match) && TableExists("books_fts")) + { + try + { + using SqliteCommand cmd = _conn.CreateCommand(); + string sql = + """ + SELECT b.id, b.book_id, b.title, b.tags, b.text, b.body, b.rating, b.meta_json + FROM books_rows b + WHERE b.rowid IN (SELECT rowid FROM books_fts WHERE books_fts MATCH $q) + """; + if (ratingFilter is not null) + { + sql += " AND lower(b.rating) = $r"; + } + sql += " LIMIT $lim"; + cmd.CommandText = sql; + cmd.Parameters.AddWithValue("$q", match); + if (ratingFilter is not null) + { + cmd.Parameters.AddWithValue("$r", ratingFilter); + } + cmd.Parameters.AddWithValue("$lim", cap); + using SqliteDataReader reader = cmd.ExecuteReader(); + while (reader.Read() && hits.Count < cap) + { + Add(reader); + } + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory books FTS: {ex.Message}"); + } + } + + if (hits.Count < cap) + { + try + { + using SqliteCommand cmd = _conn.CreateCommand(); + string sql = + """ + SELECT id, book_id, title, tags, text, body, rating, meta_json + FROM books_rows + WHERE (title LIKE $p ESCAPE '\' OR tags LIKE $p ESCAPE '\' OR text LIKE $p ESCAPE '\' OR body LIKE $p ESCAPE '\' OR search_blob LIKE $p ESCAPE '\') + """; + if (ratingFilter is not null) + { + sql += " AND lower(rating) = $r"; + } + sql += " LIMIT $lim"; + cmd.CommandText = sql; + cmd.Parameters.AddWithValue("$p", "%" + EscapeLike(query) + "%"); + if (ratingFilter is not null) + { + cmd.Parameters.AddWithValue("$r", ratingFilter); + } + cmd.Parameters.AddWithValue("$lim", cap); + using SqliteDataReader reader = cmd.ExecuteReader(); + while (reader.Read() && hits.Count < cap) + { + Add(reader); + } + } + catch (Exception ex) + { + Logs.Debug($"AssistentMemory books LIKE: {ex.Message}"); + } + } + } + + return new JArray(hits.Take(cap)); + } +} diff --git a/AssistentMemory.Heard.cs b/AssistentMemory.Heard.cs index ffcdd03..2d4075a 100644 --- a/AssistentMemory.Heard.cs +++ b/AssistentMemory.Heard.cs @@ -44,47 +44,10 @@ public sealed partial class AssistentMemory public void SetTrainSampleAgentLinked(string id, bool linked) { - if (string.IsNullOrWhiteSpace(id)) - { - return; - } - lock (_lock) - { - EnsureTrainingReady(); - if (!HasColumn("train_samples", "agent_linked")) - { - return; - } - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = "UPDATE train_samples SET agent_linked = $v, updated_at = $u WHERE id = $id"; - cmd.Parameters.AddWithValue("$v", linked ? 1 : 0); - cmd.Parameters.AddWithValue("$u", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - cmd.Parameters.AddWithValue("$id", id.Trim()); - cmd.ExecuteNonQuery(); - } + // Training UI removed — no-op. } - public int CountAgentLinkedTrainSamples() - { - lock (_lock) - { - try - { - EnsureTrainingReady(); - } - catch - { - return 0; - } - if (!HasColumn("train_samples", "agent_linked")) - { - return 0; - } - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM train_samples WHERE agent_linked = 1"; - return Convert.ToInt32(cmd.ExecuteScalar()); - } - } + public int CountAgentLinkedTrainSamples() => 0; public async Task LinkTrainSampleToAgentAsync(string baseUrl, JObject sample, string embedModel) { diff --git a/AssistentMemory.Training.cs b/AssistentMemory.Training.cs deleted file mode 100644 index 868413f..0000000 --- a/AssistentMemory.Training.cs +++ /dev/null @@ -1,385 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Data.Sqlite; -using Newtonsoft.Json.Linq; -using SwarmUI.Utils; - -namespace Mrleo1nid.SwarmAssistent; - -/// Training datasets, samples, and job metadata in assistent.sqlite. -public sealed partial class AssistentMemory -{ - public const string KvHfDatasetCache = "hf_dataset_cache"; - - void EnsureTrainingSchema() - { - Exec( - """ - CREATE TABLE IF NOT EXISTS train_datasets ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL DEFAULT '', - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - meta_json TEXT - ); - CREATE TABLE IF NOT EXISTS train_samples ( - id TEXT PRIMARY KEY, - dataset_id TEXT NOT NULL DEFAULT 'default', - source TEXT NOT NULL DEFAULT 'manual', - chat_id TEXT, - persona TEXT, - pack TEXT, - hf_repo TEXT, - messages_json TEXT NOT NULL DEFAULT '[]', - status TEXT NOT NULL DEFAULT 'draft', - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - FOREIGN KEY(dataset_id) REFERENCES train_datasets(id) ON DELETE CASCADE - ); - CREATE INDEX IF NOT EXISTS idx_train_samples_status ON train_samples(status); - CREATE INDEX IF NOT EXISTS idx_train_samples_dataset ON train_samples(dataset_id); - CREATE TABLE IF NOT EXISTS train_jobs ( - id TEXT PRIMARY KEY, - kind TEXT NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - config_json TEXT, - base_model TEXT, - output_name TEXT, - log_path TEXT, - progress_json TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - finished_at INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_train_jobs_status ON train_jobs(status); - """); - if (!HasColumn("train_samples", "agent_linked")) - { - Exec("ALTER TABLE train_samples ADD COLUMN agent_linked INTEGER NOT NULL DEFAULT 0"); - } - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = "INSERT OR IGNORE INTO train_datasets(id, title, created_at, updated_at) VALUES('default', 'Default', $u, $u)"; - cmd.Parameters.AddWithValue("$u", DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); - cmd.ExecuteNonQuery(); - } - - public List ListTrainSamples(string status = null, string persona = null, string datasetId = null, int limit = 200) - { - lock (_lock) - { - EnsureTrainingReady(); - int take = Math.Clamp(limit, 1, 2000); - List where = []; - if (!string.IsNullOrWhiteSpace(status) && !string.Equals(status, "all", StringComparison.OrdinalIgnoreCase)) - { - where.Add("status = $status"); - } - if (!string.IsNullOrWhiteSpace(persona) && !string.Equals(persona, "all", StringComparison.OrdinalIgnoreCase)) - { - where.Add("persona = $persona"); - } - if (!string.IsNullOrWhiteSpace(datasetId)) - { - where.Add("dataset_id = $ds"); - } - bool hasAgentLinked = HasColumn("train_samples", "agent_linked"); - string sql = hasAgentLinked - ? "SELECT id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at, agent_linked FROM train_samples" - : "SELECT id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at FROM train_samples"; - if (where.Count > 0) - { - sql += " WHERE " + string.Join(" AND ", where); - } - sql += " ORDER BY updated_at DESC LIMIT $lim"; - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = sql; - if (where.Any(w => w.Contains("$status"))) - { - cmd.Parameters.AddWithValue("$status", status.Trim()); - } - if (where.Any(w => w.Contains("$persona"))) - { - cmd.Parameters.AddWithValue("$persona", persona.Trim()); - } - if (where.Any(w => w.Contains("$ds"))) - { - cmd.Parameters.AddWithValue("$ds", datasetId.Trim()); - } - cmd.Parameters.AddWithValue("$lim", take); - List list = []; - using SqliteDataReader r = cmd.ExecuteReader(); - while (r.Read()) - { - list.Add(ReadTrainSampleRow(r)); - } - return list; - } - } - - static JObject ReadTrainSampleRow(SqliteDataReader r) - { - JArray messages = []; - try - { - messages = JArray.Parse(r.GetString(7)); - } - catch - { - // ignore - } - bool hasAgentLinked = r.FieldCount > 11; - return new JObject - { - ["id"] = r.GetString(0), - ["dataset_id"] = r.GetString(1), - ["source"] = r.GetString(2), - ["chat_id"] = r.IsDBNull(3) ? null : r.GetString(3), - ["persona"] = r.IsDBNull(4) ? null : r.GetString(4), - ["pack"] = r.IsDBNull(5) ? null : r.GetString(5), - ["hf_repo"] = r.IsDBNull(6) ? null : r.GetString(6), - ["messages"] = messages, - ["status"] = r.GetString(8), - ["createdAt"] = r.GetInt64(9), - ["updatedAt"] = r.GetInt64(10), - ["agent_linked"] = hasAgentLinked && !r.IsDBNull(11) && r.GetInt64(11) != 0, - }; - } - - public JObject GetTrainSample(string id) - { - if (string.IsNullOrWhiteSpace(id)) - { - return null; - } - lock (_lock) - { - EnsureOpen(); - bool hasAgentLinked = HasColumn("train_samples", "agent_linked"); - string sql = hasAgentLinked - ? "SELECT id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at, agent_linked FROM train_samples WHERE id = $id LIMIT 1" - : "SELECT id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at FROM train_samples WHERE id = $id LIMIT 1"; - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = sql; - cmd.Parameters.AddWithValue("$id", id.Trim()); - using SqliteDataReader r = cmd.ExecuteReader(); - return r.Read() ? ReadTrainSampleRow(r) : null; - } - } - - public int ImportTrainSamplesBatch(IEnumerable samples) - { - lock (_lock) - { - EnsureTrainingReady(); - int imported = 0; - using SqliteTransaction tx = _conn.BeginTransaction(); - foreach (JObject sample in samples) - { - if (sample is null) - { - continue; - } - UpsertTrainSampleCore(sample, tx); - imported++; - } - tx.Commit(); - return imported; - } - } - - public JObject UpsertTrainSample(JObject sample) - { - lock (_lock) - { - EnsureTrainingReady(); - return UpsertTrainSampleCore(sample, null); - } - } - - JObject UpsertTrainSampleCore(JObject sample, SqliteTransaction tx) - { - long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - string id = sample["id"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(id)) - { - id = $"ts_{now}_{Guid.NewGuid():N}"[..24]; - } - string datasetId = sample["dataset_id"]?.ToString()?.Trim() ?? "default"; - JArray messages = sample["messages"] as JArray ?? []; - using SqliteCommand cmd = _conn.CreateCommand(); - if (tx is not null) - { - cmd.Transaction = tx; - } - cmd.CommandText = - """ - INSERT INTO train_samples(id, dataset_id, source, chat_id, persona, pack, hf_repo, messages_json, status, created_at, updated_at) - VALUES($id, $ds, $src, $chat, $persona, $pack, $hf, $msg, $status, $c, $u) - ON CONFLICT(id) DO UPDATE SET - dataset_id = excluded.dataset_id, - source = excluded.source, - chat_id = excluded.chat_id, - persona = excluded.persona, - pack = excluded.pack, - hf_repo = excluded.hf_repo, - messages_json = excluded.messages_json, - status = excluded.status, - updated_at = excluded.updated_at - """; - cmd.Parameters.AddWithValue("$id", id); - cmd.Parameters.AddWithValue("$ds", datasetId); - cmd.Parameters.AddWithValue("$src", sample["source"]?.ToString() ?? "manual"); - cmd.Parameters.AddWithValue("$chat", (object)sample["chat_id"]?.ToString() ?? DBNull.Value); - cmd.Parameters.AddWithValue("$persona", (object)sample["persona"]?.ToString() ?? DBNull.Value); - cmd.Parameters.AddWithValue("$pack", (object)sample["pack"]?.ToString() ?? DBNull.Value); - cmd.Parameters.AddWithValue("$hf", (object)sample["hf_repo"]?.ToString() ?? DBNull.Value); - cmd.Parameters.AddWithValue("$msg", messages.ToString(Newtonsoft.Json.Formatting.None)); - cmd.Parameters.AddWithValue("$status", sample["status"]?.ToString() ?? "draft"); - long created = sample["createdAt"]?.Value() ?? now; - cmd.Parameters.AddWithValue("$c", created); - cmd.Parameters.AddWithValue("$u", now); - cmd.ExecuteNonQuery(); - return new JObject { ["id"] = id, ["updatedAt"] = now }; - } - - public bool DeleteTrainSample(string id) - { - lock (_lock) - { - EnsureTrainingReady(); - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = "DELETE FROM train_samples WHERE id = $id"; - cmd.Parameters.AddWithValue("$id", id ?? ""); - return cmd.ExecuteNonQuery() > 0; - } - } - - public int CountTrainSamples(string status = null) - { - lock (_lock) - { - EnsureTrainingReady(); - using SqliteCommand cmd = _conn.CreateCommand(); - if (string.IsNullOrWhiteSpace(status) || string.Equals(status, "all", StringComparison.OrdinalIgnoreCase)) - { - cmd.CommandText = "SELECT COUNT(*) FROM train_samples"; - } - else - { - cmd.CommandText = "SELECT COUNT(*) FROM train_samples WHERE status = $s"; - cmd.Parameters.AddWithValue("$s", status.Trim()); - } - return Convert.ToInt32(cmd.ExecuteScalar()); - } - } - - public JObject SaveTrainJob(JObject job) - { - lock (_lock) - { - EnsureOpen(); - long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - string id = job["id"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(id)) - { - id = $"tj_{now}_{Guid.NewGuid():N}"[..24]; - } - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = - """ - INSERT INTO train_jobs(id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at) - VALUES($id, $kind, $status, $cfg, $base, $out, $log, $prog, $c, $u, $f) - ON CONFLICT(id) DO UPDATE SET - status = excluded.status, - config_json = excluded.config_json, - log_path = excluded.log_path, - progress_json = excluded.progress_json, - updated_at = excluded.updated_at, - finished_at = excluded.finished_at - """; - cmd.Parameters.AddWithValue("$id", id); - cmd.Parameters.AddWithValue("$kind", job["kind"]?.ToString() ?? "qlora"); - cmd.Parameters.AddWithValue("$status", job["status"]?.ToString() ?? "pending"); - cmd.Parameters.AddWithValue("$cfg", job["config"]?.ToString(Newtonsoft.Json.Formatting.None) ?? job["config_json"]?.ToString() ?? "{}"); - cmd.Parameters.AddWithValue("$base", (object)job["base_model"]?.ToString() ?? DBNull.Value); - cmd.Parameters.AddWithValue("$out", (object)job["output_name"]?.ToString() ?? DBNull.Value); - cmd.Parameters.AddWithValue("$log", (object)job["log_path"]?.ToString() ?? DBNull.Value); - cmd.Parameters.AddWithValue("$prog", (object)(job["progress"]?.ToString(Newtonsoft.Json.Formatting.None) ?? job["progress_json"]?.ToString()) ?? DBNull.Value); - cmd.Parameters.AddWithValue("$c", job["created_at"]?.Value() ?? job["createdAt"]?.Value() ?? now); - cmd.Parameters.AddWithValue("$u", now); - cmd.Parameters.AddWithValue("$f", (object)(job["finished_at"]?.Value() ?? job["finishedAt"]?.Value()) ?? DBNull.Value); - cmd.ExecuteNonQuery(); - return new JObject { ["id"] = id }; - } - } - - public JObject GetTrainJob(string id) - { - lock (_lock) - { - EnsureTrainingReady(); - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = "SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at FROM train_jobs WHERE id = $id"; - cmd.Parameters.AddWithValue("$id", id ?? ""); - using SqliteDataReader r = cmd.ExecuteReader(); - if (!r.Read()) - { - return null; - } - return ReadTrainJobRow(r); - } - } - - public JObject GetLastTrainJob() - { - lock (_lock) - { - EnsureTrainingReady(); - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = - "SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at " - + "FROM train_jobs ORDER BY updated_at DESC LIMIT 1"; - using SqliteDataReader r = cmd.ExecuteReader(); - if (!r.Read()) - { - return null; - } - return ReadTrainJobRow(r); - } - } - - static JObject ReadTrainJobRow(SqliteDataReader r) - { - return new JObject - { - ["id"] = r.GetString(0), - ["kind"] = r.GetString(1), - ["status"] = r.GetString(2), - ["config_json"] = r.IsDBNull(3) ? null : r.GetString(3), - ["base_model"] = r.IsDBNull(4) ? null : r.GetString(4), - ["output_name"] = r.IsDBNull(5) ? null : r.GetString(5), - ["log_path"] = r.IsDBNull(6) ? null : r.GetString(6), - ["progress_json"] = r.IsDBNull(7) ? null : r.GetString(7), - ["created_at"] = r.GetInt64(8), - ["updated_at"] = r.GetInt64(9), - ["finished_at"] = r.IsDBNull(10) ? null : r.GetInt64(10), - }; - } - - public JObject GetActiveTrainJob() - { - lock (_lock) - { - EnsureTrainingReady(); - using SqliteCommand cmd = _conn.CreateCommand(); - cmd.CommandText = "SELECT id, kind, status, config_json, base_model, output_name, log_path, progress_json, created_at, updated_at, finished_at FROM train_jobs WHERE status IN ('pending','running') ORDER BY updated_at DESC LIMIT 1"; - using SqliteDataReader r = cmd.ExecuteReader(); - if (!r.Read()) - { - return null; - } - return ReadTrainJobRow(r); - } - } -} diff --git a/AssistentMemory.cs b/AssistentMemory.cs index 0142e7f..5d7c282 100644 --- a/AssistentMemory.cs +++ b/AssistentMemory.cs @@ -144,14 +144,6 @@ public sealed partial class AssistentMemory : IDisposable Logs.Debug($"AssistentMemory store schema: {ex.Message}"); } try - { - EnsureTrainingSchema(); - } - catch (Exception ex) - { - Logs.Error($"AssistentMemory training schema failed: {ex.Message}"); - } - try { EnsureUserPrefsSchema(); } @@ -193,17 +185,8 @@ public sealed partial class AssistentMemory : IDisposable internal void EnsureTrainingReady() { + // Training UI removed in 0.16 — heard examples live in memories only. EnsureOpen(); - if (!HasTable("train_samples")) - { - EnsureTrainingSchema(); - } - if (!HasTable("train_samples")) - { - throw new InvalidOperationException( - "Assistent training database unavailable (train_samples). " - + "Run gpu-rent seed-extensions and restart SwarmUI (≥0.15.6)."); - } } void MigratePersonaColumn() @@ -443,6 +426,7 @@ public sealed partial class AssistentMemory : IDisposable { TryIndexTags(); TryIndexExamples(); + TryIndexBooks(); return; } @@ -478,6 +462,7 @@ public sealed partial class AssistentMemory : IDisposable Logs.Debug($"AssistentMemory seed defer (embed unavailable): {ex.Message}"); TryIndexTags(); TryIndexExamples(); + TryIndexBooks(); return; } diff --git a/AssistentOllama.cs b/AssistentOllama.cs index 93e4975..8278157 100644 --- a/AssistentOllama.cs +++ b/AssistentOllama.cs @@ -339,7 +339,7 @@ public partial class SwarmAssistentExtension string embedModel = raw?["embed_model"]?.ToString() ?? Config.LoadSettings()["embed_model"]?.ToString(); try { - (string reply, JObject parsed, JArray civitai, int systemChars, JObject systemLayers) = await RunChatWithHops( + (string reply, JObject parsed, JArray civitai, JObject knowledge, int systemChars, JObject systemLayers) = await RunChatWithHops( session, root, modelName, packName, includeBase, contextJson, userMessages, personaId: persona, skillIds: skills, embedModel: embedModel); JObject result = new() { @@ -350,6 +350,7 @@ public partial class SwarmAssistentExtension ["persona"] = persona, ["raw"] = parsed, ["civitai_results"] = civitai, + ["knowledge"] = knowledge, ["system_chars"] = systemChars, ["system_layers"] = systemLayers, }; @@ -409,7 +410,7 @@ public partial class SwarmAssistentExtension }, API.WebsocketTimeout); } } - (string reply, JObject parsed, JArray civitai, int systemChars, JObject systemLayers) = await RunChatWithHops( + (string reply, JObject parsed, JArray civitai, JObject knowledge, int systemChars, JObject systemLayers) = await RunChatWithHops( session, root, modelName, packName, includeBase, contextJson, userMessages, OnDelta, OnHopStart, persona, skills, embedModel); JObject done = new() { @@ -421,6 +422,7 @@ public partial class SwarmAssistentExtension ["persona"] = persona, ["raw"] = parsed, ["civitai_results"] = civitai, + ["knowledge"] = knowledge, ["system_chars"] = systemChars, ["system_layers"] = systemLayers, }; diff --git a/AssistentPatch.cs b/AssistentPatch.cs index 667f198..f70fe01 100644 --- a/AssistentPatch.cs +++ b/AssistentPatch.cs @@ -313,6 +313,10 @@ public partial class SwarmAssistentExtension { return "ask_inventory"; } + if ((AskContains(patch, "knowledge") || ActionsContain(patch, "lookup_knowledge")) && !Skip("ask_knowledge")) + { + return "ask_knowledge"; + } if ((AskContains(patch, "examples") || ActionsContain(patch, "lookup_examples")) && !Skip("ask_examples")) { return "ask_examples"; diff --git a/AssistentTraining.Agent.cs b/AssistentTraining.Agent.cs deleted file mode 100644 index 3506a03..0000000 --- a/AssistentTraining.Agent.cs +++ /dev/null @@ -1,181 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using SwarmUI.Accounts; -using SwarmUI.Utils; - -namespace Mrleo1nid.SwarmAssistent; - -/// Link training dataset samples to the live agent as retrievable "heard" examples. -public partial class SwarmAssistentExtension -{ - string MemoryEmbedForTraining(string personaId = null) - { - string pid = AssistentConfig.SafeId(personaId) ?? Config.DefaultPersonaId(); - return Config.LoadSettings()["embed_model"]?.ToString() - ?? Config.LoadAssistant(pid)["embed_model"]?.ToString() - ?? "nomic-embed-text"; - } - - string MemoryBaseForTraining(JObject raw = null) - => MemoryBaseUrl(raw?["base_url"]?.ToString()); - - public async Task AssistentGetDatasetAgentSettings(Session session) - { - await Task.CompletedTask; - try - { - JObject settings = Config.LoadTrainingAgent(); - return new JObject - { - ["success"] = true, - ["settings"] = settings, - ["linked"] = Memory.CountAgentLinkedTrainSamples(), - ["approved"] = Memory.CountTrainSamples("approved"), - }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message }; - } - } - - public async Task AssistentSaveDatasetAgentSettings(Session session, JObject settings) - { - await Task.CompletedTask; - if (settings is null) - { - return new JObject { ["error"] = "settings required" }; - } - Config.SaveTrainingAgent(settings); - return new JObject { ["success"] = true, ["settings"] = Config.LoadTrainingAgent() }; - } - - public async Task AssistentLinkTrainSampleToAgent(Session session, string id, JObject raw = null) - { - if (string.IsNullOrWhiteSpace(id)) - { - return new JObject { ["error"] = "id required" }; - } - JObject sample = Memory.GetTrainSample(id.Trim()); - if (sample is null) - { - return new JObject { ["error"] = "sample not found" }; - } - if (!string.Equals(sample["status"]?.ToString(), "approved", StringComparison.OrdinalIgnoreCase)) - { - return new JObject { ["error"] = "only approved samples can be linked to the agent" }; - } - try - { - string embed = MemoryEmbedForTraining(sample["persona"]?.ToString()); - bool ok = await Memory.LinkTrainSampleToAgentAsync(MemoryBaseForTraining(raw), sample, embed); - return new JObject { ["success"] = ok, ["linked"] = Memory.CountAgentLinkedTrainSamples() }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message }; - } - } - - public async Task AssistentUnlinkTrainSampleFromAgent(Session session, string id) - { - await Task.CompletedTask; - if (string.IsNullOrWhiteSpace(id)) - { - return new JObject { ["error"] = "id required" }; - } - JObject sample = Memory.GetTrainSample(id.Trim()); - if (sample is null) - { - return new JObject { ["error"] = "sample not found" }; - } - Memory.UnlinkTrainSampleFromAgent(sample); - return new JObject { ["success"] = true, ["linked"] = Memory.CountAgentLinkedTrainSamples() }; - } - - public async Task AssistentSyncDatasetToAgent(Session session, JObject raw = null) - { - bool approvedOnly = raw?["approved_only"]?.Value() ?? true; - bool relink = raw?["relink"]?.Value() ?? false; - string persona = raw?["persona"]?.ToString(); - string status = approvedOnly ? "approved" : "all"; - List samples = Memory.ListTrainSamples(status, persona, null, 2000); - string baseUrl = MemoryBaseForTraining(raw); - int linked = 0; - int skipped = 0; - List errors = []; - foreach (JObject sample in samples) - { - bool already = sample["agent_linked"]?.Value() ?? false; - if (already && !relink) - { - skipped++; - continue; - } - if (!string.Equals(sample["status"]?.ToString(), "approved", StringComparison.OrdinalIgnoreCase)) - { - skipped++; - continue; - } - try - { - string embed = MemoryEmbedForTraining(sample["persona"]?.ToString()); - if (await Memory.LinkTrainSampleToAgentAsync(baseUrl, sample, embed)) - { - linked++; - } - else - { - skipped++; - } - } - catch (Exception ex) - { - errors.Add($"{sample["id"]}: {ex.Message}"); - } - } - return new JObject - { - ["success"] = true, - ["linked_now"] = linked, - ["skipped"] = skipped, - ["total_linked"] = Memory.CountAgentLinkedTrainSamples(), - ["errors"] = new JArray(errors.Take(8)), - }; - } - - async Task TryAutoLinkTrainSample(Session session, JObject sample, JObject raw = null) - { - if (sample is null || Memory is null) - { - return; - } - JObject agent = Config.LoadTrainingAgent(); - if (agent["enabled"]?.Value() == false) - { - return; - } - if (agent["auto_link_on_approve"]?.Value() == false) - { - return; - } - string status = sample["status"]?.ToString() ?? ""; - if (!string.Equals(status, "approved", StringComparison.OrdinalIgnoreCase)) - { - Memory.UnlinkTrainSampleFromAgent(sample); - return; - } - try - { - string embed = MemoryEmbedForTraining(sample["persona"]?.ToString()); - await Memory.LinkTrainSampleToAgentAsync(MemoryBaseForTraining(raw), sample, embed); - } - catch (Exception ex) - { - Logs.Debug($"TryAutoLinkTrainSample: {ex.Message}"); - } - } -} diff --git a/AssistentTraining.cs b/AssistentTraining.cs deleted file mode 100644 index fc04e3c..0000000 --- a/AssistentTraining.cs +++ /dev/null @@ -1,456 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using SwarmUI.Accounts; -using SwarmUI.Utils; - -namespace Mrleo1nid.SwarmAssistent; - -/// Training samples, dataset import/export, Ollama Modelfile builder. -public partial class SwarmAssistentExtension -{ - static string TrainingRoot() - { - string root = Path.Combine(DataRoot(), "Assistent", "training"); - Directory.CreateDirectory(root); - Directory.CreateDirectory(Path.Combine(root, "datasets")); - Directory.CreateDirectory(Path.Combine(root, "jobs")); - Directory.CreateDirectory(Path.Combine(root, "adapters")); - return root; - } - - public async Task AssistentListTrainSamples(Session session, string status = null, string persona = null, int limit = 200) - { - await Task.CompletedTask; - try - { - List list = Memory.ListTrainSamples(status, persona, null, limit); - return new JObject - { - ["success"] = true, - ["samples"] = new JArray(list), - ["approved"] = Memory.CountTrainSamples("approved"), - ["draft"] = Memory.CountTrainSamples("draft"), - ["total"] = Memory.CountTrainSamples(null), - }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message }; - } - } - - public async Task AssistentUpsertTrainSample(Session session, JObject raw) - { - await Task.CompletedTask; - if (raw is null) - { - return new JObject { ["error"] = "body required" }; - } - try - { - JObject saved = Memory.UpsertTrainSample(raw); - JObject full = Memory.GetTrainSample(saved["id"]?.ToString()) ?? raw; - await TryAutoLinkTrainSample(session, full, raw); - return new JObject { ["success"] = true, ["sample"] = full }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message }; - } - } - - public async Task AssistentDeleteTrainSample(Session session, string id) - { - await Task.CompletedTask; - if (string.IsNullOrWhiteSpace(id)) - { - return new JObject { ["error"] = "id required" }; - } - JObject sample = Memory.GetTrainSample(id.Trim()); - bool ok = Memory.DeleteTrainSample(id.Trim()); - if (ok && sample is not null) - { - Memory.UnlinkTrainSampleFromAgent(sample); - } - return new JObject { ["success"] = true, ["deleted"] = ok }; - } - - public async Task AssistentBuildDatasetFromChats(Session session, bool approved_only = false) - { - await Task.CompletedTask; - try - { - List chats = Memory.ListChats(withMessages: true, limit: AssistentMemory.MaxChatsStored); - int added = 0; - foreach (JObject chat in chats) - { - JArray messages = chat["messages"] as JArray ?? []; - for (int i = 0; i < messages.Count - 1; i++) - { - if (messages[i] is not JObject u || messages[i + 1] is not JObject a) - { - continue; - } - if (!string.Equals(u["role"]?.ToString(), "user", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - if (!string.Equals(a["role"]?.ToString(), "assistant", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - Memory.UpsertTrainSample(new JObject - { - ["source"] = "chat", - ["chat_id"] = chat["id"], - ["persona"] = a["persona"] ?? u["persona"], - ["pack"] = a["pack"] ?? u["pack"], - ["status"] = approved_only ? "approved" : "draft", - ["messages"] = new JArray { u.DeepClone(), a.DeepClone() }, - }); - added++; - } - } - return new JObject { ["success"] = true, ["added"] = added }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message }; - } - } - - public async Task AssistentImportDataset(Session session, JObject raw) - { - await Task.CompletedTask; - string content = raw?["content"]?.ToString(); - string format = raw?["format"]?.ToString() ?? "auto"; - if (string.IsNullOrWhiteSpace(content)) - { - return new JObject { ["error"] = "content required" }; - } - try - { - int imported = 0; - string fmt = (format ?? "auto").Trim().ToLowerInvariant(); - List records = ParseDatasetContent(content, fmt); - foreach (JObject rec in records) - { - JArray messages = rec["messages"] as JArray; - if (messages is null || messages.Count == 0) - { - continue; - } - Memory.UpsertTrainSample(new JObject - { - ["source"] = "import", - ["messages"] = messages, - ["status"] = "draft", - }); - imported++; - } - return new JObject { ["success"] = true, ["imported"] = imported }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message }; - } - } - - static List ParseDatasetContent(string content, string format) - { - List list = []; - string trimmed = content.Trim(); - if (trimmed.StartsWith('[')) - { - JArray arr = JArray.Parse(trimmed); - foreach (JToken t in arr) - { - if (t is JObject o) - { - JArray msgs = ExtractMessagesFromRecord(o); - if (msgs != null) - { - list.Add(new JObject { ["messages"] = msgs }); - } - } - } - return list; - } - if (format == "csv" || LooksLikeCsv(trimmed)) - { - return ParseCsvDataset(trimmed); - } - foreach (string line in trimmed.Split('\n')) - { - string ln = line.Trim(); - if (string.IsNullOrWhiteSpace(ln)) - { - continue; - } - try - { - JObject o = JObject.Parse(ln); - JArray msgs = ExtractMessagesFromRecord(o); - if (msgs != null) - { - list.Add(new JObject { ["messages"] = msgs }); - } - } - catch - { - // skip bad line - } - } - return list; - } - - static bool LooksLikeCsv(string s) => s.Contains(',') && s.Contains('\n') && !s.TrimStart().StartsWith('{'); - - static List ParseCsvDataset(string csv) - { - List list = []; - string[] lines = csv.Split('\n').Select(l => l.Trim()).Where(l => l.Length > 0).ToArray(); - if (lines.Length < 2) - { - return list; - } - string[] headers = lines[0].Split(',').Select(h => h.Trim().Trim('"')).ToArray(); - int promptIdx = Array.FindIndex(headers, h => h.Equals("prompt", StringComparison.OrdinalIgnoreCase) || h.Equals("question", StringComparison.OrdinalIgnoreCase) || h.Equals("instruction", StringComparison.OrdinalIgnoreCase)); - int respIdx = Array.FindIndex(headers, h => h.Equals("response", StringComparison.OrdinalIgnoreCase) || h.Equals("answer", StringComparison.OrdinalIgnoreCase) || h.Equals("output", StringComparison.OrdinalIgnoreCase) || h.Equals("completion", StringComparison.OrdinalIgnoreCase)); - if (promptIdx < 0 || respIdx < 0) - { - return list; - } - for (int i = 1; i < lines.Length; i++) - { - string[] cols = SplitCsvLine(lines[i]); - if (cols.Length <= Math.Max(promptIdx, respIdx)) - { - continue; - } - list.Add(new JObject - { - ["messages"] = new JArray - { - new JObject { ["role"] = "user", ["content"] = cols[promptIdx] }, - new JObject { ["role"] = "assistant", ["content"] = cols[respIdx] }, - }, - }); - } - return list; - } - - static string[] SplitCsvLine(string line) - { - List parts = []; - StringBuilder cur = new(); - bool inQ = false; - foreach (char c in line) - { - if (c == '"') - { - inQ = !inQ; - continue; - } - if (c == ',' && !inQ) - { - parts.Add(cur.ToString().Trim()); - cur.Clear(); - continue; - } - cur.Append(c); - } - parts.Add(cur.ToString().Trim()); - return parts.ToArray(); - } - - static JArray ExtractMessagesFromRecord(JObject o) - { - if (o["messages"] is JArray msgs) - { - return NormalizeMessagesArray(msgs); - } - if (o["conversations"] is JArray conv) - { - return ConvertHfRowToMessages(new JObject { ["conversations"] = conv }, new JObject { ["kind"] = "conversations" }, null); - } - if (o["instruction"] != null && o["output"] != null) - { - return ConvertHfRowToMessages(o, new JObject { ["kind"] = "alpaca" }, null); - } - if (o["prompt"] != null && (o["response"] != null || o["completion"] != null)) - { - return ConvertHfRowToMessages(o, new JObject { ["kind"] = "prompt_response", ["response_col"] = o["response"] != null ? "response" : "completion" }, null); - } - return null; - } - - public async Task AssistentExportDataset(Session session, string status = "approved", string format = "jsonl") - { - await Task.CompletedTask; - try - { - List samples = Memory.ListTrainSamples(status, null, null, 5000); - StringBuilder sb = new(); - int exported = 0; - foreach (JObject s in samples) - { - JArray messages = s["messages"] as JArray ?? []; - if (messages.Count == 0) - { - continue; - } - exported++; - if (string.Equals(format, "sharegpt", StringComparison.OrdinalIgnoreCase)) - { - JArray conv = []; - foreach (JToken m in messages) - { - if (m is not JObject mo) - { - continue; - } - string role = mo["role"]?.ToString() ?? "user"; - conv.Add(new JObject - { - ["from"] = role == "assistant" ? "gpt" : "human", - ["value"] = mo["content"]?.ToString() ?? "", - }); - } - sb.AppendLine(new JObject { ["conversations"] = conv }.ToString(Newtonsoft.Json.Formatting.None)); - } - else - { - sb.AppendLine(new JObject { ["messages"] = messages }.ToString(Newtonsoft.Json.Formatting.None)); - } - } - string path = Path.Combine(TrainingRoot(), "datasets", $"export_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}.jsonl"); - await File.WriteAllTextAsync(path, sb.ToString(), Encoding.UTF8); - return new JObject { ["success"] = true, ["path"] = path, ["count"] = exported, ["content"] = sb.ToString() }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message }; - } - } - - public async Task AssistentCreateOllamaModel(Session session, JObject raw) - { - if (raw is null) - { - return new JObject { ["error"] = "body required" }; - } - string baseUrl = NormalizeBaseUrl(raw["base_url"]?.ToString()); - string baseModel = raw["base_model"]?.ToString()?.Trim(); - string name = raw["name"]?.ToString()?.Trim(); - string system = raw["system"]?.ToString() ?? ""; - int shots = raw["shots"]?.Value() ?? 8; - if (string.IsNullOrWhiteSpace(baseModel) || string.IsNullOrWhiteSpace(name)) - { - return new JObject { ["error"] = "base_model and name required" }; - } - if (string.IsNullOrWhiteSpace(system)) - { - string persona = AssistentConfig.SafeId(raw["persona"]?.ToString()) ?? Config.DefaultPersonaId(); - system = Config.LoadCorePrompt(persona) + "\n\n" + Config.RenderIdentityBlock(persona, includeAllShelves: true); - } - StringBuilder mf = new(); - mf.AppendLine($"FROM {baseModel}"); - mf.AppendLine($"SYSTEM \"\"\"{system}\"\"\""); - List samples = Memory.ListTrainSamples("approved", null, null, Math.Clamp(shots, 0, 32)); - foreach (JObject s in samples.Take(shots)) - { - JArray messages = s["messages"] as JArray ?? []; - foreach (JToken m in messages) - { - if (m is not JObject mo) - { - continue; - } - string role = mo["role"]?.ToString() ?? "user"; - string content = mo["content"]?.ToString() ?? ""; - if (string.IsNullOrWhiteSpace(content)) - { - continue; - } - mf.AppendLine($"MESSAGE {role} \"\"\"{content.Replace("\"\"\"", "\"\"\"\"\"\"\"")}\"\"\""); - } - } - if (raw["num_ctx"] != null) - { - mf.AppendLine($"PARAMETER num_ctx {raw["num_ctx"]}"); - } - if (raw["temperature"] != null) - { - mf.AppendLine($"PARAMETER temperature {raw["temperature"]}"); - } - string modelfilePath = Path.Combine(TrainingRoot(), "jobs", $"modelfile_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}.Modelfile"); - Directory.CreateDirectory(Path.GetDirectoryName(modelfilePath)!); - await File.WriteAllTextAsync(modelfilePath, mf.ToString(), Encoding.UTF8); - try - { - JObject payload = new() - { - ["name"] = name, - ["modelfile"] = mf.ToString(), - ["stream"] = false, - }; - using StringContent content = new(payload.ToString(Newtonsoft.Json.Formatting.None), Encoding.UTF8, "application/json"); - using HttpResponseMessage resp = await HttpClient.PostAsync($"{baseUrl}/api/create", content); - string body = await resp.Content.ReadAsStringAsync(); - if (!resp.IsSuccessStatusCode) - { - return new JObject { ["error"] = $"Ollama create HTTP {(int)resp.StatusCode}: {Clip(body, 400)}", ["modelfile_path"] = modelfilePath }; - } - return new JObject { ["success"] = true, ["name"] = name, ["modelfile_path"] = modelfilePath, ["ollama"] = body }; - } - catch (Exception ex) - { - return new JObject { ["error"] = ex.Message, ["modelfile_path"] = modelfilePath }; - } - } - - public async Task AssistentGetTrainJob(Session session, string id = null) - { - await Task.CompletedTask; - JObject job = string.IsNullOrWhiteSpace(id) ? Memory.GetActiveTrainJob() : Memory.GetTrainJob(id); - JObject lastJob = Memory.GetLastTrainJob(); - if (job is not null && TrainingJobManager.IsRunning && string.Equals(job["id"]?.ToString(), TrainingJobManager.CurrentJobId, StringComparison.OrdinalIgnoreCase)) - { - JObject live = TrainingJobManager.GetProgress(); - job["progress_json"] = live.ToString(Newtonsoft.Json.Formatting.None); - job["status"] = live["status"]?.ToString() ?? job["status"]; - } - return new JObject - { - ["success"] = true, - ["job"] = job, - ["last_job"] = lastJob, - ["training_active"] = TrainingJobManager.IsRunning, - ["progress"] = TrainingJobManager.IsRunning ? TrainingJobManager.GetProgress() : null, - }; - } - - public async Task AssistentSaveRunnerSettings(Session session, JObject settings) - { - await Task.CompletedTask; - if (settings is null) - { - return new JObject { ["error"] = "settings required" }; - } - Config.SaveTrainingRunner(settings); - return new JObject { ["success"] = true }; - } - - public async Task AssistentGetRunnerSettings(Session session) - { - await Task.CompletedTask; - return new JObject { ["success"] = true, ["settings"] = Config.LoadTrainingRunner() }; - } -} diff --git a/AssistentTrainingJobs.cs b/AssistentTrainingJobs.cs deleted file mode 100644 index c9bee12..0000000 --- a/AssistentTrainingJobs.cs +++ /dev/null @@ -1,607 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.WebSockets; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Newtonsoft.Json.Linq; -using SwarmUI.Accounts; -using SwarmUI.Utils; -using SwarmUI.WebAPI; - -namespace Mrleo1nid.SwarmAssistent; - -/// QLoRA training job runner with VRAM lock and progress streaming. -public partial class SwarmAssistentExtension -{ - static readonly TrainingJobManager TrainingJobManager = new(); - - public async Task AssistentStartTrainJob(Session session, JObject raw) - { - if (raw is null) - { - return new JObject { ["error"] = "body required" }; - } - if (TrainingJobManager.IsRunning) - { - return new JObject { ["error"] = "Тренировка уже идёт" }; - } - string hfBase = raw["base_model"]?.ToString()?.Trim(); - string outputName = raw["output_name"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(hfBase) || string.IsNullOrWhiteSpace(outputName)) - { - return new JObject { ["error"] = "base_model and output_name required" }; - } - string hfDataset = null; - JObject hfCheck = null; - JObject hfMapping = raw["hf_mapping"] as JObject; - if (!string.IsNullOrWhiteSpace(raw["hf_dataset"]?.ToString())) - { - hfDataset = NormalizeHfDatasetId(raw["hf_dataset"]?.ToString()); - if (hfDataset is null) - { - return new JObject { ["error"] = "invalid hf_dataset id" }; - } - hfCheck = await CheckHfDatasetInternal(session, hfDataset, useCache: true); - if (hfCheck["gate"]?.ToString() == "rejected") - { - return new JObject { ["error"] = hfCheck["reason"]?.ToString() ?? "hf dataset rejected" }; - } - hfMapping = ResolveHfMapping(hfCheck, hfMapping); - if (MappingRequired(hfCheck, hfMapping)) - { - return new JObject { ["error"] = "Нужен маппинг колонок для HF набора", ["check"] = hfCheck }; - } - } - JObject runner = Config.LoadTrainingRunner(); - string kind = runner["kind"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(kind)) - { - kind = "builtin"; - } - string baseUrl = NormalizeBaseUrl(raw["base_url"]?.ToString()); - string chatModel = raw["chat_model"]?.ToString()?.Trim(); - if (!string.IsNullOrWhiteSpace(chatModel)) - { - await AssistentParkLlm(session, baseUrl, chatModel); - } - string datasetPath = null; - int exportCount = 0; - if (string.IsNullOrWhiteSpace(hfDataset)) - { - JObject export = await AssistentExportDataset(session, "approved", "jsonl"); - datasetPath = export["path"]?.ToString(); - exportCount = export["count"]?.Value() ?? 0; - if (string.IsNullOrWhiteSpace(datasetPath) || !File.Exists(datasetPath) || exportCount <= 0) - { - return new JObject { ["error"] = "Нет одобренных примеров для тренировки (или укажи hf_dataset)" }; - } - } - else - { - JObject export = await AssistentExportDataset(session, "approved", "jsonl"); - exportCount = export["count"]?.Value() ?? 0; - if (exportCount > 0 && File.Exists(export["path"]?.ToString() ?? "")) - { - datasetPath = export["path"]?.ToString(); - } - } - long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - string jobId = $"tj_{now}"; - string jobDir = Path.Combine(TrainingRoot(), "jobs", jobId); - Directory.CreateDirectory(jobDir); - string configPath = Path.Combine(jobDir, "config.json"); - string logPath = Path.Combine(jobDir, "log.txt"); - string adapterDir = Path.Combine(TrainingRoot(), "adapters", SanitizeAdapterName(outputName)); - Directory.CreateDirectory(adapterDir); - JObject jobConfig = new() - { - ["base_model"] = hfBase, - ["output_name"] = outputName, - ["ollama_base"] = raw["ollama_base"]?.ToString()?.Trim(), - ["gguf_base_path"] = raw["gguf_base_path"]?.ToString()?.Trim() ?? runner["gguf_base_path"]?.ToString()?.Trim(), - ["dataset_path"] = datasetPath, - ["hf_dataset"] = hfDataset, - ["hf_mapping"] = hfMapping, - ["hf_schema"] = hfCheck?["schema"], - ["max_samples"] = raw["max_samples"] ?? 0, - ["rank"] = raw["rank"] ?? 16, - ["alpha"] = raw["alpha"] ?? 32, - ["lr"] = raw["lr"] ?? 0.0002, - ["epochs"] = raw["epochs"] ?? 3, - ["seq_len"] = raw["seq_len"] ?? 2048, - ["four_bit"] = raw["four_bit"] ?? true, - ["batch_size"] = raw["batch_size"] ?? 1, - ["gradient_accumulation_steps"] = raw["gradient_accumulation_steps"] ?? 4, - ["adapter_dir"] = adapterDir, - ["local_export_count"] = exportCount, - }; - await File.WriteAllTextAsync(configPath, jobConfig.ToString(Newtonsoft.Json.Formatting.Indented), Encoding.UTF8); - Memory.SaveTrainJob(new JObject - { - ["id"] = jobId, - ["kind"] = "qlora", - ["status"] = "running", - ["config"] = jobConfig, - ["base_model"] = hfBase, - ["output_name"] = outputName, - ["log_path"] = logPath, - ["created_at"] = now, - }); - string cmdLine = BuildRunnerCommand(runner, configPath, logPath, jobDir); - bool started = TrainingJobManager.Start(this, session, jobId, cmdLine, logPath, baseUrl, chatModel, GetHfToken(session)); - if (!started) - { - Memory.SaveTrainJob(new JObject { ["id"] = jobId, ["status"] = "failed", ["progress"] = new JObject { ["error"] = "process start failed" } }); - return new JObject { ["error"] = "Не удалось запустить процесс тренировки" }; - } - return new JObject { ["success"] = true, ["job_id"] = jobId, ["log_path"] = logPath }; - } - - static string SanitizeAdapterName(string name) - { - if (string.IsNullOrWhiteSpace(name)) - { - return "adapter"; - } - char[] bad = Path.GetInvalidFileNameChars(); - StringBuilder sb = new(); - foreach (char c in name) - { - sb.Append(Array.IndexOf(bad, c) >= 0 ? '_' : c); - } - return sb.ToString().Trim(); - } - - string BuildRunnerCommand(JObject runner, string configPath, string logPath, string workDir) - { - string python = runner["python"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(python)) - { - python = "python"; - } - string kind = runner["kind"]?.ToString()?.Trim() ?? "builtin"; - string custom = runner["cmd"]?.ToString()?.Trim(); - string scriptPath = Path.Combine(FilePath, "scripts", "train_qlora.py"); - if (string.Equals(kind, "custom", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(custom)) - { - return custom - .Replace("{python}", python, StringComparison.OrdinalIgnoreCase) - .Replace("{config}", configPath, StringComparison.OrdinalIgnoreCase) - .Replace("{log}", logPath, StringComparison.OrdinalIgnoreCase) - .Replace("{workdir}", workDir, StringComparison.OrdinalIgnoreCase); - } - return $"\"{python}\" \"{scriptPath}\" --config \"{configPath}\" --log \"{logPath}\""; - } - - public async Task AssistentCancelTrainJob(Session session, string id = null) - { - await Task.CompletedTask; - TrainingJobManager.Cancel(); - string jobId = id ?? TrainingJobManager.CurrentJobId; - if (!string.IsNullOrWhiteSpace(jobId)) - { - Memory.SaveTrainJob(new JObject - { - ["id"] = jobId, - ["status"] = "cancelled", - ["finished_at"] = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), - }); - } - return new JObject { ["success"] = true, ["cancelled"] = true }; - } - - public async Task AssistentTrainWS(Session session, WebSocket ws, JObject raw) - { - await Task.CompletedTask; - try - { - while (TrainingJobManager.IsRunning && ws.State == System.Net.WebSockets.WebSocketState.Open) - { - JObject progress = TrainingJobManager.GetProgress(); - string msg = progress.ToString(Newtonsoft.Json.Formatting.None); - await ws.SendAsync(Encoding.UTF8.GetBytes(msg), System.Net.WebSockets.WebSocketMessageType.Text, true, CancellationToken.None); - await Task.Delay(800); - } - JObject final = TrainingJobManager.GetProgress(); - final["done"] = true; - await ws.SendAsync(Encoding.UTF8.GetBytes(final.ToString(Newtonsoft.Json.Formatting.None)), System.Net.WebSockets.WebSocketMessageType.Text, true, CancellationToken.None); - } - catch (Exception ex) - { - Logs.Debug($"AssistentTrainWS: {ex.Message}"); - } - return new JObject { ["success"] = true }; - } - - internal async Task FinishTrainJobAsync( - string jobId, - bool success, - string logPath, - Session session, - string baseUrl, - string chatModel, - JObject jobConfig, - JObject runner) - { - long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - string adapterDir = jobConfig?["adapter_dir"]?.ToString() ?? ""; - string outputName = jobConfig?["output_name"]?.ToString() ?? ""; - JObject progress = TrainingJobManager.GetProgress(); - string finalStatus = success ? "completed" : "failed"; - if (success && Directory.Exists(adapterDir)) - { - JObject reg = await RegisterAdapterPipeline(session, baseUrl, outputName, adapterDir, jobConfig, runner); - progress["ollama"] = reg; - if (reg["success"]?.Value() != true && reg["skipped"]?.Value() != true) - { - finalStatus = "completed_with_warnings"; - } - } - progress["status"] = finalStatus; - Memory.SaveTrainJob(new JObject - { - ["id"] = jobId, - ["status"] = finalStatus, - ["finished_at"] = now, - ["progress"] = progress, - }); - if (!string.IsNullOrWhiteSpace(chatModel)) - { - await AssistentWarmLlm(session, baseUrl, chatModel); - } - TrainingJobManager.ClearRunning(); - } - - async Task RegisterAdapterPipeline(Session session, string baseUrl, string outputName, string adapterDir, JObject jobConfig, JObject runner) - { - string safetensors = Directory.GetFiles(adapterDir, "adapter_model.safetensors").FirstOrDefault(); - if (string.IsNullOrWhiteSpace(safetensors)) - { - return new JObject { ["success"] = false, ["error"] = "adapter_model.safetensors not found" }; - } - string ggufPath = Directory.GetFiles(adapterDir, "*.gguf").FirstOrDefault(); - string ggufScript = runner?["gguf_script"]?.ToString()?.Trim(); - string ggufBase = jobConfig?["gguf_base_path"]?.ToString()?.Trim() ?? runner?["gguf_base_path"]?.ToString()?.Trim(); - string python = runner?["python"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(python)) - { - python = "python"; - } - if (string.IsNullOrWhiteSpace(ggufPath) && !string.IsNullOrWhiteSpace(ggufScript) && File.Exists(ggufScript)) - { - if (string.IsNullOrWhiteSpace(ggufBase) || !File.Exists(ggufBase)) - { - return new JObject - { - ["success"] = false, - ["skipped"] = true, - ["error"] = "gguf_base_path не задан или файл не найден — адаптер сохранён как safetensors", - ["adapter_dir"] = adapterDir, - }; - } - ggufPath = Path.Combine(adapterDir, "adapter.gguf"); - string ggufCmd = runner?["gguf_cmd"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(ggufCmd)) - { - ggufCmd = "\"{python}\" \"{script}\" \"{base}\" \"{lora}\" \"{out}\""; - } - string cmd = ggufCmd - .Replace("{python}", python, StringComparison.OrdinalIgnoreCase) - .Replace("{script}", ggufScript, StringComparison.OrdinalIgnoreCase) - .Replace("{base}", ggufBase, StringComparison.OrdinalIgnoreCase) - .Replace("{lora}", adapterDir, StringComparison.OrdinalIgnoreCase) - .Replace("{out}", ggufPath, StringComparison.OrdinalIgnoreCase); - int code = await RunShellCommandAsync(cmd, adapterDir); - if (code != 0 || !File.Exists(ggufPath)) - { - return new JObject - { - ["success"] = false, - ["error"] = $"GGUF convert failed exit={code}", - ["adapter_dir"] = adapterDir, - }; - } - } - if (string.IsNullOrWhiteSpace(ggufPath) || !File.Exists(ggufPath)) - { - return new JObject - { - ["success"] = false, - ["skipped"] = true, - ["note"] = "Настрой convert_lora_to_gguf.py и gguf_base_path для регистрации в Ollama", - ["adapter_dir"] = adapterDir, - }; - } - string ollamaBase = jobConfig?["ollama_base"]?.ToString()?.Trim(); - if (string.IsNullOrWhiteSpace(ollamaBase)) - { - return new JObject - { - ["success"] = false, - ["skipped"] = true, - ["error"] = "ollama_base не задан — укажи базовую Ollama-модель на форме QLoRA", - ["adapter_dir"] = adapterDir, - ["gguf"] = ggufPath, - }; - } - return await RegisterAdapterInOllama(baseUrl, outputName, ollamaBase, ggufPath); - } - - static async Task RunShellCommandAsync(string commandLine, string workDir) - { - try - { - ProcessStartInfo psi = new() - { - FileName = "cmd.exe", - Arguments = $"/c {commandLine}", - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WorkingDirectory = workDir ?? Environment.CurrentDirectory, - }; - using Process proc = Process.Start(psi); - if (proc is null) - { - return -1; - } - await proc.WaitForExitAsync(); - return proc.ExitCode; - } - catch (Exception ex) - { - Logs.Debug($"RunShellCommand: {ex.Message}"); - return -1; - } - } - - async Task RegisterAdapterInOllama(string baseUrl, string outputName, string ollamaBase, string adapterGguf) - { - StringBuilder mf = new(); - mf.AppendLine($"FROM {ollamaBase}"); - mf.AppendLine($"ADAPTER {adapterGguf.Replace("\\", "/")}"); - JObject payload = new() - { - ["name"] = outputName, - ["modelfile"] = mf.ToString(), - ["stream"] = false, - }; - using StringContent content = new(payload.ToString(Newtonsoft.Json.Formatting.None), Encoding.UTF8, "application/json"); - using HttpResponseMessage resp = await HttpClient.PostAsync($"{NormalizeBaseUrl(baseUrl)}/api/create", content); - string body = await resp.Content.ReadAsStringAsync(); - if (!resp.IsSuccessStatusCode) - { - return new JObject - { - ["success"] = false, - ["error"] = $"ollama create HTTP {(int)resp.StatusCode}: {Clip(body, 400)}", - ["modelfile"] = mf.ToString(), - }; - } - return new JObject - { - ["success"] = true, - ["name"] = outputName, - ["ollama_base"] = ollamaBase, - ["adapter"] = adapterGguf, - ["response"] = body, - }; - } -} - -sealed class TrainingJobManager -{ - static readonly Regex LossRe = new(@"loss[:\s]+([0-9.]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - static readonly Regex StepRe = new(@"step\s+(\d+)\s*/\s*(\d+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - - Process _process; - readonly object _lock = new(); - JObject _progress = new() { ["status"] = "idle" }; - string _logPath; - SwarmAssistentExtension _ext; - Session _session; - string _jobId; - string _baseUrl; - string _chatModel; - long _lastProgressSaveMs; - int _lastSavedStep = -1; - - public bool IsRunning { get; private set; } - public string CurrentJobId => _jobId; - - public bool Start(SwarmAssistentExtension ext, Session session, string jobId, string commandLine, string logPath, string baseUrl, string chatModel, string hfToken) - { - lock (_lock) - { - if (IsRunning) - { - return false; - } - _ext = ext; - _session = session; - _jobId = jobId; - _logPath = logPath; - _baseUrl = baseUrl; - _chatModel = chatModel; - _lastProgressSaveMs = 0; - _lastSavedStep = -1; - _progress = new JObject { ["status"] = "running", ["step"] = 0, ["loss"] = null, ["log"] = "" }; - try - { - ProcessStartInfo psi = new() - { - FileName = "cmd.exe", - Arguments = $"/c {commandLine}", - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - WorkingDirectory = Path.GetDirectoryName(logPath) ?? Environment.CurrentDirectory, - }; - if (!string.IsNullOrWhiteSpace(hfToken)) - { - psi.Environment["HF_TOKEN"] = hfToken; - psi.Environment["HUGGING_FACE_HUB_TOKEN"] = hfToken; - } - _process = new Process { StartInfo = psi, EnableRaisingEvents = true }; - _process.OutputDataReceived += (_, e) => AppendLog(e.Data); - _process.ErrorDataReceived += (_, e) => AppendLog(e.Data); - _process.Exited += async (_, _) => await OnExited(); - _process.Start(); - _process.BeginOutputReadLine(); - _process.BeginErrorReadLine(); - IsRunning = true; - return true; - } - catch (Exception ex) - { - _progress["error"] = ex.Message; - IsRunning = false; - return false; - } - } - } - - void AppendLog(string line) - { - if (string.IsNullOrWhiteSpace(line)) - { - return; - } - lock (_lock) - { - try - { - File.AppendAllText(_logPath, line + Environment.NewLine); - } - catch - { - // ignore - } - string prev = _progress["log"]?.ToString() ?? ""; - string combined = prev + line + "\n"; - if (combined.Length > 12000) - { - combined = combined[^12000..]; - } - _progress["log"] = combined; - Match lossM = LossRe.Match(line); - if (lossM.Success) - { - _progress["loss"] = lossM.Groups[1].Value; - } - Match stepM = StepRe.Match(line); - if (stepM.Success) - { - int step = int.Parse(stepM.Groups[1].Value); - int total = int.Parse(stepM.Groups[2].Value); - _progress["step"] = step; - _progress["total_steps"] = total; - _progress["percent"] = total > 0 ? (int)(100.0 * step / total) : 0; - } - MaybeSaveProgress(stepM.Success ? int.Parse(stepM.Groups[1].Value) : -1); - } - } - - void MaybeSaveProgress(int step) - { - long now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - bool stepChanged = step >= 0 && step != _lastSavedStep; - if (!stepChanged && now - _lastProgressSaveMs < 2500) - { - return; - } - _lastProgressSaveMs = now; - if (step >= 0) - { - _lastSavedStep = step; - } - try - { - _ext?.Memory?.SaveTrainJob(new JObject - { - ["id"] = _jobId, - ["status"] = "running", - ["progress"] = _progress, - }); - } - catch - { - // ignore - } - } - - async Task OnExited() - { - bool ok = false; - JObject jobConfig = new(); - lock (_lock) - { - ok = _process?.ExitCode == 0; - IsRunning = false; - _progress["status"] = ok ? "completed" : "failed"; - _progress["exit_code"] = _process?.ExitCode; - } - if (_ext != null) - { - JObject job = _ext.Memory.GetTrainJob(_jobId); - try - { - string cfgRaw = job?["config_json"]?.ToString(); - if (!string.IsNullOrWhiteSpace(cfgRaw)) - { - jobConfig = JObject.Parse(cfgRaw); - } - } - catch - { - // ignore - } - JObject runner = _ext.Config.LoadTrainingRunner(); - await _ext.FinishTrainJobAsync(_jobId, ok, _logPath, _session, _baseUrl, _chatModel, jobConfig, runner); - } - } - - public void Cancel() - { - lock (_lock) - { - try - { - if (_process != null && !_process.HasExited) - { - _process.Kill(entireProcessTree: true); - } - } - catch - { - // ignore - } - IsRunning = false; - _progress["status"] = "cancelled"; - } - } - - public JObject GetProgress() - { - lock (_lock) - { - return (JObject)_progress.DeepClone(); - } - } - - public void ClearRunning() - { - lock (_lock) - { - IsRunning = false; - } - } -} diff --git a/Config/_base/patch-keys.json b/Config/_base/patch-keys.json index 10d2861..cd1b18b 100644 --- a/Config/_base/patch-keys.json +++ b/Config/_base/patch-keys.json @@ -8,6 +8,6 @@ "snapshot_generate", "select_slot", "aspect", "images", "batch", "vary", "lock_seed", "creativity", "intensity", "complexity", "movement", "clear_prompt_images", "slot_to_prompt_image", "pack", "persona", "controls", - "inventory_query", "variants" + "inventory_query", "knowledge_query", "example_query", "knowledge_rating", "example_rating", "variants" ] } diff --git a/Config/_base/skills/knowledge.json b/Config/_base/skills/knowledge.json new file mode 100644 index 0000000..638add5 --- /dev/null +++ b/Config/_base/skills/knowledge.json @@ -0,0 +1,6 @@ +{ + "id": "knowledge", + "title": "Books + knowledge search", + "default": false, + "prompt_file": "knowledge.md" +} diff --git a/Config/_base/skills/knowledge.md b/Config/_base/skills/knowledge.md new file mode 100644 index 0000000..2263283 --- /dev/null +++ b/Config/_base/skills/knowledge.md @@ -0,0 +1,22 @@ +# Knowledge skill + +Attached **books** are read-only FTS corpora (Civitai prompts, RU fic style, …). They are **not** mutable memory. + +## When to search + +- User asks for reference prompts, scene wording, style/tone, or «как на Civitai». +- You need concrete tag/prompt patterns before generating. + +## How to search (server hop) + +```json +{ "ask": ["knowledge"], "knowledge_query": "redhead stockings cinematic window light" } +``` + +Optional: `knowledge_rating` (`pg`, `pg13`, …). Legacy `ask:["examples"]` + `example_query` still works for Civitai-only. + +## Rules + +- **Remix** — never paste long verbatim from books. +- Pure chat / opinions: **prose only**, no JSON. +- Do **not** emit `generate:true` on knowledge Q&A turns. diff --git a/Config/_base/skills/memory.md b/Config/_base/skills/memory.md index 7d944b8..c8865cc 100644 --- a/Config/_base/skills/memory.md +++ b/Config/_base/skills/memory.md @@ -6,7 +6,7 @@ You have five memory tools: 2. **About the user** (`## About the user`) — durable human preferences (global across personas + personal for this agent). Tunable weight in settings. Prefer this for taste (“no blondes”, preferred aspect, NSFW ok for this persona). 3. **Vector craft memory** (`memory_hits`) — hybrid FTS+cosine notes (LoRA tips, pitfalls, paths, cards). Shared + this persona; personal overwrites shared on the same `kind`+`key`. 4. **Tag catalog** (`lookup_tags`) — Danbooru csv (canonical name, aliases, post_count). **Not** RAG. Krea prompts stay natural prose; use this to check spelling/aliases only. -5. **Civitai examples** (`ask: examples` / `lookup_examples`) — popular Krea 2 prompts indexed with FTS (no embeddings). Use when you want a **reference** for how others phrased a scene — remix, do not copy 1:1. +5. **Civitai / knowledge books** (`ask: knowledge` / `ask: examples`) — FTS over attached books (gpu-rent seeded) and legacy Civitai examples. Remix, do not copy 1:1. ## Priority diff --git a/Config/_base/training-qlora.json b/Config/_base/training-qlora.json deleted file mode 100644 index bd8f4e8..0000000 --- a/Config/_base/training-qlora.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "hf_models": [ - { - "id": "qwen2.5-7b-instruct", - "title": "Qwen2.5 7B Instruct", - "hf_id": "Qwen/Qwen2.5-7B-Instruct", - "ollama_hint": "qwen2.5:7b-instruct", - "default_output": "assistent-qwen25-7b:v1", - "rank": 16, - "seq_len": 2048 - }, - { - "id": "qwen2.5-3b-instruct", - "title": "Qwen2.5 3B Instruct", - "hf_id": "Qwen/Qwen2.5-3B-Instruct", - "ollama_hint": "qwen2.5:3b-instruct", - "default_output": "assistent-qwen25-3b:v1", - "rank": 16, - "seq_len": 2048 - }, - { - "id": "llama-3.2-3b-instruct", - "title": "Llama 3.2 3B Instruct", - "hf_id": "meta-llama/Llama-3.2-3B-Instruct", - "ollama_hint": "llama3.2:3b-instruct", - "default_output": "assistent-llama32-3b:v1", - "rank": 16, - "seq_len": 2048 - }, - { - "id": "llama-3.1-8b-instruct", - "title": "Llama 3.1 8B Instruct", - "hf_id": "meta-llama/Llama-3.1-8B-Instruct", - "ollama_hint": "llama3.1:8b-instruct", - "default_output": "assistent-llama31-8b:v1", - "rank": 16, - "seq_len": 2048 - }, - { - "id": "phi-3-mini-instruct", - "title": "Phi-3 Mini 4K Instruct", - "hf_id": "microsoft/Phi-3-mini-4k-instruct", - "ollama_hint": "phi3:mini", - "default_output": "assistent-phi3-mini:v1", - "rank": 16, - "seq_len": 2048 - } - ] -} diff --git a/SwarmAssistentExtension.cs b/SwarmAssistentExtension.cs index a837959..3be7d58 100644 --- a/SwarmAssistentExtension.cs +++ b/SwarmAssistentExtension.cs @@ -33,8 +33,8 @@ public partial class SwarmAssistentExtension : Extension ExtensionAuthor = "mrleo1nid"; Description = "Collaborative Krea 2 assistant: Ollama chat, persona presets, vector memory, model cards, Generate loop."; License = "MIT"; - Version = "0.15.17"; - Tags = ["tabs", "ui", "llm", "ollama", "krea", "inpaint", "memory", "training", "heard", "qlora"]; + Version = "0.16.0"; + Tags = ["tabs", "ui", "llm", "ollama", "krea", "inpaint", "memory", "knowledge", "heard", "books"]; } public override void OnInit() @@ -64,6 +64,10 @@ public partial class SwarmAssistentExtension : Extension API.RegisterAPICall(AssistentGetMemory, false, PermUse); API.RegisterAPICall(AssistentLookupTags, false, PermUse); API.RegisterAPICall(AssistentLookupExamples, false, PermUse); + API.RegisterAPICall(AssistentListKnowledgeCatalog, false, PermUse); + API.RegisterAPICall(AssistentSearchKnowledge, false, PermUse); + API.RegisterAPICall(AssistentGetKnowledgeAttach, false, PermUse); + API.RegisterAPICall(AssistentSaveKnowledgeAttach, true, PermUse); API.RegisterAPICall(AssistentSaveControls, true, PermUse); API.RegisterAPICall(AssistentGetPersonaShelves, false, PermUse); API.RegisterAPICall(AssistentClonePersona, true, PermUse); @@ -77,29 +81,7 @@ public partial class SwarmAssistentExtension : Extension API.RegisterAPICall(AssistentForgetUserPref, true, PermUse); API.RegisterAPICall(AssistentClearUserPrefs, true, PermUse); API.RegisterAPICall(AssistentClearMemory, true, PermUse); - API.RegisterAPICall(AssistentListTrainSamples, false, PermUse); - API.RegisterAPICall(AssistentUpsertTrainSample, true, PermUse); - API.RegisterAPICall(AssistentDeleteTrainSample, true, PermUse); - API.RegisterAPICall(AssistentBuildDatasetFromChats, false, PermUse); - API.RegisterAPICall(AssistentImportDataset, true, PermUse); - API.RegisterAPICall(AssistentExportDataset, false, PermUse); - API.RegisterAPICall(AssistentCreateOllamaModel, true, PermUse); - API.RegisterAPICall(AssistentSearchHfDatasets, false, PermUse); - API.RegisterAPICall(AssistentCheckHfDataset, false, PermUse); - API.RegisterAPICall(AssistentPreviewHfDataset, false, PermUse); - API.RegisterAPICall(AssistentImportHfDataset, true, PermUse); - API.RegisterAPICall(AssistentStartTrainJob, true, PermUse); - API.RegisterAPICall(AssistentCancelTrainJob, true, PermUse); - API.RegisterAPICall(AssistentGetTrainJob, false, PermUse); - API.RegisterAPICall(AssistentTrainWS, true, PermUse); - API.RegisterAPICall(AssistentSaveRunnerSettings, true, PermUse); - API.RegisterAPICall(AssistentGetRunnerSettings, false, PermUse); - API.RegisterAPICall(AssistentGetDatasetAgentSettings, false, PermUse); - API.RegisterAPICall(AssistentSaveDatasetAgentSettings, true, PermUse); - API.RegisterAPICall(AssistentLinkTrainSampleToAgent, true, PermUse); - API.RegisterAPICall(AssistentUnlinkTrainSampleFromAgent, true, PermUse); - API.RegisterAPICall(AssistentSyncDatasetToAgent, true, PermUse); - Logs.Init("Swarm Assistent extension loaded (0.15.0 persona packs / dreamer)"); + Logs.Init("Swarm Assistent extension loaded (0.16.0 knowledge books hub)"); } int CfgInt(string key, int fallback) diff --git a/Tabs/Text2Image/Assistent.html b/Tabs/Text2Image/Assistent.html index 1abd53f..4e8c27d 100644 --- a/Tabs/Text2Image/Assistent.html +++ b/Tabs/Text2Image/Assistent.html @@ -10,13 +10,8 @@
-
-
@@ -161,140 +156,6 @@
- -