From d57969d66d13c8b976421bdc39b2bdf831e8ffdd Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 28 Apr 2026 03:35:51 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20MCP=20=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E5=9C=A8=E6=B5=81=E5=BC=8F=E4=B8=8E=E5=8E=86=E5=8F=B2=E4=BC=9A?= =?UTF-8?q?=E8=AF=9D=E4=B8=AD=E7=9A=84=E6=98=BE=E7=A4=BA=E4=B8=80=E8=87=B4?= =?UTF-8?q?=E6=80=A7=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 tool_result 图片落库为 attachments，并在聊天渲染中保留单气泡与时序位置，避免流式结束后图片消失或刷新后错位。 Made-with: Cursor --- docs/RUNBOOK.md | 73 +++++++++ interfaces/admin/static/app.js | 210 +++++++++++++++++++++++++ interfaces/admin/static/chat.js | 261 +++++++++++++++++++++++++++----- runtime/chat/tool_runtime.py | 90 +++++++++++ tests/test_tool_loop_guard.py | 40 +++++ 5 files changed, 636 insertions(+), 38 deletions(-) diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 115ae92a..9d2effc5 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -111,6 +111,30 @@ Linux/macOS: - `http://127.0.0.1:8787/admin` - `http://127.0.0.1:8787/chat` +--- + +## 7. Admin Chat：WS 请求超时（`ws_send_timeout:*`） + +现象： + +- 管理台聊天页流式请求中途提示：`请求失败: Error: ws_send_timeout:180000`（或其他毫秒值） + +说明： + +- 这是**前端**在 `interfaces/admin/static/chat.js` 中实现的 WS “无活动超时”看门狗（不是后端限流/超时）。 +- 触发条件通常为：在指定毫秒窗口内（默认曾为 180000ms）没有收到任何 WS 活动（delta / event / ack 等）。 +- 某些模型/网关可能会出现“长时间无增量、最终一次性返回”的情况，此时该看门狗会误判并提前报错。 + +当前默认行为： + +- 默认已禁用（`WS_CHAT_SEND_TIMEOUT_MS = 0`）。 + +如何调整： + +- 修改 `interfaces/admin/static/chat.js` 中常量 `WS_CHAT_SEND_TIMEOUT_MS`： + - `0`：禁用 + - `> 0`：启用并作为无活动超时毫秒数 + ### 6.1 初始化管理员（幂等） ```bash @@ -211,6 +235,55 @@ Linux/macOS: 详细开发与接入参考：`docs/MCP_LOCAL_SERVER.md` +### 8.1 `images-mcp`（Windows）安装与 Health 通过指引 + +现象： + +- `Install` 显示成功，但 `Health` 报错：`mcp_runtime_empty_response` +- 手动执行 `npx -y images-mcp` 只输出 CLI 帮助并退出 + +根因： + +- `images-mcp` 的 npm bin 入口默认是 CLI（`cli.ts`），不是 MCP stdio 服务入口。 +- 健康检查要求进程保持 MCP JSON-RPC（stdio）模式；CLI 进程会立即退出，导致空响应。 + +前置检查（PowerShell）： + +```powershell +where bun +where bunx +bun --version +bunx --version +``` + +若 `where bun` 无结果，先安装/修复 Bun，再重启终端与网关进程。 + +推荐安装方式（本地 runtime 目录）： + +```powershell +mkdir D:\tools\images-mcp-runtime -Force +cd D:\tools\images-mcp-runtime +npm init -y +npm install images-mcp +``` + +将 MCP Server 启动配置改为（管理台或数据库）： + +- `entry_command`: `powershell` +- `entry_args`: + `["-NoProfile","-Command","Set-Location D:\\tools\\images-mcp-runtime; bun run node_modules/images-mcp/mcp.ts"]` + +预期结果： + +- 手动验证启动命令时，会看到 `Images MCP server running on stdio` +- 管理台 `Health` 返回 `ok` +- `Sync Tools` 后应看到 2 个工具（OpenAI / Gemini） + +排障补充： + +- 若仍失败，优先检查 `entry_args` 是否仍是 `npx -y images-mcp`（该配置会回到 CLI 模式，Health 继续失败）。 +- 若提示找不到 `bun`，确认网关进程启动时的 PATH 已包含 `C:\Users\<用户名>\.bun\bin`。 + --- ## 9. 向量记忆配置（可选） diff --git a/interfaces/admin/static/app.js b/interfaces/admin/static/app.js index 2e6be147..2377c456 100644 --- a/interfaces/admin/static/app.js +++ b/interfaces/admin/static/app.js @@ -341,6 +341,13 @@ const I18N = { "models.baseUrl": "Base URL", "models.baseUrlPlaceholder": "可选，留空用环境默认", "models.apiKey": "API Key", + "models.thinkMode": "Think 模式（仅当前 API）", + "models.thinkModeHint": "开启后仅对此配置回放完整 reasoning_content（默认关闭）", + "models.reasoningEffort": "Reasoning 强度", + "models.reasoningEffortDefault": "默认", + "models.reasoningEffortLow": "低", + "models.reasoningEffortMedium": "中", + "models.reasoningEffortHigh": "高", "models.rememberKey": "记住密钥（存库加密）", "models.save": "保存", "models.delete": "删除此配置", @@ -760,6 +767,13 @@ const I18N = { "models.baseUrl": "Base URL", "models.baseUrlPlaceholder": "Optional", "models.apiKey": "API key", + "models.thinkMode": "Think mode (this API profile only)", + "models.thinkModeHint": "When on, replay full assistant reasoning_content for this profile only (default off).", + "models.reasoningEffort": "Reasoning effort", + "models.reasoningEffortDefault": "Default", + "models.reasoningEffortLow": "Low", + "models.reasoningEffortMedium": "Medium", + "models.reasoningEffortHigh": "High", "models.rememberKey": "Remember key (encrypted in DB)", "models.save": "Save", "models.delete": "Delete profile", @@ -3269,6 +3283,13 @@ async function renderModels() { const modeSel = el("select", { class: "input" }); const modelInp = el("input", { class: "input" }); const baseInp = el("input", { class: "input", placeholder: t("models.baseUrlPlaceholder") }); + const thinkingModeCb = el("input", { type: "checkbox" }); + const reasoningEffortSel = el("select", { class: "input", style: "max-width:180px" }, [ + el("option", { value: "", text: t("models.reasoningEffortDefault") }), + el("option", { value: "low", text: t("models.reasoningEffortLow") }), + el("option", { value: "medium", text: t("models.reasoningEffortMedium") }), + el("option", { value: "high", text: t("models.reasoningEffortHigh") }), + ]); const keyInp = el("input", { class: "input", type: "password", autocomplete: "off" }); const rememberCb = el("input", { type: "checkbox" }); const readonlyProfileHint = el("div", { class: "muted", text: "" }); @@ -3661,12 +3682,16 @@ async function renderModels() { } modelInp.value = String(selProf.model || ""); baseInp.value = String(selProf.base_url || ""); + thinkingModeCb.checked = !!selProf.thinking_mode_enabled; + reasoningEffortSel.value = String(selProf.reasoning_effort || ""); keyInp.value = String(state.profile_secret || ""); rememberCb.checked = !!selProf.has_key; profName.disabled = !canEditFields; modelInp.disabled = !canEditFields; baseInp.disabled = !canEditFields; + thinkingModeCb.disabled = !canEditFields; + reasoningEffortSel.disabled = !canEditFields; keyInp.disabled = !canEditFields; rememberCb.disabled = !canEditFields; @@ -3739,6 +3764,8 @@ async function renderModels() { mode: modeSave, model: modelInp.value.trim(), base_url: baseInp.value.trim(), + thinking_mode_enabled: !!thinkingModeCb.checked, + reasoning_effort: String(reasoningEffortSel.value || ""), }); await apiRequest("POST", "/admin/api/models/profiles/" + encodeURIComponent(pid) + "/secret", { remember: rememberCb.checked, @@ -3954,6 +3981,9 @@ async function renderModels() { el("div", { class: "row" }, [el("label", { text: t("models.mode") }), modeSel]), el("div", { class: "row" }, [el("label", { text: t("models.model") }), modelInp]), el("div", { class: "row" }, [el("label", { text: t("models.baseUrl") }), baseInp]), + el("div", { class: "row" }, [el("label", { text: t("models.thinkMode") }), thinkingModeCb]), + el("div", { class: "muted", text: t("models.thinkModeHint") }), + el("div", { class: "row" }, [el("label", { text: t("models.reasoningEffort") }), reasoningEffortSel]), warnKey, openaiHint, ollamaHint, @@ -4530,6 +4560,96 @@ async function renderPlugins() { "Paste install JSON: one object, array of objects, or { servers: [...] } / { payload: {...} } (seed file shape)", rows: "6", }); + const cliInstallInput = el("input", { + class: "input", + placeholder: "CLI one-liner, e.g. npx -y mcp-fetch-server | pip install mcp-server-time | pip install git+https://... && python -m ", + }); + const _parseCliInstall = (raw) => { + const s = String(raw || "").trim(); + if (!s) return { ok: false, error: "empty_command" }; + // Support "A && B" one-liners (we only parse safe shapes; never shell-eval). + const segs = s.split(/\s*&&\s*/).map((x) => String(x || "").trim()).filter(Boolean); + const s0 = String(segs[0] || "").trim(); + const s1 = String(segs[1] || "").trim(); + // normalize spaces for simple token parse (no shell eval). + const parts = s0.split(/\s+/).filter(Boolean); + const low0 = String(parts[0] || "").toLowerCase(); + const low1 = String(parts[1] || "").toLowerCase(); + const idxPkg = (arr) => arr.findIndex((x) => !String(x || "").startsWith("-")); + if (low0 === "npx" || (low0 === "npm" && low1 === "exec")) { + // npx -y + const rest = low0 === "npx" ? parts.slice(1) : parts.slice(2); + const i = idxPkg(rest); + const pkg = i >= 0 ? String(rest[i] || "").trim() : ""; + if (!pkg) return { ok: false, error: "npx_package_missing" }; + const tail = i >= 0 ? rest.slice(i + 1).map((x) => String(x || "").trim()).filter(Boolean) : []; + return { + ok: true, + payload: { + source_type: "npm", + source_ref: pkg, + server_id: pkg, + entry_command: "npx", + // Keep package tail args (e.g. "run ") for MCP CLIs that require subcommands/auth. + entry_args: ["-y", pkg, ...tail], + }, + }; + } + if (low0 === "npm" && low1 === "install") { + // npm install [-g] + const rest = parts.slice(2); + const i = idxPkg(rest); + const pkg = i >= 0 ? String(rest[i] || "").trim() : ""; + if (!pkg) return { ok: false, error: "npm_package_missing" }; + return { + ok: true, + payload: { + source_type: "npm", + source_ref: pkg, + server_id: pkg, + entry_command: "npx", + entry_args: ["-y", pkg], + }, + }; + } + if ((low0 === "pip" && low1 === "install") || (low0 === "python" && low1 === "-m" && String(parts[2] || "").toLowerCase() === "pip")) { + const rest = low0 === "pip" ? parts.slice(2) : parts.slice(4); + const i = idxPkg(rest); + const pkg = i >= 0 ? String(rest[i] || "").trim() : ""; + if (!pkg) return { ok: false, error: "pypi_package_missing" }; + // If pip install uses git+... URL, require explicit entry module via "&& python -m ". + if (pkg.toLowerCase().startsWith("git+")) { + const p1 = s1.split(/\s+/).filter(Boolean); + const p10 = String(p1[0] || "").toLowerCase(); + const p11 = String(p1[1] || "").toLowerCase(); + const mod = (p10 === "python" && p11 === "-m") ? String(p1[2] || "").trim() : ""; + if (!mod) return { ok: false, error: "git_pip_requires_python_m" }; + const repoName = pkg.replace(/^git\+/i, "").split(/[?#]/)[0].split("/").filter(Boolean).pop() || "mcp-server"; + const sid = String(repoName).replace(/\.git$/i, ""); + return { + ok: true, + payload: { + source_type: "pypi", + source_ref: pkg, + server_id: sid, + entry_command: "python", + entry_args: ["-m", mod], + }, + }; + } + return { + ok: true, + payload: { + source_type: "pypi", + source_ref: pkg, + server_id: pkg, + entry_command: "python", + entry_args: ["-m", pkg.replace(/-/g, "_")], + }, + }; + } + return { ok: false, error: "unsupported_cli_command" }; + }; const installBtn = el("button", { class: "btn btn--primary", text: "Install MCP", @@ -4659,6 +4779,72 @@ async function renderPlugins() { router(); }, }); + const cliInstallModal = el("div", { class: "session-monitor-modal", style: "display:none;" }); + const cliInstallModalTitle = el("div", { class: "card__title", text: "MCP CLI install" }); + const cliInstallModalBody = el("pre", { class: "pre", text: "" }); + const closeCliInstallModal = () => { + cliInstallModal.style.display = "none"; + }; + const openCliInstallModal = (title, text) => { + cliInstallModalTitle.textContent = String(title || "MCP CLI install"); + cliInstallModalBody.textContent = String(text || ""); + cliInstallModal.style.display = "flex"; + }; + const setCliInstallModal = (title, text) => { + if (title) cliInstallModalTitle.textContent = String(title); + if (text != null) cliInstallModalBody.textContent = String(text); + }; + cliInstallModal.addEventListener("click", (e) => { + if (e.target === cliInstallModal) closeCliInstallModal(); + }); + cliInstallModal.appendChild( + el("div", { class: "card session-monitor-modal__card", style: "width:min(720px,96vw);" }, [ + cliInstallModalTitle, + cliInstallModalBody, + el("div", { class: "row", style: "justify-content:flex-end;margin-top:10px;" }, [ + el("button", { class: "btn", text: "Close", onclick: closeCliInstallModal }), + ]), + ]), + ); + const cliInstallBtn = el("button", { + class: "btn", + text: "Install from CLI", + onclick: async () => { + preflightFixWrap.innerHTML = ""; + openCliInstallModal("MCP CLI install", "Parsing command..."); + const parsed = _parseCliInstall(cliInstallInput.value); + if (!parsed.ok) { + installStatus.textContent = `[cli] ${String(parsed.error || "parse_failed")}`; + setCliInstallModal("MCP CLI install failed", installStatus.textContent); + return; + } + const payload = { + source_type: String(parsed.payload.source_type || "").trim(), + source_ref: String(parsed.payload.source_ref || "").trim(), + server_id: String(parsed.payload.server_id || "").trim(), + entry_command: String(parsed.payload.entry_command || "").trim(), + entry_args: Array.isArray(parsed.payload.entry_args) ? parsed.payload.entry_args.map((x) => String(x)) : [], + version: "", + }; + setCliInstallModal("MCP CLI install", "Running preflight..."); + const pre = await apiPost("/admin/api/mcp/preflight", payload); + if (!pre.ok) { + installStatus.textContent = `[cli/preflight] ${String(pre.error_code || "")} ${String(pre.error || "")}`.trim(); + setCliInstallModal("MCP CLI preflight failed", installStatus.textContent); + return; + } + setCliInstallModal("MCP CLI install", "Installing..."); + const res = await apiPost("/admin/api/mcp/install", payload); + installStatus.textContent = `[cli] ` + JSON.stringify(res); + if (res && res.ok) { + setCliInstallModal("MCP CLI install success", installStatus.textContent); + markPrewarmReminder("mcp_installed"); + router(); + } else { + setCliInstallModal("MCP CLI install failed", installStatus.textContent); + } + }, + }); const mcpServerList = Array.isArray(mcp.servers) ? mcp.servers : []; const expertBindingDashTbody = el("tbody"); repaintExpertBindingDashboard = () => { @@ -5492,6 +5678,29 @@ async function renderPlugins() { const foldMcpInstall = pluginsFold("【3】MCP 安装（表单 / JSON / 运维）", [ el("div", { class: "row" }, [sourceType, sourceRef, version]), el("div", { class: "row" }, [entryCmd, entryArgs, installBtn]), + el("div", { class: "muted", text: "CLI direct install (paste one command)" }), + el("div", { class: "row" }, [cliInstallInput, cliInstallBtn]), + el("div", { class: "muted", text: "常用命令行安装示例（可先本机验证，再填上方表单）" }), + el("pre", { + class: "pre", + text: +`# npm 包（本地安装） +npm install mcp-fetch-server + +# 直接运行（推荐） +npx -y mcp-fetch-server + +# 全局安装后运行 +npm install -g mcp-fetch-server +mcp-fetch-server + +# Python 包示例 +pip install mcp-server-time +python -m mcp_server_time + +# Python（Git URL / VCS）示例：必须显式指定 entry module +pip install git+https://github.com/philschmid/code-sandbox-mcp.git && python -m code_sandbox_mcp`, + }), el("div", { class: "muted", text: "JSON install (single object or array)" }), jsonInstallInput, el("div", { class: "row" }, [jsonInstallBtn]), @@ -5739,6 +5948,7 @@ async function renderPlugins() { foldExpertBindingDash, el("div", { id: "plugins-binding" }, [foldMcpBinding]), ]), + cliInstallModal, ]); } diff --git a/interfaces/admin/static/chat.js b/interfaces/admin/static/chat.js index ce13be05..63eb2852 100644 --- a/interfaces/admin/static/chat.js +++ b/interfaces/admin/static/chat.js @@ -389,10 +389,32 @@ function _appendAssistantTextSegments(inner, rawText, collapsedItems) { function _buildAggregatedAssistantBubble(tsIso, items) { const inner = el("div", { class: "chat-msg chat-msg--assistant chat-msg--rich" }); const collapsedItems = []; + const _inlineImageHtmlFromAttachments = (raw) => { + const list = parseAttachments(raw); + if (!Array.isArray(list) || !list.length) return ""; + const blocks = []; + for (const att of list) { + if (!att || typeof att !== "object") continue; + const typ = String(att.type || "").toLowerCase(); + if (typ === "image" || typ === "input_image") { + const b64 = String(att.image_base64 || att.data || "").trim(); + if (!b64) continue; + const mime = String(att.mime || att.mime_type || "image/png").trim() || "image/png"; + const src = `data:${mime};base64,${b64.replace(/\s/g, "")}`; + blocks.push(`
tool image
`); + } else if (typ === "image_url") { + const src = String(att.url || att.image_url || "").trim(); + if (!src) continue; + blocks.push(`
tool image
`); + } + } + return blocks.join(""); + }; for (const it of items || []) { const kind = String((it && it.kind) || ""); const text = String((it && it.text) || ""); - if (!text.trim()) continue; + const hasInlineAtt = !!(it && it.attachments); + if (!text.trim() && !(kind === "tool_result" && hasInlineAtt)) continue; if (kind === "assistant_text") { _appendAssistantTextSegments(inner, text, collapsedItems); } else if (kind === "reasoning") { @@ -401,6 +423,8 @@ function _buildAggregatedAssistantBubble(tsIso, items) { if (adminChatShowToolOutput) collapsedItems.push({ title: _toolSummaryTitle("tool_call"), text }); } else if (kind === "tool_result") { if (adminChatShowToolOutput) collapsedItems.push({ title: _toolSummaryTitle("tool"), text }); + const inlineImages = _inlineImageHtmlFromAttachments(it && it.attachments); + if (inlineImages) inner.appendChild(el("div", { class: "chat-msg__md", html: inlineImages })); } else { inner.appendChild(el("div", { class: "chat-msg__md", html: renderMarkdownHtml(text) })); } @@ -488,7 +512,8 @@ function _buildRenderRows(msgs) { } if (m.attachments) agg.attachments = m.attachments; } else if (String(content || "").trim()) { - agg._items.push({ kind: "tool_result", text: content }); + agg._items.push({ kind: "tool_result", text: content, attachments: m.attachments || null }); + if (m.attachments) agg.attachments = m.attachments; } continue; } @@ -1304,6 +1329,71 @@ function extractSqlAuditPayload(payload) { }; } +function extractToolImageItems(payload) { + const _toObj = (v) => { + if (v && typeof v === "object") return v; + if (typeof v !== "string") return null; + const s = String(v || "").trim(); + if (!s) return null; + try { + const j = JSON.parse(s); + return j && typeof j === "object" ? j : null; + } catch (_) { + return null; + } + }; + const p = _toObj(payload) || {}; + const cands = [p, _toObj(p.payload), _toObj(p.result), _toObj(p.payload && p.payload.result), _toObj(p.result && p.result.result)].filter(Boolean); + const out = []; + for (const cand of cands) { + const content = Array.isArray(cand.content) ? cand.content : []; + for (const item of content) { + if (!item || typeof item !== "object") continue; + const typ = String(item.type || "").trim().toLowerCase(); + if (typ === "image" || typ === "input_image") { + const srcObj = item.source && typeof item.source === "object" ? item.source : {}; + const b64 = String(item.image_base64 || item.data || srcObj.data || "").trim(); + if (!b64) continue; + const mime = String(item.mime_type || item.mime || srcObj.media_type || "image/png").trim() || "image/png"; + out.push({ type: "image", src: `data:${mime};base64,${b64.replace(/\s/g, "")}` }); + continue; + } + if (typ === "image_url") { + const urlObj = item.image_url && typeof item.image_url === "object" ? item.image_url : {}; + const url = String(item.url || item.image_url || urlObj.url || "").trim(); + if (!url) continue; + out.push({ type: "image_url", src: url }); + } + } + } + // Fallback: direct attachment-like payload shape. + const direct = _toObj(p.attachments); + const atts = Array.isArray(direct) ? direct : []; + for (const a of atts) { + if (!a || typeof a !== "object") continue; + const t = String(a.type || "").toLowerCase(); + if (t === "image" || t === "input_image") { + const b64 = String(a.image_base64 || a.data || "").trim(); + if (!b64) continue; + const mime = String(a.mime || a.mime_type || "image/png").trim() || "image/png"; + out.push({ type: "image", src: `data:${mime};base64,${b64.replace(/\s/g, "")}` }); + } else if (t === "image_url") { + const src = String(a.url || a.image_url || "").trim(); + if (src) out.push({ type: "image_url", src }); + } + } + // De-dup by src. + const uniq = []; + const seen = new Set(); + for (const it of out) { + const k = String((it && it.src) || ""); + if (!k || seen.has(k)) continue; + seen.add(k); + uniq.push(it); + } + return uniq; +} + function _sqlLimitSuffix(inputSql, executedSql) { const a = String(inputSql || "").trim(); const b = String(executedSql || "").trim(); @@ -1616,7 +1706,9 @@ async function appendMessageRow(messagesEl, m) { const bubble = Array.isArray(m._items) ? _buildAggregatedAssistantBubble(ts, m._items) : await buildMessageBubble(role, content, ts); - const att = await renderAttachmentsEl(m.attachments); + // For aggregated assistant bubbles, attachments should be rendered inline + // at tool_result positions, not appended at bubble tail. + const att = Array.isArray(m._items) ? null : await renderAttachmentsEl(m.attachments); if (att) { const innerBubble = bubble.querySelector(".chat-msg-col .chat-msg"); if (innerBubble) innerBubble.appendChild(att); @@ -2784,7 +2876,9 @@ async function renderChatUi() { let currentStreamAbortController = null; let currentWsTransport = null; let currentAbortMeta = { sessionId: "", runId: "" }; - const WS_CHAT_SEND_TIMEOUT_MS = 180000; + // Disable WS send inactivity timeout by default (0 = disabled). + // Some gateways can legitimately stream slower than 3 minutes. + const WS_CHAT_SEND_TIMEOUT_MS = 0; const isAbortError = (err) => { const name = String(err && err.name ? err.name : ""); const msg = String(err && err.message ? err.message : err || ""); @@ -2822,6 +2916,9 @@ async function renderChatUi() { let streamDisplayTarget = ""; let streamDisplayShown = ""; let streamTextBuffer = ""; + const streamStitcher = createStreamStitcher(); + let perCharNewlineMode = false; + let perCharNewlineScore = 0; let toolSeq = 0; let hasRealStreamText = false; let sawWsChatEvent = false; @@ -2944,6 +3041,38 @@ async function renderChatUi() { // Never inject separators here; otherwise typewriter offsets drift. return decodeEscapedNewlines(chunks.join("")); }; + const _normalizeStreamTargetForRender = (raw) => { + let s = String(raw || "").replace(/\r/g, ""); + if (!s) return ""; + // Streaming-only guard: some gateways emit per-char newlines in delta chunks, + // producing vertical "one character per line" layout. + const lines = s.split("\n"); + if (lines.length >= 4) { + let short = 0; + let nonEmpty = 0; + let totalLen = 0; + for (const ln of lines) { + const t = String(ln || ""); + if (!t) continue; + nonEmpty += 1; + totalLen += t.length; + if (t.length <= 2) short += 1; + } + const avgLen = nonEmpty ? totalLen / nonEmpty : 0; + const shortRatio = nonEmpty ? short / nonEmpty : 0; + if (shortRatio >= 0.75 && avgLen <= 2.0) { + // Join characters back for live display. + s = lines.join(""); + } else { + // Fallback: if newline density is abnormally high, collapse line breaks for live view. + const nl = lines.length - 1; + if (nl >= 8 && nl >= Math.floor(s.length * 0.2)) { + s = s.replace(/\n+/g, ""); + } + } + } + return s; + }; const _renderStreamCompositeNow = () => { const bubble = ensureStreamBubble(); const md = bubble.querySelector(".chat-msg__md"); @@ -2977,6 +3106,17 @@ async function renderChatUi() { } } let textIdx = 0; + let textBuf = ""; + const flushTextBuf = () => { + if (!String(textBuf || "").trim()) { + textBuf = ""; + return; + } + blocks.push( + `
${escapeHtml(decodeEscapedNewlines(textBuf)).replace(/\\n/g, "
")}
`, + ); + textBuf = ""; + }; for (const seg of chatStreamSegments) { if (!seg) continue; if (seg.type === "text") { @@ -2986,18 +3126,23 @@ async function renderChatUi() { const shown = full.slice(0, showLen); textIdx += full.length; if (!shown) continue; - blocks.push(`
${renderMarkdownHtml(decodeEscapedNewlines(shown))}
`); - } else if (seg.type === "tool") { - if (!showToolOutput) continue; + textBuf += shown; + continue; + } + if (seg.type === "tool") { const title = String(seg.title || "tool"); const body = String(seg.body || ""); + const images = Array.isArray(seg.images) ? seg.images : []; + if (!showToolOutput && !images.length) continue; + flushTextBuf(); const audit = seg.sqlAudit && typeof seg.sqlAudit === "object" ? seg.sqlAudit : null; - if (audit) { - const guard = audit.guard && typeof audit.guard === "object" ? audit.guard : {}; - const autoLimit = _sqlLimitSuffix(audit.inputSql, audit.executedSql); - const executedDiffHtml = _renderExecutedSqlWithAddedHighlight(audit.inputSql, audit.executedSql); - blocks.push( - `
${escapeHtml(title)} · SQL audit + if (showToolOutput) { + if (audit) { + const guard = audit.guard && typeof audit.guard === "object" ? audit.guard : {}; + const autoLimit = _sqlLimitSuffix(audit.inputSql, audit.executedSql); + const executedDiffHtml = _renderExecutedSqlWithAddedHighlight(audit.inputSql, audit.executedSql); + blocks.push( + `
${escapeHtml(title)} · SQL audit
input SQL
${escapeHtml(String(audit.inputSql || ""))}
executed SQL (added highlighted)
${executedDiffHtml}
@@ -3012,18 +3157,23 @@ ${autoLimit ? `
auto-added claus rows_returned=${escapeHtml(String(audit.rowsReturned != null ? audit.rowsReturned : ""))}
`, - ); - } else { - blocks.push( - `
${escapeHtml(title)}
${escapeHtml(body)}
`, - ); + ); + } else { + blocks.push( + `
${escapeHtml(title)}
${escapeHtml(body)}
`, + ); + } + } + for (const im of images) { + const src = String((im && im.src) || "").trim(); + if (!src) continue; + blocks.push(`
tool image
`); } } } const currentShown = streamDisplayShown.slice(textIdx); - if (String(currentShown || "").trim()) { - blocks.push(`
${renderMarkdownHtml(decodeEscapedNewlines(currentShown))}
`); - } + if (String(currentShown || "").trim()) textBuf += currentShown; + flushTextBuf(); md.innerHTML = blocks.join(""); } scrollMessagesToBottom(); @@ -3066,14 +3216,33 @@ ${autoLimit ? `
auto-added claus }, 14); }); }; + const _sanitizeStreamDelta = (delta) => { + let s = streamStitcher.push(delta); + if (!s) return ""; + // Detect and suppress "one char per line" noise early in the stream. + // Typical pattern: "好\n问\n题\n" or chunks like "好\n". + const nl = (s.match(/\n/g) || []).length; + const nonNlLen = s.replace(/\n/g, "").length; + if (nl > 0 && nonNlLen > 0 && nonNlLen <= 2 && nl >= nonNlLen) { + perCharNewlineScore += 1; + } else if (nonNlLen >= 4 && nl === 0) { + perCharNewlineScore = Math.max(0, perCharNewlineScore - 1); + } + if (!perCharNewlineMode && perCharNewlineScore >= 3) perCharNewlineMode = true; + if (perCharNewlineMode) { + // Drop all newlines in this mode; rely on natural wrapping. + s = s.replace(/\n+/g, ""); + } + return s; + }; const appendStreamTextChunk = (chunk) => { - const piece = String(chunk || ""); + const piece = _sanitizeStreamDelta(chunk); if (!piece) return; streamTextBuffer = `${streamTextBuffer}${piece}`; chatStream = streamTextBuffer; // Keep strict chronological order: append text deltas as independent segments. chatStreamSegments.push({ type: "text", text: piece }); - streamDisplayTarget = _composeStreamPlainText(); + streamDisplayTarget = _normalizeStreamTargetForRender(_composeStreamPlainText()); _scheduleTypingTick(); }; const appendToolSegment = (payload) => { @@ -3084,7 +3253,8 @@ ${autoLimit ? `
auto-added claus const rawPayload = p.payload != null ? p.payload : p; const body = formatToolPanelText(name, rawPayload, { streamMode: true }); const sqlAudit = extractSqlAuditPayload(rawPayload); - chatStreamSegments.push({ type: "tool", key, title: `${name} ${liveTag}`, body, sqlAudit }); + const images = extractToolImageItems(rawPayload); + chatStreamSegments.push({ type: "tool", key, title: `${name} ${liveTag}`, body, sqlAudit, images }); }; const appendFinalAssistant = async (message, fallbackText) => { const normalized = _normalizeAssistantMessage(message, { requireRole: false, requireContentArray: false }); @@ -3198,7 +3368,7 @@ ${autoLimit ? `
auto-added claus _setStreamStatusBase("tool"); // Keep strict order: tool card appears exactly where tool event arrives. appendToolSegment(payload); - streamDisplayTarget = _composeStreamPlainText(); + streamDisplayTarget = _normalizeStreamTargetForRender(_composeStreamPlainText()); _scheduleTypingTick(); return; } @@ -3233,7 +3403,18 @@ ${autoLimit ? `
auto-added claus sawWsTerminalEvent = true; _stopDynamicStreamStatus(); streamDisplayShown = streamDisplayTarget; - const ok = await appendFinalAssistant(payload.message, chatStream || extractWsAssistantText(payload.message || {})); + const hasStreamToolImages = chatStreamSegments.some( + (seg) => seg && seg.type === "tool" && Array.isArray(seg.images) && seg.images.length > 0, + ); + let ok = false; + if (hasStreamToolImages) { + // Keep stream bubble as final UI when it already contains image blocks. + renderStreamComposite(); + _markStreamTerminal("end", t("chat.status.end")); + ok = true; + } else { + ok = await appendFinalAssistant(payload.message, chatStream || extractWsAssistantText(payload.message || {})); + } if (!ok) _markStreamTerminal("end", t("chat.status.end")); chatStream = ""; streamTextBuffer = ""; @@ -3280,24 +3461,28 @@ ${autoLimit ? `
auto-added claus }, }); let wsWatchdog = 0; - const wsInactivityTimeoutPromise = new Promise((_, reject) => { - wsWatchdog = setInterval(() => { - if (Date.now() - wsLastActivityAt > WS_CHAT_SEND_TIMEOUT_MS) { - if (wsWatchdog) { - clearInterval(wsWatchdog); - wsWatchdog = 0; - } - reject(new Error(`ws_send_timeout:${WS_CHAT_SEND_TIMEOUT_MS}`)); - } - }, 1000); - }); const wsSendObserved = wsSendPromise.finally(() => { if (wsWatchdog) { clearInterval(wsWatchdog); wsWatchdog = 0; } }); - doneMeta = await Promise.race([wsSendObserved, wsInactivityTimeoutPromise]); + if (Number(WS_CHAT_SEND_TIMEOUT_MS || 0) > 0) { + const wsInactivityTimeoutPromise = new Promise((_, reject) => { + wsWatchdog = setInterval(() => { + if (Date.now() - wsLastActivityAt > WS_CHAT_SEND_TIMEOUT_MS) { + if (wsWatchdog) { + clearInterval(wsWatchdog); + wsWatchdog = 0; + } + reject(new Error(`ws_send_timeout:${WS_CHAT_SEND_TIMEOUT_MS}`)); + } + }, 1000); + }); + doneMeta = await Promise.race([wsSendObserved, wsInactivityTimeoutPromise]); + } else { + doneMeta = await wsSendObserved; + } if (doneMeta && typeof doneMeta === "object") doneMeta.__transport = "ws"; if (doneMeta && typeof doneMeta === "object") { const startToRunning = diff --git a/runtime/chat/tool_runtime.py b/runtime/chat/tool_runtime.py index 288b3c06..1b34b92c 100644 --- a/runtime/chat/tool_runtime.py +++ b/runtime/chat/tool_runtime.py @@ -52,6 +52,95 @@ _VIDEO_QUERY_TOOL_NAMES = { } +def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]: + """Extract renderable attachments from tool results for durable chat history.""" + if not isinstance(result, dict): + return [] + out: list[dict[str, Any]] = [] + aid = str(result.get("attachment_id") or "").strip() + if aid: + out.append( + { + "type": "image_ref", + "attachment_id": aid, + "name": str(result.get("name") or "generated-image"), + "mime": str(result.get("mime") or "image/png"), + "bytes": result.get("bytes"), + "width": result.get("width"), + "height": result.get("height"), + } + ) + refs = result.get("attachments") + if isinstance(refs, list): + for r in refs: + if not isinstance(r, dict): + continue + p_uri = str(r.get("pointer_uri") or "").strip() + if p_uri: + out.append( + { + "type": "relay_pointer", + "pointer_uri": p_uri, + "rel_path": str(r.get("rel_path") or ""), + "mime": str(r.get("mime_type") or r.get("mime") or ""), + "bytes": r.get("bytes"), + "sha256": str(r.get("sha256") or ""), + "name": str(r.get("name") or ""), + } + ) + continue + r_aid = str(r.get("attachment_id") or "").strip() + if r_aid: + out.append( + { + "type": "image_ref", + "attachment_id": r_aid, + "name": str(r.get("name") or "generated-image"), + "mime": str(r.get("mime") or "image/png"), + "bytes": r.get("bytes"), + "width": r.get("width"), + "height": r.get("height"), + } + ) + inner = result.get("result") + if isinstance(inner, dict): + content = inner.get("content") + if isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + typ = str(item.get("type") or "").strip().lower() + if typ in {"image", "input_image"}: + b64 = item.get("image_base64") or item.get("data") + if isinstance(b64, str) and b64.strip(): + out.append( + { + "type": "image", + "data": b64.strip(), + "mime": str(item.get("mime_type") or item.get("mime") or "image/png"), + "name": str(item.get("name") or "tool-image"), + } + ) + elif typ == "image_url": + src = str(item.get("url") or item.get("image_url") or "").strip() + if src: + out.append({"type": "image_url", "url": src, "name": str(item.get("name") or "tool-image")}) + uniq: list[dict[str, Any]] = [] + seen: set[str] = set() + for a in out: + k = str( + a.get("attachment_id") + or a.get("pointer_uri") + or a.get("url") + or (f"b64:{a.get('mime')}:{len(str(a.get('data') or ''))}" if a.get("data") else "") + ).strip() + if not k or k in seen: + continue + seen.add(k) + uniq.append(a) + return uniq + + def _message_has_tabular_ref(raw_attachments: Any) -> bool: if raw_attachments is None: return False @@ -781,6 +870,7 @@ class ToolExecutor: role="tool", content=tool_content, tool_calls={"tool_call_id": tc.id, "name": tc.name, "assistant_message_id": assistant_msg_id}, + attachments=_attachments_from_tool_result(result) or None, turn_uuid=ctx.turn_uuid, event_type="tool_result", event_payload={"tool_name": tc.name, "observed_rows": int(observed_rows_this_call)}, diff --git a/tests/test_tool_loop_guard.py b/tests/test_tool_loop_guard.py index beb90562..cc2e66d1 100644 --- a/tests/test_tool_loop_guard.py +++ b/tests/test_tool_loop_guard.py @@ -392,3 +392,43 @@ def test_video_tools_allowed_with_video_ref(tmp_path: Path) -> None: assert bool(ok_res.get("ok")) assert calls["n"] == 1 + +def test_tool_result_image_payload_persisted_as_attachments(tmp_path: Path) -> None: + store = SqliteStore(str(tmp_path / "g12.sqlite")) + sess = store.create_session("t") + + def _handler(_args): + return { + "ok": True, + "result": { + "content": [ + {"type": "image", "data": "YWJj", "mime_type": "image/png"}, + {"type": "image_url", "url": "https://example.com/a.png"}, + ] + }, + } + + reg = ToolRegistry( + [ + ToolSpec( + name="make_image", + description="make image", + parameters={"type": "object", "properties": {}}, + handler=_handler, + read_only=True, + ) + ] + ) + tool_uses = [LLMToolCall(id="c1", name="make_image", arguments={})] + ToolExecutor().execute_tool_uses( + ctx=ToolExecutionContext(store=store, tools=reg, session_id=sess.id), + assistant_msg_id=1, + tool_uses=tool_uses, + ) + rows = store.get_messages(session_id=sess.id, limit=10) + tool_rows = [m for m in rows if str(getattr(m, "role", "") or "") == "tool"] + assert len(tool_rows) == 1 + attachments = json.loads(str(getattr(tool_rows[0], "attachments", "") or "[]")) + assert any(str(a.get("type") or "") == "image" and str(a.get("data") or "") == "YWJj" for a in attachments) + assert any(str(a.get("type") or "") == "image_url" and str(a.get("url") or "").endswith("/a.png") for a in attachments) +