From 6cfaff06f601e119dd75d572b202df0615851677 Mon Sep 17 00:00:00 2001 From: oliver Date: Tue, 28 Apr 2026 10:30:27 +0800 Subject: [PATCH] Refactor media payload handling to persist base64 blobs as attachment refs and keep non-turn model paths safe by degrading risky payloads. This preserves multimodal fidelity for the latest user turn while preventing historical/tool replay bloat, and adds admin UI support for referenced attachment preview/download flows. Made-with: Cursor --- interfaces/admin/chat_api.py | 16 + interfaces/admin/static/chat.html | 76 +++- interfaces/admin/static/chat.js | 469 ++++++++++++++++++++- interfaces/admin/static/theme-deepseek.css | 184 ++++++++ platform/persistence/sqlite_store.py | 23 + runtime/chat/agent_messages.py | 98 +++-- runtime/chat/media_redact.py | 198 +++++++++ runtime/chat/model_path_audit.py | 87 ++++ runtime/chat/tool_runtime.py | 70 ++- runtime/direct_loop.py | 61 ++- runtime/gateway.py | 18 +- runtime/router.py | 2 + runtime/skills/aiops-telecom/SKILL.md | 144 +++++++ runtime/worker.py | 9 +- tests/test_media_redact.py | 44 ++ tests/test_model_path_audit.py | 46 ++ tests/test_oclaw_tool_result_guard.py | 40 ++ tests/test_tool_loop_guard.py | 96 ++++- tests/test_tool_pairing_messages.py | 16 +- tests/test_tool_runtime_attachments.py | 30 ++ 20 files changed, 1626 insertions(+), 101 deletions(-) create mode 100644 runtime/chat/media_redact.py create mode 100644 runtime/chat/model_path_audit.py create mode 100644 runtime/skills/aiops-telecom/SKILL.md create mode 100644 tests/test_media_redact.py create mode 100644 tests/test_model_path_audit.py create mode 100644 tests/test_tool_runtime_attachments.py diff --git a/interfaces/admin/chat_api.py b/interfaces/admin/chat_api.py index 0d314b56..ec7ebfdb 100644 --- a/interfaces/admin/chat_api.py +++ b/interfaces/admin/chat_api.py @@ -877,6 +877,22 @@ def include_chat_routes(router: APIRouter, *, resolve_auth: Callable[[SqliteStor "messages": [_serialize_message(m) for m in msgs], } + @chat.delete("/sessions/{session_id}/messages/{message_id}") + def api_chat_delete_message( + session_id: str, + message_id: int, + authorization: str | None = Header(default=None), + ) -> dict[str, Any]: + store = SqliteStore(db_path()) + ctx = resolve_auth(store, authorization) + sess = _resolve_chat_session(store, ctx, session_id) + if not sess: + raise HTTPException(status_code=404, detail="session_not_found") + ok = store.delete_message(session_id=str(session_id), message_id=int(message_id)) + if not ok: + raise HTTPException(status_code=404, detail="message_not_found") + return {"ok": True, "deleted": 1} + @chat.get("/sessions/{session_id}/wiki-events") def api_chat_wiki_events( session_id: str, diff --git a/interfaces/admin/static/chat.html b/interfaces/admin/static/chat.html index 2211b173..78600cd6 100644 --- a/interfaces/admin/static/chat.html +++ b/interfaces/admin/static/chat.html @@ -243,6 +243,68 @@ font-size: 12px; opacity: 0.9; } + .chat-att-ref { + min-width: 180px; + max-width: 260px; + border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.14)); + border-radius: 8px; + background: rgba(255, 255, 255, 0.03); + padding: 8px 10px; + box-sizing: border-box; + } + .chat-att-ref__name { + font-size: 12px; + font-weight: 600; + line-height: 1.35; + margin-bottom: 4px; + word-break: break-all; + } + .chat-att-ref__meta { + font-size: 11px; + line-height: 1.35; + opacity: 0.75; + word-break: break-all; + } + .chat-att-ref__link { + display: inline-block; + margin-top: 6px; + font-size: 12px; + color: var(--ds-accent, #5eb3ff); + text-decoration: none; + } + .chat-att-ref__link:hover { + text-decoration: underline; + } + .chat-att-ref__btn { + margin-top: 6px; + border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.14)); + background: rgba(255, 255, 255, 0.04); + color: inherit; + border-radius: 6px; + font-size: 12px; + line-height: 1.2; + padding: 4px 8px; + cursor: pointer; + } + .chat-att-ref__btn:hover { + background: rgba(255, 255, 255, 0.08); + } + .chat-att-ref__preview { + margin: 6px 0 0; + padding: 8px; + border-radius: 6px; + border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.14)); + background: rgba(0, 0, 0, 0.26); + max-height: 180px; + overflow: auto; + white-space: pre-wrap; + word-break: break-word; + font-size: 11px; + line-height: 1.35; + } + .chat-att-ref__preview:empty { + display: none; + } .chat-composer-shell { border-radius: 12px; border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.12)); @@ -484,13 +546,19 @@ white-space: pre-wrap; } - - - + + + + diff --git a/interfaces/admin/static/chat.js b/interfaces/admin/static/chat.js index 63eb2852..6defe772 100644 --- a/interfaces/admin/static/chat.js +++ b/interfaces/admin/static/chat.js @@ -21,6 +21,11 @@ const I18N = { "chat.sessionMenu": "会话操作", "chat.rename": "重命名", "chat.delete": "删除", + "chat.copy": "复制", + "chat.deleteMessage": "删除消息", + "chat.deleteMessageConfirm": "删除这条消息？", + "chat.copyOk": "已复制", + "chat.copyFail": "复制失败", "chat.deleteConfirm": "删除此会话？", "chat.exportMd": "导出 Markdown", "chat.exportJson": "导出 JSON", @@ -138,6 +143,11 @@ const I18N = { "chat.specialistImageShort": "图像", "chat.specialistMemoryShort": "记忆", "chat.specialistManagerSelfShort": "全能者", + "chat.attachment.download": "下载", + "chat.attachment.preview": "预览", + "chat.attachment.previewLoading": "加载中…", + "chat.attachment.previewError": "预览失败", + "chat.attachment.previewEmpty": "（空内容）", }, en: { "chat.pageTitle": "oliver", @@ -157,6 +167,11 @@ const I18N = { "chat.sessionMenu": "Session actions", "chat.rename": "Rename", "chat.delete": "Delete", + "chat.copy": "Copy", + "chat.deleteMessage": "Delete message", + "chat.deleteMessageConfirm": "Delete this message?", + "chat.copyOk": "Copied", + "chat.copyFail": "Copy failed", "chat.deleteConfirm": "Delete this session?", "chat.exportMd": "Export Markdown", "chat.exportJson": "Export JSON", @@ -274,6 +289,11 @@ const I18N = { "chat.specialistImageShort": "Image", "chat.specialistMemoryShort": "Memory", "chat.specialistManagerSelfShort": "Manager", + "chat.attachment.download": "Download", + "chat.attachment.preview": "Preview", + "chat.attachment.previewLoading": "Loading…", + "chat.attachment.previewError": "Preview failed", + "chat.attachment.previewEmpty": "(empty)", }, }; @@ -490,8 +510,10 @@ function _buildRenderRows(msgs) { timestamp: (m && m.timestamp) != null ? m.timestamp : "", attachments: null, _items: [], + _message_ids: [], }; } + if (m && m.id != null) agg._message_ids.push(m.id); if (role === "assistant") { if (eventType === "reasoning") { if (String(content || "").trim()) { @@ -1035,6 +1057,134 @@ function openChatImageLightbox(src, alt) { document.body.appendChild(backdrop); } +function openChatMermaidLightbox(svg) { + const raw = String(svg || "").trim(); + if (!raw) return; + closeChatImageLightbox(); + _chatLightboxPrevOverflow = document.body.style.overflow; + document.body.style.overflow = "hidden"; + _chatLightboxKeyHandler = (ev) => { + if (ev.key === "Escape") closeChatImageLightbox(); + }; + document.addEventListener("keydown", _chatLightboxKeyHandler); + + const backdrop = el("div", { + class: "chat-img-lightbox", + role: "dialog", + "aria-modal": "true", + "aria-label": "Mermaid diagram viewer", + }); + const inner = el("div", { class: "chat-img-lightbox__inner" }); + let scale = 1.0; + const clamp = (v) => Math.max(0.2, Math.min(3.0, Number(v || 1))); + const applyScale = () => { + scale = clamp(scale); + viewport.style.transform = `scale(${scale})`; + zoomText.textContent = `${Math.round(scale * 100)}%`; + }; + const closeBtn = el("button", { + type: "button", + class: "chat-img-lightbox__close", + text: "×", + "aria-label": t("chat.imageViewerClose"), + onclick: (e) => { + e.stopPropagation(); + closeChatImageLightbox(); + }, + }); + const toolbar = el("div", { class: "chat-mermaid-lightbox__toolbar" }); + const btnMinus = el("button", { + type: "button", + class: "chat-mermaid-lightbox__btn", + text: "−", + onclick: (e) => { + e.stopPropagation(); + scale = clamp(scale - 0.1); + applyScale(); + }, + }); + const btnPlus = el("button", { + type: "button", + class: "chat-mermaid-lightbox__btn", + text: "+", + onclick: (e) => { + e.stopPropagation(); + scale = clamp(scale + 0.1); + applyScale(); + }, + }); + const btnReset = el("button", { + type: "button", + class: "chat-mermaid-lightbox__btn", + text: "100%", + onclick: (e) => { + e.stopPropagation(); + scale = 1.0; + applyScale(); + }, + }); + const zoomText = el("span", { class: "chat-mermaid-lightbox__zoom", text: "100%" }); + toolbar.appendChild(btnMinus); + toolbar.appendChild(btnReset); + toolbar.appendChild(btnPlus); + toolbar.appendChild(zoomText); + + const wrap = el("div", { class: "chat-mermaid-lightbox__svg" }); + const viewport = el("div", { class: "chat-mermaid-lightbox__viewport" }); + viewport.innerHTML = raw; + wrap.appendChild(viewport); + // Drag-to-pan (scroll) inside the zoomable viewport. + let dragging = false; + let dragStartX = 0; + let dragStartY = 0; + let dragScrollLeft = 0; + let dragScrollTop = 0; + const onDragStart = (e) => { + // Ignore drags started on toolbar/buttons. + if (e && e.target && e.target.closest && e.target.closest(".chat-mermaid-lightbox__toolbar")) return; + dragging = true; + wrap.classList.add("chat-mermaid-lightbox__svg--dragging"); + dragStartX = Number(e.clientX || 0); + dragStartY = Number(e.clientY || 0); + dragScrollLeft = wrap.scrollLeft; + dragScrollTop = wrap.scrollTop; + }; + const onDragMove = (e) => { + if (!dragging) return; + const x = Number(e.clientX || 0); + const y = Number(e.clientY || 0); + wrap.scrollLeft = dragScrollLeft - (x - dragStartX); + wrap.scrollTop = dragScrollTop - (y - dragStartY); + }; + const onDragEnd = () => { + dragging = false; + wrap.classList.remove("chat-mermaid-lightbox__svg--dragging"); + }; + wrap.addEventListener("mousedown", (e) => onDragStart(e)); + window.addEventListener("mousemove", (e) => onDragMove(e)); + window.addEventListener("mouseup", () => onDragEnd()); + wrap.addEventListener( + "wheel", + (e) => { + e.preventDefault(); + const dy = Number(e.deltaY || 0); + const step = dy > 0 ? -0.08 : 0.08; + scale = clamp(scale + step); + applyScale(); + }, + { passive: false }, + ); + inner.appendChild(closeBtn); + inner.appendChild(toolbar); + inner.appendChild(wrap); + backdrop.appendChild(inner); + backdrop.addEventListener("click", (e) => { + if (e.target === backdrop) closeChatImageLightbox(); + }); + document.body.appendChild(backdrop); + applyScale(); +} + function bindChatImageViewer(messagesEl) { messagesEl.addEventListener("click", (ev) => { const img = ev.target && ev.target.closest && ev.target.closest("img"); @@ -1047,6 +1197,16 @@ function bindChatImageViewer(messagesEl) { }); } +function bindChatMermaidViewer(messagesEl) { + messagesEl.addEventListener("click", (ev) => { + const svg = ev.target && ev.target.closest && ev.target.closest(".mermaid svg"); + if (!svg || !messagesEl.contains(svg)) return; + ev.preventDefault(); + ev.stopPropagation(); + openChatMermaidLightbox(svg.outerHTML || ""); + }); +} + /** Keep markdown output safe but allow images (USE_PROFILES html strips img in many DOMPurify builds). */ function renderMarkdownHtml(src) { const raw = String(src ?? ""); @@ -1068,6 +1228,103 @@ function renderMarkdownHtml(src) { return `
${escapeHtml(raw).replace(/\n/g, "
")}
`; } +let _mermaidBootstrapped = false; +let _mermaidRetryTimer = null; +window.__oclawHydrateMermaidAll = () => { + try { + const root = document.getElementById("app") || document.body; + hydrateMermaidIn(root); + } catch (_) {} +}; +function hydrateMermaidIn(root) { + const host = root && root.querySelectorAll ? root : null; + if (!host) return; + const codeNodes = host.querySelectorAll("pre > code.language-mermaid, pre > code.lang-mermaid"); + for (const code of codeNodes) { + const pre = code.parentElement; + if (!pre || !pre.parentElement) continue; + const txt = String(code.textContent || "").trim(); + if (!txt) continue; + const box = document.createElement("div"); + box.className = "mermaid"; + box.setAttribute("data-mermaid-raw", txt); + box.textContent = txt; + pre.parentElement.replaceChild(box, pre); + } + const _attachFallback = (node, errText = "") => { + if (!node || node.querySelector(".mermaid-fallback")) return; + const raw = String(node.getAttribute("data-mermaid-raw") || "").trim() || String(node.textContent || "").trim(); + const msg = String(errText || "").trim(); + node.innerHTML = `
${msg ? `
${escapeHtml(msg)}
` : ""}
${escapeHtml(raw)}
`; + }; + if (typeof mermaid === "undefined") { + if (_mermaidRetryTimer != null) return; + _mermaidRetryTimer = setTimeout(() => { + _mermaidRetryTimer = null; + try { + window.__oclawHydrateMermaidAll(); + } catch (_) {} + }, 350); + return; + } + try { + if (!_mermaidBootstrapped && typeof mermaid.initialize === "function") { + mermaid.initialize({ + startOnLoad: false, + securityLevel: "loose", + theme: "base", + themeVariables: { + background: "transparent", + primaryColor: "#1f2937", + primaryBorderColor: "#94a3b8", + primaryTextColor: "#e5e7eb", + lineColor: "#94a3b8", + textColor: "#e5e7eb", + fontFamily: "Inter, system-ui, -apple-system, Segoe UI, Roboto, sans-serif", + }, + }); + _mermaidBootstrapped = true; + } + const nodes = Array.from(host.querySelectorAll(".mermaid")).filter((n) => n && n.isConnected); + if (!nodes.length) return; + // Mermaid skips already processed nodes. In streaming / rerender scenarios, DOM can be replaced. + // Remove the marker so Mermaid treats these nodes as fresh. + for (const n of nodes) { + try { + n.removeAttribute("data-processed"); + } catch (_) {} + } + if (typeof mermaid.run === "function") { + Promise.resolve(mermaid.run({ nodes })) + .then(() => { + for (const n of nodes) { + if (!n || !n.isConnected) continue; + if (!n.querySelector("svg")) _attachFallback(n, "Mermaid render failed (no svg output)"); + } + }) + .catch((e) => { + const msg = String((e && e.message) || e || "Mermaid render failed"); + for (const n of nodes) _attachFallback(n, msg); + }); + } else if (typeof mermaid.init === "function") { + try { + mermaid.init(undefined, nodes); + for (const n of nodes) { + if (!n || !n.isConnected) continue; + if (!n.querySelector("svg")) _attachFallback(n, "Mermaid render failed (no svg output)"); + } + } catch (e) { + const msg = String((e && e.message) || e || "Mermaid render failed"); + for (const n of nodes) _attachFallback(n, msg); + } + } + } catch (e) { + const msg = String((e && e.message) || e || "Mermaid render failed"); + const nodes = host.querySelectorAll(".mermaid"); + for (const n of nodes) _attachFallback(n, msg); + } +} + const RE_REDACTED_THINKING = new RegExp("\\s*([\\s\\S]*?)\\s*", "i"); const RE_THINK_TAG = new RegExp("\\s*([\\s\\S]*?)\\s*", "i"); @@ -1611,6 +1868,7 @@ async function buildMessageBubble(role, content, tsIso) { } const _blobUrlCache = new Map(); +const _attachmentTextPreviewCache = new Map(); async function fetchAttachmentBlobUrl(attachmentId) { const aid = String(attachmentId || "").trim(); @@ -1627,6 +1885,22 @@ async function fetchAttachmentBlobUrl(attachmentId) { return url; } +async function fetchAttachmentTextPreview(attachmentId, maxChars = 1800) { + const aid = String(attachmentId || "").trim(); + if (!aid) return ""; + if (_attachmentTextPreviewCache.has(aid)) return _attachmentTextPreviewCache.get(aid) || ""; + const token = localStorage.getItem(AUTH_TOKEN_KEY) || ""; + const res = await fetch(`/admin/api/chat/attachments/${encodeURIComponent(aid)}`, { + headers: token ? { authorization: `Bearer ${token}` } : {}, + }); + if (!res.ok) throw new Error(`preview_http_${res.status}`); + const blob = await res.blob(); + const txt = String(await blob.text()); + const out = txt.length > maxChars ? `${txt.slice(0, maxChars)}\n…` : txt; + _attachmentTextPreviewCache.set(aid, out); + return out; +} + function parseAttachments(raw) { if (raw == null || raw === "") return []; if (Array.isArray(raw)) return raw.filter((x) => x && typeof x === "object"); @@ -1651,6 +1925,77 @@ async function renderAttachmentsEl(raw) { const list = parseAttachments(raw); if (!list.length) return null; const wrap = el("div", { class: "chat-att-wrap" }); + const isTextLikeMime = (mime) => { + const m = String(mime || "").trim().toLowerCase(); + if (!m) return false; + if (m.startsWith("text/")) return true; + return ( + m.includes("json") || + m.includes("xml") || + m.includes("yaml") || + m.includes("yml") || + m.includes("csv") || + m.includes("javascript") || + m.includes("typescript") + ); + }; + const buildRefCard = async (att, typ) => { + const aid = String(att.attachment_id || att.attachmentId || "").trim(); + const mime = String(att.mime || att.mime_type || "application/octet-stream").trim(); + const name = String(att.name || `${typ || "attachment"}`).trim(); + const bytes = Number(att.bytes || 0); + const sizeLabel = bytes > 0 ? `${Math.round((bytes / 1024) * 10) / 10} KB` : ""; + const card = el("div", { class: "chat-att-ref" }); + card.appendChild(el("div", { class: "chat-att-ref__name", text: name })); + card.appendChild(el("div", { class: "chat-att-ref__meta", text: `${typ} · ${mime}${sizeLabel ? ` · ${sizeLabel}` : ""}` })); + if (aid) card.appendChild(el("div", { class: "chat-att-ref__meta", text: `id: ${aid.slice(0, 16)}...` })); + if (aid) { + const url = await fetchAttachmentBlobUrl(aid); + if (url) { + card.appendChild( + el("a", { + class: "chat-att-ref__link", + href: url, + target: "_blank", + rel: "noopener noreferrer", + download: name || undefined, + text: t("chat.attachment.download"), + }), + ); + } + } + const canPreviewText = !!aid && (String(typ || "") === "text_ref" || isTextLikeMime(mime)); + if (canPreviewText) { + const preview = el("button", { + type: "button", + class: "chat-att-ref__btn", + text: t("chat.attachment.preview"), + }); + const pre = el("pre", { class: "chat-att-ref__preview" }); + preview.addEventListener("click", async () => { + if (!pre.hidden) { + pre.hidden = true; + preview.textContent = t("chat.attachment.preview"); + return; + } + preview.disabled = true; + preview.textContent = t("chat.attachment.previewLoading"); + try { + const txt = await fetchAttachmentTextPreview(aid); + pre.textContent = txt || t("chat.attachment.previewEmpty"); + } catch (_) { + pre.textContent = t("chat.attachment.previewError"); + } finally { + pre.hidden = false; + preview.disabled = false; + preview.textContent = t("chat.attachment.preview"); + } + }); + card.appendChild(preview); + card.appendChild(pre); + } + return card; + }; for (const att of list) { if (!att || typeof att !== "object") continue; const typ = String(att.type || ""); @@ -1683,6 +2028,8 @@ async function renderAttachmentsEl(raw) { const src = String(att.url || att.image_url || "").trim(); const img = el("img", { class: "chat-att-img", src, alt: "" }); wrap.appendChild(img); + } else if (typ === "video_ref" || typ === "text_ref" || typ === "binary_ref") { + wrap.appendChild(await buildRefCard(att, typ)); } else if (typ === "image" || typ === "input_image") { const b64 = att.image_base64 || att.data; const mime = String(att.mime || "image/jpeg"); @@ -1693,13 +2040,15 @@ async function renderAttachmentsEl(raw) { wrap.appendChild(el("span", { class: "chat-att-chip", text: String(att.name || "image") })); } } else { - wrap.appendChild(el("span", { class: "chat-att-chip", text: `📄 ${String(att.name || "file")}` })); + const maybeAid = String(att.attachment_id || att.attachmentId || "").trim(); + if (maybeAid) wrap.appendChild(await buildRefCard(att, typ || "attachment_ref")); + else wrap.appendChild(el("span", { class: "chat-att-chip", text: `📄 ${String(att.name || "file")}` })); } } return wrap.children.length ? wrap : null; } -async function appendMessageRow(messagesEl, m) { +async function appendMessageRow(messagesEl, m, options = {}) { const role = String(m.role || ""); const content = String(m.content || ""); const ts = m.timestamp != null ? m.timestamp : ""; @@ -1714,7 +2063,67 @@ async function appendMessageRow(messagesEl, m) { if (innerBubble) innerBubble.appendChild(att); else bubble.appendChild(att); } + const colNode = bubble.querySelector(".chat-msg-col"); + const innerBubble = bubble.querySelector(".chat-msg-col .chat-msg"); + if (innerBubble) { + const copyText = (() => { + const v = String(innerBubble.innerText || "").trim(); + return v; + })(); + const ids = Array.isArray(m._message_ids) ? m._message_ids.filter((x) => x != null) : (m.id != null ? [m.id] : []); + const canDelete = ids.length === 1 && typeof options.onDeleteMessage === "function"; + if (copyText || canDelete) { + const bar = el("div", { class: "chat-msg__actions" }); + if (copyText) { + bar.appendChild( + el("button", { + type: "button", + class: "chat-msg__action-btn", + text: "⧉", + title: t("chat.copy"), + "aria-label": t("chat.copy"), + onclick: async (e) => { + e.preventDefault(); + e.stopPropagation(); + try { + await navigator.clipboard.writeText(copyText); + if (typeof options.onActionStatus === "function") options.onActionStatus(t("chat.copyOk")); + } catch (_) { + if (typeof options.onActionStatus === "function") options.onActionStatus(t("chat.copyFail")); + } + }, + }), + ); + } + if (canDelete) { + bar.appendChild( + el("button", { + type: "button", + class: "chat-msg__action-btn chat-msg__action-btn--danger", + text: "🗑", + title: t("chat.deleteMessage"), + "aria-label": t("chat.deleteMessage"), + onclick: async (e) => { + e.preventDefault(); + e.stopPropagation(); + let ok = false; + if (typeof options.onConfirm === "function") { + ok = await options.onConfirm(t("chat.deleteMessageConfirm")); + } else { + ok = window.confirm(t("chat.deleteMessageConfirm")); + } + if (!ok) return; + await options.onDeleteMessage(ids[0]); + }, + }), + ); + } + if (colNode) colNode.appendChild(bar); + else innerBubble.appendChild(bar); + } + } messagesEl.appendChild(bubble); + hydrateMermaidIn(bubble); } function mount(node) { @@ -1917,6 +2326,7 @@ async function renderChatUi() { const loadMoreWrap = el("div", { class: "chat-load-more" }); const messagesEl = el("div", { class: "chat-messages" }); bindChatImageViewer(messagesEl); + bindChatMermaidViewer(messagesEl); const AUTO_SCROLL_BOTTOM_GAP_PX = 56; let shouldFollowMessages = true; const isNearBottom = () => { @@ -2471,6 +2881,22 @@ async function renderChatUi() { return Number.isFinite(n) ? n : d; } + const handleDeleteMessage = async (messageId) => { + const sid = String(activeId || "").trim(); + const mid = parseInt(String(messageId || 0), 10); + if (!sid || !Number.isFinite(mid) || mid <= 0) return; + await apiDelete(`/admin/api/chat/sessions/${encodeURIComponent(sid)}/messages/${mid}`); + await loadMessagesForActive(); + }; + + const rowRenderOptions = { + onDeleteMessage: handleDeleteMessage, + onConfirm: confirmChatAction, + onActionStatus: (msg) => { + statusBar.textContent = String(msg || ""); + }, + }; + const loadMessagesForActive = async () => { loadMessagesForActive._rid = (loadMessagesForActive._rid || 0) + 1; const rid = loadMessagesForActive._rid; @@ -2501,7 +2927,7 @@ async function renderChatUi() { messagesEl.appendChild(el("div", { class: "muted", text: t("chat.empty") })); } else { for (const m of renderRows) { - await appendMessageRow(messagesEl, m); + await appendMessageRow(messagesEl, m, rowRenderOptions); } } statusBar.textContent = ""; @@ -2917,6 +3343,7 @@ async function renderChatUi() { let streamDisplayShown = ""; let streamTextBuffer = ""; const streamStitcher = createStreamStitcher(); + let streamMermaidTimerId = null; let perCharNewlineMode = false; let perCharNewlineScore = 0; let toolSeq = 0; @@ -3178,6 +3605,25 @@ ${autoLimit ? `
auto-added claus } scrollMessagesToBottom(); }; + const _maybeScheduleStreamMermaidHydrate = () => { + // Mermaid render is expensive and can crash if DOM is re-written mid-run. + // For streaming, only hydrate when we see a complete mermaid fence block. + const bubble = streamBubble; + if (!bubble) return; + const md = bubble.querySelector(".chat-msg__md"); + if (!md) return; + const raw = String(streamDisplayShown || ""); + if (!raw.includes("```mermaid")) return; + // Require a closed fence to avoid parsing partial streams. + if (!/```mermaid[\s\S]*?\n```/m.test(raw)) return; + if (streamMermaidTimerId != null) clearTimeout(streamMermaidTimerId); + streamMermaidTimerId = setTimeout(() => { + streamMermaidTimerId = null; + try { + hydrateMermaidIn(md); + } catch (_) {} + }, 180); + }; const renderStreamComposite = () => { if (renderPending) return; renderPending = true; @@ -3185,6 +3631,7 @@ ${autoLimit ? `
auto-added claus renderPending = false; renderRafId = null; _renderStreamCompositeNow(); + _maybeScheduleStreamMermaidHydrate(); }); }; const _scheduleTypingTick = () => { @@ -3270,7 +3717,7 @@ ${autoLimit ? `
auto-added claus const last = rows && rows.length ? rows[rows.length - 1] : null; if (last) { if (streamRow && streamRow.parentNode) streamRow.remove(); - await appendMessageRow(messagesEl, last); + await appendMessageRow(messagesEl, last, rowRenderOptions); scrollMessagesToBottom(true); return true; } @@ -3282,7 +3729,7 @@ ${autoLimit ? `
auto-added claus role: "assistant", content: decodeEscapedNewlines(t0), timestamp: new Date().toISOString(), - }); + }, rowRenderOptions); scrollMessagesToBottom(true); return true; } @@ -3423,6 +3870,10 @@ ${autoLimit ? `
auto-added claus turnFinalized = true; turnStreamedEnough = streamedEnough; chatStreamSegments = []; + if (streamMermaidTimerId != null) { + clearTimeout(streamMermaidTimerId); + streamMermaidTimerId = null; + } // Avoid end-of-turn flash: only reload history when stream had no usable content. if (!streamedEnough) { setTimeout(() => { @@ -3444,6 +3895,10 @@ ${autoLimit ? `
auto-added claus streamTextBuffer = ""; chatRunId = null; chatStreamSegments = []; + if (streamMermaidTimerId != null) { + clearTimeout(streamMermaidTimerId); + streamMermaidTimerId = null; + } return; } if (state === "error") { @@ -3456,6 +3911,10 @@ ${autoLimit ? `
auto-added claus streamTextBuffer = ""; chatRunId = null; chatStreamSegments = []; + if (streamMermaidTimerId != null) { + clearTimeout(streamMermaidTimerId); + streamMermaidTimerId = null; + } statusBar.textContent = `${t("chat.error")}: ${String(payload.errorMessage || "chat error")}`; } }, diff --git a/interfaces/admin/static/theme-deepseek.css b/interfaces/admin/static/theme-deepseek.css index ffbba424..9192979d 100644 --- a/interfaces/admin/static/theme-deepseek.css +++ b/interfaces/admin/static/theme-deepseek.css @@ -576,6 +576,190 @@ body.theme-ds-body .card { border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.08)); } +.chat-msg__actions { + margin-top: 6px; + display: flex; + gap: 8px; + justify-content: flex-start; + opacity: 0; + pointer-events: none; + transition: opacity 120ms ease; +} + +.chat-msg-col:hover .chat-msg__actions { + opacity: 1; + pointer-events: auto; +} + +.chat-msg__action-btn { + border: 1px solid rgba(148, 163, 184, 0.25); + background: transparent; + color: rgba(203, 213, 225, 0.92); + border-radius: 999px; + font-size: 15px; + width: 32px; + height: 32px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0; + cursor: pointer; +} + +.chat-msg__action-btn:hover { + background: rgba(30, 41, 59, 0.45); +} + +.chat-msg__action-btn--danger { + border-color: rgba(248, 113, 113, 0.45); + color: #fca5a5; +} + +.chat-msg__md .mermaid { + display: block; + width: 100%; + overflow: auto; + padding: 8px; + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.35); + background: rgba(15, 23, 42, 0.35); +} + +.chat-msg__md .mermaid svg { + display: block; + max-width: 100%; + height: auto; + margin: 0 auto; +} + +.chat-img-lightbox { + position: fixed; + inset: 0; + z-index: 9999; + display: flex; + align-items: center; + justify-content: center; + padding: 18px; + background: rgba(0, 0, 0, 0.78); + backdrop-filter: blur(2px); +} + +.chat-img-lightbox__inner { + position: relative; + width: min(96vw, 1400px); + max-height: 92vh; + display: flex; + align-items: center; + justify-content: center; + border-radius: 14px; + border: 1px solid rgba(148, 163, 184, 0.28); + background: rgba(2, 6, 23, 0.55); + overflow: hidden; +} + +.chat-img-lightbox__close { + position: absolute; + top: 10px; + right: 10px; + width: 36px; + height: 36px; + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.35); + background: rgba(15, 23, 42, 0.85); + color: rgba(255, 255, 255, 0.9); + cursor: pointer; + font-size: 22px; + line-height: 32px; +} + +.chat-img-lightbox__close:hover { + background: rgba(30, 41, 59, 0.9); +} + +.chat-img-lightbox__img { + max-width: 96vw; + max-height: 92vh; + width: auto; + height: auto; + object-fit: contain; +} + +.chat-mermaid-lightbox__svg { + width: min(96vw, 1400px); + max-height: 92vh; + overflow: auto; + padding: 14px; + cursor: grab; +} + +.chat-mermaid-lightbox__svg--dragging { + cursor: grabbing; +} + +.chat-mermaid-lightbox__viewport { + transform-origin: 0 0; +} + +.chat-mermaid-lightbox__toolbar { + position: absolute; + left: 10px; + top: 10px; + display: flex; + gap: 8px; + align-items: center; + padding: 6px 8px; + border-radius: 12px; + border: 1px solid rgba(148, 163, 184, 0.35); + background: rgba(15, 23, 42, 0.85); + z-index: 2; +} + +.chat-mermaid-lightbox__btn { + width: 40px; + height: 32px; + border-radius: 10px; + border: 1px solid rgba(148, 163, 184, 0.3); + background: rgba(2, 6, 23, 0.45); + color: rgba(255, 255, 255, 0.92); + cursor: pointer; + font-size: 16px; + line-height: 1; +} + +.chat-mermaid-lightbox__btn:hover { + background: rgba(30, 41, 59, 0.75); +} + +.chat-mermaid-lightbox__zoom { + color: rgba(226, 232, 240, 0.9); + font-size: 12px; + padding-left: 2px; +} + +.chat-mermaid-lightbox__svg svg { + width: max(980px, 96vw); + max-width: 100%; + height: auto; +} + +.chat-msg__md .mermaid .mermaid-fallback { + padding: 8px; + border-radius: 8px; + background: rgba(2, 6, 23, 0.55); +} + +.chat-msg__md .mermaid .mermaid-fallback__err { + color: #fca5a5; + margin-bottom: 6px; + font-size: 12px; +} + +.chat-msg__md .mermaid .mermaid-fallback pre { + margin: 0; + white-space: pre-wrap; + color: #e5e7eb; +} + .chat-msg--tool { align-self: flex-start; font-family: ui-monospace, monospace; diff --git a/platform/persistence/sqlite_store.py b/platform/persistence/sqlite_store.py index eb1673ba..684160ad 100644 --- a/platform/persistence/sqlite_store.py +++ b/platform/persistence/sqlite_store.py @@ -1613,6 +1613,29 @@ class SqliteStore: conn.execute("DELETE FROM chat_session WHERE id = ?", (str(session_id),)) return True + def delete_message(self, *, session_id: str, message_id: int) -> bool: + sid = str(session_id or "").strip() + mid = int(message_id or 0) + if not sid or mid <= 0: + return False + with self._connect() as conn: + cur = conn.execute( + "DELETE FROM chat_message WHERE session_id = ? AND id = ?", + (sid, mid), + ) + if int(cur.rowcount or 0) <= 0: + return False + last_row = conn.execute( + "SELECT MAX(timestamp) AS ts FROM chat_message WHERE session_id = ?", + (sid,), + ).fetchone() + last_ts = str((last_row["ts"] if last_row else "") or "").strip() or None + conn.execute( + "UPDATE chat_session SET last_message_at = ? WHERE id = ?", + (last_ts, sid), + ) + return True + def add_message( self, session_id: str, diff --git a/runtime/chat/agent_messages.py b/runtime/chat/agent_messages.py index e1469714..ee225cdf 100644 --- a/runtime/chat/agent_messages.py +++ b/runtime/chat/agent_messages.py @@ -14,6 +14,7 @@ import re from typing import Any from oclaw.platform.llm.chat_models import _normalize_image_b64_payload, gemini_openai_compat_client, ChatModel +from oclaw.runtime.chat.media_redact import redact_embedded_image_blobs from oclaw.runtime.chat.tool_runtime import tool_llm_message_max_chars, truncate_tool_result_for_llm_messages from oclaw.prompts import render_prompt from oclaw.platform.files.attachment_assets import attachment_id_to_data_url @@ -196,8 +197,18 @@ def build_llm_messages( model: ChatModel, lang: str, tool_context_truncate_enabled: bool = True, + active_turn_uuid: str | None = None, ) -> list[dict[str, Any]]: - """把 DB 中的消息序列转换为 LLM messages。""" + """把 DB 中的消息序列转换为 LLM messages。 + + When ``active_turn_uuid`` matches a tool/user row ``turn_uuid``, that turn is treated as the + in-flight MCP turn: tool JSON is not stripped of nested image payloads here (see also + :func:`~oclaw.runtime.direct_loop._guard_tool_results_for_llm_context`). Omit or leave empty + to apply image-blob stripping for every tool row (safe default for callers without turn context). + + Only the **last** user message may expand attachments into native multimodal ``input_image``; + older user attachments are replayed as text metadata only. + """ out: list[dict[str, Any]] = [{"role": "system", "content": (system_prompt or "").strip()}] thinking_mode_enabled = bool(getattr(model, "thinking_mode_enabled", False)) allow_signature_replay = _allow_reasoning_signature_replay(model) @@ -239,6 +250,15 @@ def build_llm_messages( seen_tool_ids.add(tcid) tool_ids_after.append(set(seen_tool_ids)) tool_ids_after.reverse() + + last_user_msg_idx = -1 + for _ui, _um in enumerate(store_messages or []): + if str(getattr(_um, "role", "") or "") != "user": + continue + if str(getattr(_um, "event_type", "") or "").strip().lower() == "reasoning": + continue + last_user_msg_idx = _ui + def _attach_reasoning_content(row: dict[str, Any], m: Any) -> dict[str, Any]: if not thinking_mode_enabled: return row @@ -291,34 +311,53 @@ def build_llm_messages( if not isinstance(att, dict): continue att_type = att.get("type") + expand_user_image_for_model = bool(i == last_user_msg_idx) if att_type in ("image", "input_image"): - b64 = _normalize_image_b64_payload(att.get("image_base64") or att.get("data")) - if not b64: - continue - content_list.append( - { - "type": "input_image", - "image_base64": b64, - "mime": att.get("mime") or "image/jpeg", - } - ) + if expand_user_image_for_model: + b64 = _normalize_image_b64_payload(att.get("image_base64") or att.get("data")) + if not b64: + continue + content_list.append( + { + "type": "input_image", + "image_base64": b64, + "mime": att.get("mime") or "image/jpeg", + } + ) + else: + name = str(att.get("name") or "image") + mime = str(att.get("mime") or "image/jpeg") + hs = "(historical attachment; pixels not replayed into model)" + hs_zh = "（历史附件；不向模型回放像素）" + hint = hs_zh if not str(lang or "").startswith("en") else hs + meta_line = f"- name={name} mime={mime} {hint}" + content_list.append( + { + "type": "text", + "text": render_prompt( + "tools/image_attachment_meta.md", + variables={"meta_line": meta_line}, + strict=True, + ), + } + ) elif att_type == "image_ref": - # Prefer actual image bytes so multi-agent/image specialist can truly "see" history images. name = str(att.get("name") or "image") mime = str(att.get("mime") or "image/jpeg") aid = str(att.get("attachment_id") or "") - data_url = attachment_id_to_data_url(aid, mime=mime) if aid else "" - if data_url: - if ";base64," in data_url: - b64 = data_url.split(";base64,", 1)[1] - content_list.append( - { - "type": "input_image", - "image_base64": b64, - "mime": mime, - } - ) - continue + if expand_user_image_for_model: + data_url = attachment_id_to_data_url(aid, mime=mime) if aid else "" + if data_url: + if ";base64," in data_url: + b64 = data_url.split(";base64,", 1)[1] + content_list.append( + { + "type": "input_image", + "image_base64": b64, + "mime": mime, + } + ) + continue w = att.get("width") h = att.get("height") sz = att.get("bytes") @@ -423,7 +462,7 @@ def build_llm_messages( aid = str(_fid or "").strip() except Exception: aid = "" - if aid and mime.startswith("image/"): + if expand_user_image_for_model and aid and mime.startswith("image/"): data_url = attachment_id_to_data_url(aid, mime=mime) if data_url and ";base64," in data_url: b64 = data_url.split(";base64,", 1)[1] @@ -575,6 +614,15 @@ def build_llm_messages( ) continue raw_tc_content = getattr(m, "content", "") or "" + _tun = str(getattr(m, "turn_uuid", "") or "").strip() + _aus = str(active_turn_uuid or "").strip() + if (not _aus) or (_tun != _aus): + try: + _p = json.loads(raw_tc_content) + _p2 = redact_embedded_image_blobs(_p) + raw_tc_content = json.dumps(_p2, ensure_ascii=False, default=str) + except Exception: + pass tool_content_out = raw_tc_content cap = tool_llm_message_max_chars() if str(tool_call_id) in historical_tool_ids: diff --git a/runtime/chat/media_redact.py b/runtime/chat/media_redact.py new file mode 100644 index 00000000..3fe4113b --- /dev/null +++ b/runtime/chat/media_redact.py @@ -0,0 +1,198 @@ +"""Strip/ingest embedded binary payloads from tool/MCP-shaped JSON. + +Persistence is untouched; callers use copies when building model context.""" + +from __future__ import annotations + +import base64 +from typing import Any +from oclaw.platform.files.attachment_assets import AttachmentAssetStore + +_IMAGE_CONTENT_TYPES = frozenset({"image", "input_image"}) +_BASE64_PAYLOAD_KEYS = ("data", "image_base64", "base64", "content_base64", "body_base64") +# Below this length we keep values (tiny icons / markers). +_MIN_B64_CHARS = 200 + + +def redact_embedded_image_blobs(obj: Any) -> Any: + """Deep-copy-ish transform: replace large base64 payloads with metadata placeholders.""" + if isinstance(obj, dict): + return _redact_dict(obj) + if isinstance(obj, list): + return [redact_embedded_image_blobs(x) for x in obj] + return obj + + +def _looks_like_large_payload(s: str) -> bool: + t = str(s or "").strip() + if len(t) < _MIN_B64_CHARS: + return False + allowed = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r-_") + if not t[: min(512, len(t))]: + return False + noise = sum(1 for ch in t[: min(2000, len(t))] if ch not in allowed) + return noise <= max(2, len(t[: min(2000, len(t))]) // 200) + + +def _redact_dict(d: dict[str, Any]) -> dict[str, Any]: + typ = str(d.get("type") or "").strip().lower() + payload_keys = [k for k in _BASE64_PAYLOAD_KEYS if isinstance(d.get(k), str) and _looks_like_large_payload(str(d.get(k) or ""))] + if payload_keys: + plen = max(len(str(d.get(k) or "")) for k in payload_keys) + dup: dict[str, Any] = {} + for k, v in d.items(): + if k in payload_keys: + continue + if isinstance(v, dict): + dup[k] = _redact_dict(v) + elif isinstance(v, list): + dup[k] = [redact_embedded_image_blobs(x) for x in v] + else: + dup[k] = v + is_image = typ in _IMAGE_CONTENT_TYPES + dup["_image_payload_redacted" if is_image else "_binary_payload_redacted"] = True + dup["_redacted_payload_chars"] = int(plen) + dup["_redacted_payload_keys"] = list(payload_keys) + return dup + + out: dict[str, Any] = {} + for k, v in d.items(): + if isinstance(v, dict): + out[k] = _redact_dict(v) + elif isinstance(v, list): + out[k] = [redact_embedded_image_blobs(x) for x in v] + else: + out[k] = v + return out + + +def ingest_embedded_image_blobs_as_refs( + obj: Any, + *, + root_dir: str | None = None, + filename_prefix: str = "tool-image", +) -> tuple[Any, list[dict[str, Any]]]: + """Persist nested base64 blobs and replace them with attachment refs. + + Returns transformed object and newly created attachment refs. + """ + store = AttachmentAssetStore(root_dir=root_dir) if root_dir else AttachmentAssetStore() + refs: list[dict[str, Any]] = [] + + def _ingest(node: Any, idx_seed: list[int]) -> Any: + if isinstance(node, list): + return [_ingest(x, idx_seed) for x in node] + if not isinstance(node, dict): + return node + typ = str(node.get("type") or "").strip().lower() + raw = _pick_base64_payload(node) + if raw: + blob = _decode_image_bytes(raw) + if blob: + idx_seed[0] += 1 + mime = str(node.get("mime") or node.get("mime_type") or "image/png").strip() or "image/png" + ext = _filename_ext_for_mime(mime) + name = str(node.get("name") or f"{filename_prefix}-{idx_seed[0]}{ext}").strip() + meta = store.save_bytes( + blob, + filename=name, + mime=mime, + width=_safe_int(node.get("width")), + height=_safe_int(node.get("height")), + ) + ref_type = _ref_type_for_mime(mime, typ) + ref = { + "type": ref_type, + "attachment_id": meta.attachment_id, + "name": meta.name, + "mime": meta.mime, + "bytes": meta.bytes, + "width": meta.width, + "height": meta.height, + } + refs.append(ref) + return ref + redacted = _redact_dict(node) + redacted["type"] = _ref_type_for_mime( + str(node.get("mime") or node.get("mime_type") or "application/octet-stream"), + typ, + ) + redacted.setdefault("name", str(node.get("name") or "attachment")) + redacted.setdefault("mime", str(node.get("mime") or node.get("mime_type") or "application/octet-stream")) + return redacted + out: dict[str, Any] = {} + for k, v in node.items(): + out[k] = _ingest(v, idx_seed) + return out + + transformed = _ingest(obj, [0]) + uniq: list[dict[str, Any]] = [] + seen: set[str] = set() + for r in refs: + aid = str(r.get("attachment_id") or "").strip() + if not aid or aid in seen: + continue + seen.add(aid) + uniq.append(r) + return transformed, uniq + + +def _decode_image_bytes(raw: Any) -> bytes: + s = str(raw or "").strip() + if not s: + return b"" + if s.startswith("data:") and ";base64," in s: + s = s.split(";base64,", 1)[1] + try: + return base64.b64decode(s.encode("ascii"), validate=False) + except Exception: + return b"" + + +def _pick_base64_payload(node: dict[str, Any]) -> str: + for k in _BASE64_PAYLOAD_KEYS: + v = node.get(k) + if isinstance(v, str) and str(v).strip(): + return v + return "" + + +def _ref_type_for_mime(mime: str, typ: str = "") -> str: + m = str(mime or "").strip().lower() + t = str(typ or "").strip().lower() + if t in _IMAGE_CONTENT_TYPES or m.startswith("image/"): + return "image_ref" + if m.startswith("video/"): + return "video_ref" + if m.startswith("text/"): + return "text_ref" + return "binary_ref" + + +def _filename_ext_for_mime(mime: str) -> str: + m = str(mime or "").strip().lower() + if m == "image/png": + return ".png" + if m in {"image/jpeg", "image/jpg"}: + return ".jpg" + if m == "image/webp": + return ".webp" + if m == "image/gif": + return ".gif" + if m == "video/mp4": + return ".mp4" + if m == "text/plain": + return ".txt" + return ".bin" + + +def _safe_int(raw: Any) -> int | None: + try: + if raw is None: + return None + return int(raw) + except Exception: + return None + + +__all__ = ["redact_embedded_image_blobs", "ingest_embedded_image_blobs_as_refs"] diff --git a/runtime/chat/model_path_audit.py b/runtime/chat/model_path_audit.py new file mode 100644 index 00000000..5ac400a5 --- /dev/null +++ b/runtime/chat/model_path_audit.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from typing import Any + +_MIN_B64_CHARS = 200 + + +def ensure_no_tool_or_embedded_image_payload(*, messages: list[dict[str, Any]], path: str) -> None: + """Guard non-turn model paths and degrade in place instead of raising. + + - `role=tool` is downgraded to assistant text summary. + - Embedded image/base64 payloads are replaced with safe text placeholders. + """ + for m in messages or []: + if not isinstance(m, dict): + continue + role = str(m.get("role") or "").strip().lower() + if role == "tool": + m["role"] = "assistant" + m["content"] = f"[model_path_audit:{path}] tool payload omitted" + continue + content = m.get("content") + if _contains_embedded_image_payload(content): + m["content"] = _sanitize_content(content, path=path) + + +def _contains_embedded_image_payload(obj: Any) -> bool: + if isinstance(obj, str): + return _contains_large_base64_like_text(obj) + if isinstance(obj, list): + return any(_contains_embedded_image_payload(x) for x in obj) + if not isinstance(obj, dict): + return False + typ = str(obj.get("type") or "").strip().lower() + if typ in {"image", "input_image"}: + for k in ("data", "image_base64"): + v = obj.get(k) + if isinstance(v, str) and len(v.strip()) >= _MIN_B64_CHARS: + return True + for v in obj.values(): + if _contains_embedded_image_payload(v): + return True + return False + + +def _contains_large_base64_like_text(text: str) -> bool: + s = str(text or "").strip() + if len(s) < _MIN_B64_CHARS: + return False + if s.startswith("data:") and ";base64," in s: + s = s.split(";base64,", 1)[1] + head = s[: min(4096, len(s))] + if len(head) < _MIN_B64_CHARS: + return False + allowed = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r-_") + noise = sum(1 for ch in head if ch not in allowed) + # Similar heuristic to media redaction: mostly base64 alphabet over a long span. + return noise <= max(4, len(head) // 200) + + +def _sanitize_content(content: Any, *, path: str) -> Any: + if isinstance(content, str): + if _contains_large_base64_like_text(content): + return f"[model_path_audit:{path}] base64 payload omitted" + return content + if isinstance(content, list): + out: list[Any] = [] + for item in content: + if isinstance(item, dict): + typ = str(item.get("type") or "").strip().lower() + if typ in {"image", "input_image"}: + out.append({"type": "text", "text": f"[model_path_audit:{path}] image payload omitted"}) + continue + out.append(_sanitize_content(item, path=path)) + return out + if isinstance(content, dict): + out: dict[str, Any] = {} + for k, v in content.items(): + if str(k) in {"data", "image_base64"} and isinstance(v, str) and _contains_large_base64_like_text(v): + out[k] = f"[model_path_audit:{path}] payload omitted" + continue + out[k] = _sanitize_content(v, path=path) + return out + return content + + +__all__ = ["ensure_no_tool_or_embedded_image_payload"] diff --git a/runtime/chat/tool_runtime.py b/runtime/chat/tool_runtime.py index 1b34b92c..f2a36f38 100644 --- a/runtime/chat/tool_runtime.py +++ b/runtime/chat/tool_runtime.py @@ -21,6 +21,7 @@ from oclaw.platform.persistence.sqlite_store import SqliteStore from oclaw.runtime.tools.base import ToolRegistry from oclaw.platform.llm.chat_models import LLMToolCall from oclaw.runtime.tools.tool_validation import validate_tool_arguments +from oclaw.runtime.chat.media_redact import ingest_embedded_image_blobs_as_refs from oclaw.runtime.tools.experts.workspace.workspace_base import ( workspace_path_access_scope, workspace_write_namespace_scope, @@ -58,13 +59,15 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]: return [] out: list[dict[str, Any]] = [] aid = str(result.get("attachment_id") or "").strip() + root_mime = str(result.get("mime") or "").strip() if aid: + ref_type = _ref_type_for_mime(root_mime) out.append( { - "type": "image_ref", + "type": ref_type, "attachment_id": aid, "name": str(result.get("name") or "generated-image"), - "mime": str(result.get("mime") or "image/png"), + "mime": root_mime or "application/octet-stream", "bytes": result.get("bytes"), "width": result.get("width"), "height": result.get("height"), @@ -91,12 +94,16 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]: continue r_aid = str(r.get("attachment_id") or "").strip() if r_aid: + r_typ = str(r.get("type") or "").strip().lower() + r_mime = str(r.get("mime_type") or r.get("mime") or "").strip() + if r_typ not in {"image_ref", "video_ref", "text_ref", "binary_ref"}: + r_typ = _ref_type_for_mime(r_mime) out.append( { - "type": "image_ref", + "type": r_typ, "attachment_id": r_aid, "name": str(r.get("name") or "generated-image"), - "mime": str(r.get("mime") or "image/png"), + "mime": r_mime or "application/octet-stream", "bytes": r.get("bytes"), "width": r.get("width"), "height": r.get("height"), @@ -110,15 +117,18 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]: if not isinstance(item, dict): continue typ = str(item.get("type") or "").strip().lower() - if typ in {"image", "input_image"}: - b64 = item.get("image_base64") or item.get("data") - if isinstance(b64, str) and b64.strip(): + if typ in {"image_ref", "video_ref", "text_ref", "binary_ref"}: + a_id = str(item.get("attachment_id") or "").strip() + if a_id: out.append( { - "type": "image", - "data": b64.strip(), - "mime": str(item.get("mime_type") or item.get("mime") or "image/png"), - "name": str(item.get("name") or "tool-image"), + "type": typ, + "attachment_id": a_id, + "mime": str(item.get("mime_type") or item.get("mime") or "application/octet-stream"), + "name": str(item.get("name") or "tool-attachment"), + "bytes": item.get("bytes"), + "width": item.get("width"), + "height": item.get("height"), } ) elif typ == "image_url": @@ -132,7 +142,7 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]: a.get("attachment_id") or a.get("pointer_uri") or a.get("url") - or (f"b64:{a.get('mime')}:{len(str(a.get('data') or ''))}" if a.get("data") else "") + or "" ).strip() if not k or k in seen: continue @@ -828,6 +838,10 @@ class ToolExecutor: ) result, duration_ms = results_by_id[tc.id] result = normalize_tool_result(result) + persisted_result, ingested_refs = ingest_embedded_image_blobs_as_refs( + result, + filename_prefix=f"{str(tc.name or 'tool')}-{str(tc.id or '')}", + ) logger.info( "tool_runtime tool session=%s name=%s duration_ms=%d ok=%s", ctx.session_id[:12], @@ -840,7 +854,7 @@ class ToolExecutor: session_id=ctx.session_id, tool_name=tc.name, args=tc.arguments, - result=result, + result=persisted_result, specialist=ctx.specialist, duration_ms=duration_ms, ) @@ -849,7 +863,7 @@ class ToolExecutor: # until the turn finishes, so current-round model context remains lossless. t_trunc = time.perf_counter() observed_rows_this_call = int(_estimate_observed_rows(result)) - result_for_llm = dict(result or {}) + result_for_llm = dict(persisted_result or {}) if tc.name in _SQL_REPLAY_COMPACT_TOOL_NAMES: current = int(local_turn_tool_name_counts.get(tc.name, 0)) current_rows = int(local_turn_tool_observed_rows.get(tc.name, 0)) @@ -870,7 +884,7 @@ class ToolExecutor: role="tool", content=tool_content, tool_calls={"tool_call_id": tc.id, "name": tc.name, "assistant_message_id": assistant_msg_id}, - attachments=_attachments_from_tool_result(result) or None, + attachments=(_merge_attachments(_attachments_from_tool_result(persisted_result), ingested_refs) or None), turn_uuid=ctx.turn_uuid, event_type="tool_result", event_payload={"tool_name": tc.name, "observed_rows": int(observed_rows_this_call)}, @@ -971,3 +985,29 @@ __all__ = [ "truncate_tool_result_for_llm_messages", "compact_turn_tool_messages_for_storage", ] + + +def _merge_attachments(*parts: list[dict[str, Any]]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for part in parts: + for a in part or []: + if not isinstance(a, dict): + continue + k = str(a.get("attachment_id") or a.get("pointer_uri") or a.get("url") or "").strip() + if not k or k in seen: + continue + seen.add(k) + out.append(a) + return out + + +def _ref_type_for_mime(mime: str) -> str: + m = str(mime or "").strip().lower() + if m.startswith("image/"): + return "image_ref" + if m.startswith("video/"): + return "video_ref" + if m.startswith("text/"): + return "text_ref" + return "binary_ref" diff --git a/runtime/direct_loop.py b/runtime/direct_loop.py index 4a96d67d..ed9a3be7 100644 --- a/runtime/direct_loop.py +++ b/runtime/direct_loop.py @@ -13,6 +13,7 @@ from types import SimpleNamespace from typing import Any, Callable, Optional from oclaw.runtime.chat.agent_messages import build_llm_messages +from oclaw.runtime.chat.media_redact import redact_embedded_image_blobs from oclaw.runtime.chat.tool_runtime import ToolExecutionConfig from oclaw.runtime.chat.turn_types import TurnRunOutcome from oclaw.runtime.skill_executor import SkillExecutionContext, SkillExecutor @@ -269,6 +270,21 @@ def _json_dumps_safe(obj: Any) -> str: return json.dumps({"ok": False, "error": "not_json_serializable"}, ensure_ascii=False) +def _tool_message_with_content(m: Any, content: str, *, sid: str = "") -> SimpleNamespace: + return SimpleNamespace( + id=getattr(m, "id", 0), + session_id=str(getattr(m, "session_id", None) or sid or ""), + role="tool", + content=content, + tool_calls=getattr(m, "tool_calls", None), + timestamp=getattr(m, "timestamp", ""), + attachments=getattr(m, "attachments", None), + turn_uuid=getattr(m, "turn_uuid", None), + event_type=getattr(m, "event_type", None), + event_payload=getattr(m, "event_payload", None), + ) + + def _split_reasoning_and_body(text: str, *, explicit_reasoning: str | None = None) -> tuple[list[str], str]: explicit = str(explicit_reasoning or "").strip() raw = str(text or "") @@ -317,6 +333,12 @@ def _guard_tool_results_for_llm_context( out.append(m) continue raw = str(getattr(m, "content", "") or "") + try: + _parsed0 = json.loads(raw) + _parsed1 = redact_embedded_image_blobs(_parsed0) + raw = _json_dumps_safe(_parsed1) + except Exception: + pass # Best-effort parse tool JSON for image-query specific guard and overflow metadata. ok = None error_code = "" @@ -350,17 +372,7 @@ def _guard_tool_results_for_llm_context( "图片分析结果在上下文回放中已截断，请缩小 query_image_attachment 的问题范围。" ) guarded = _json_dumps_safe(guarded_obj) - out.append( - SimpleNamespace( - id=getattr(m, "id", 0), - session_id=getattr(m, "session_id", session_id), - role="tool", - content=guarded, - tool_calls=getattr(m, "tool_calls", None), - timestamp=getattr(m, "timestamp", ""), - attachments=getattr(m, "attachments", None), - ) - ) + out.append(_tool_message_with_content(m, guarded, sid=session_id)) continue # Guard video transcript replay similarly (usually long). if str(obj.get("task") or "").strip().lower() == "transcript" and has_attachment_id and len(text) > video_cap: @@ -371,20 +383,10 @@ def _guard_tool_results_for_llm_context( guarded_obj["video_result_original_chars"] = len(text) guarded_obj["video_result_replay_cap_chars"] = video_cap guarded = _json_dumps_safe(guarded_obj) - out.append( - SimpleNamespace( - id=getattr(m, "id", 0), - session_id=getattr(m, "session_id", session_id), - role="tool", - content=guarded, - tool_calls=getattr(m, "tool_calls", None), - timestamp=getattr(m, "timestamp", ""), - attachments=getattr(m, "attachments", None), - ) - ) + out.append(_tool_message_with_content(m, guarded, sid=session_id)) continue if len(raw) <= cap: - out.append(m) + out.append(_tool_message_with_content(m, raw, sid=session_id)) continue preview = raw[: max(1, min(4000, cap - 400))] + "\n..." guarded_obj = { @@ -402,17 +404,7 @@ def _guard_tool_results_for_llm_context( ), } guarded = _json_dumps_safe(guarded_obj) - out.append( - SimpleNamespace( - id=getattr(m, "id", 0), - session_id=getattr(m, "session_id", session_id), - role="tool", - content=guarded, - tool_calls=getattr(m, "tool_calls", None), - timestamp=getattr(m, "timestamp", ""), - attachments=getattr(m, "attachments", None), - ) - ) + out.append(_tool_message_with_content(m, guarded, sid=session_id)) if trace_id: _emit_direct_loop_trace( store=store, @@ -631,6 +623,7 @@ def _build_model_context( model=model, lang=lang, tool_context_truncate_enabled=tool_context_truncate_enabled, + active_turn_uuid=active_turn_uuid, ) diff --git a/runtime/gateway.py b/runtime/gateway.py index 20f55f75..0880a14c 100644 --- a/runtime/gateway.py +++ b/runtime/gateway.py @@ -35,6 +35,7 @@ from oclaw.runtime.router import decide_route from oclaw.runtime.worker import ensure_worker_started from oclaw.runtime.orchestration.trace import new_span_id, new_trace_id from oclaw.runtime.chat.tool_runtime import compact_turn_tool_messages_for_storage +from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload _OC_STAGE_BY_EVENT: dict[str, str] = { "gateway_received": "ingress", @@ -203,11 +204,9 @@ class OclawGateway: else "仅基于以下用户正文生成简短会话标题。请使用对话内容主体语言命名。" "只返回标题文本，不要引号，不要markdown，最多18个字。" ) - resp = model.chat( - [{"role": "system", "content": sys}, {"role": "user", "content": body}], - [], - on_token=None, - ) + messages = [{"role": "system", "content": sys}, {"role": "user", "content": body}] + ensure_no_tool_or_embedded_image_payload(messages=messages, path="gateway.auto_title") + resp = model.chat(messages, [], on_token=None) title = str(getattr(resp, "content", "") or "").strip().replace("\n", " ") title = title.strip("\"'` ").strip() if not title: @@ -317,6 +316,7 @@ class OclawGateway: ), }, ] + ensure_no_tool_or_embedded_image_payload(messages=messages, path="gateway.manager_select") resp = model.chat(messages, [], on_token=None) obj = self._parse_json_object(str(getattr(resp, "content", "") or "")) if not isinstance(obj, dict): @@ -404,11 +404,9 @@ class OclawGateway: f"专家结果:\n{str(specialist_reply or '').strip()}\n\n" "要求：保持简洁、准确，不要暴露内部流程。" ) - resp = model.chat( - [{"role": "system", "content": manager_context}, {"role": "user", "content": user_text}], - [], - on_token=on_token, - ) + messages = [{"role": "system", "content": manager_context}, {"role": "user", "content": user_text}] + ensure_no_tool_or_embedded_image_payload(messages=messages, path="gateway.manager_finalize") + resp = model.chat(messages, [], on_token=on_token) final_text = str(getattr(resp, "content", "") or "").strip() return final_text or str(specialist_reply or "") except Exception: diff --git a/runtime/router.py b/runtime/router.py index 96356785..228f5295 100644 --- a/runtime/router.py +++ b/runtime/router.py @@ -8,6 +8,7 @@ from typing import Any from oclaw.runtime.types import StandardMessage from oclaw.runtime.types import normalize_interaction_mode, normalize_requested_specialist from oclaw.prompts.loader import render_runtime_prompt +from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload @dataclass(frozen=True) @@ -86,6 +87,7 @@ def _decide_llm_json(msg: StandardMessage, *, model: Any | None) -> RouterDecisi }, {"role": "user", "content": user_block}, ] + ensure_no_tool_or_embedded_image_payload(messages=messages, path="router.llm_json") resp = model.chat(messages, [], on_token=None) raw = str(getattr(resp, "content", "") or "") obj = _parse_router_json_object(raw) diff --git a/runtime/skills/aiops-telecom/SKILL.md b/runtime/skills/aiops-telecom/SKILL.md new file mode 100644 index 00000000..e661860b --- /dev/null +++ b/runtime/skills/aiops-telecom/SKILL.md @@ -0,0 +1,144 @@ +# AIOps-电信数通智能运维技能包 + +## 概述 +面向电信/数通网络的 AI 智能运维技能包。支持告警日志智能分析、配置错漏检查、故障模式学习与根因推荐。适用于华为、中兴、Cisco、Juniper 等主流数通设备。 + +## 能力说明 + +### 1. 告警日志智能分析 + +#### 1.1 输入格式支持 +- Excel/CSV 格式的告警清单（如网管导出的告警报表） +- 纯文本格式的 syslog/告警日志 +- 设备 CLI 输出的告警信息 + +#### 1.2 分析维度 + +| 维度 | 说明 | +|------|------| +| **告警分级统计** | Critical / Major / Warning 级别分布及占比 | +| **设备维度聚合** | 按设备（ME/NE）统计告警数量 Top N | +| **告警类型归类** | 物理层/链路层/网络层/路由层/安全层分类 | +| **时间维度分析** | 告警爆发时间窗口、频次趋势 | +| **关联分析** | 同设备/同链路的多层告警关联，推测根因 | +| **影响评估** | 评估受影响业务（L3VPN/Tunnel/Ethernet）范围 | + +#### 1.3 告警类型知识库 + +##### 物理层告警 +| 告警名称 | 级别 | 含义 | 常见根因 | 推荐操作 | +|----------|------|------|---------|---------| +| Ethernet Physical LOS | Critical | 光口信号丢失 | 光纤断/松、光模块故障、对端设备断电 | 检查光纤链路、光功率、更换光模块 | +| Ethernet Physical Laser Temperature | Major | 激光器温度越限 | 光模块老化、设备散热不良、环境温度过高 | 检查设备风扇/温度、更换光模块 | +| Ethernet Physical Input Optical Power | Major | 接收光功率越限 | 光纤衰减过大、接口脏污、对端发光异常 | 清洁光纤接头、检查光功率预算 | +| Ethernet Physical CRC Error | Major | CRC 误码越限 | 光纤质量差、接头污染、电磁干扰 | 清洁光纤、检查物理链路、更换尾纤 | +| SmartGroup RX CRC Error | Major | 聚合口 CRC 误码 | 聚合成员链路质量问题 | 排查各成员链路、更换问题链路 | + +##### 路由/MPLS层告警 +| 告警名称 | 级别 | 含义 | 常见根因 | 推荐操作 | +|----------|------|------|---------|---------| +| OSPF Neighbor Down | Major | OSPF 邻居中断 | 物理链路故障、Hello 超时、配置不匹配 | 检查邻居间链路、验证 OSPF 配置参数 | +| RSVP LSP BFD Session Down | Major | MPLS TE隧道 BFD 检测失败 | 隧道途经链路故障、节点故障 | 检查隧道路径、确认中间节点状态 | +| BGP Peer Flap | Major | BGP 邻居震荡 | 链路不稳、Keepalive 超时、策略变更 | 检查 BGP 配置、链路稳定性 | + +##### 安全/控制面告警 +| 告警名称 | 级别 | 含义 | 常见根因 | 推荐操作 | +|----------|------|------|---------|---------| +| CPSBP 超阈值 | Warning | 控制面报文速率超阈值 | 攻击流量、广播风暴、配置过低 | 分析流量源、调整 CP 保护阈值 | +| ARP IP冲突 | Warning | 检测到 IP 地址冲突 | 私接设备、IP 配置错误 | 定位冲突源、释放/更改 IP 地址 | + +### 2. 配置错漏检查 + +#### 2.1 支持检查项 + +| 检查类别 | 典型检查项 | +|----------|-----------| +| **接口配置** | VLAN 配置一致性、MTU 匹配、描述规范、端口模式(access/trunk) | +| **路由协议** | OSPF area 一致性、BGP AS 号核对、Route-map 逻辑、邻居配置完整性 | +| **MPLS/TE** | LSP 配置完整性、隧道保护策略、FRR 配置 | +| **安全策略** | ACL 规则匹配、控制面保护(CoPP)、端口安全、MAC 漂移检测 | +| **高可用** | VRRP/HSRP 配置一致性、BFD 联动、链路聚合(LACP)配置 | +| **QoS** | 队列策略、带宽限制、优先级映射 | + +#### 2.2 分析模式 +- **配置比对**：新旧配置 diff，快速定位变更点 +- **合规检查**：基于标准模板检查配置是否符合规范 +- **逻辑验证**：检查配置逻辑矛盾（如 ACL 冗余/冲突、路由黑洞） + +### 3. 故障模式学习 (self-learning) + +系统会记录每次分析过程中的： +- 告警 → 修复措施 对应关系 +- 配置错误 → 正确配置 修正方案 +- 经用户确认的根因分析结论 + +这些 learnings 会存入 `.learnings/` 目录，在后续分析中自动参考，越用越准确。 + +## 使用示例 + +### 示例1：告警日志分析 +``` +用户: "分析这份告警日志" +助手: 按以下格式输出分析报告： + +📊 告警概览 +- 总告警数：24条 +- Critical：2条 (8.3%) ⛔ +- Major：9条 (37.5%) ⚠️ +- Warning：13条 (54.2%) ⚡ + +🔴 Critical 告警详情 +1. [SMD-LIIR-EN1-Z20HS] ETPI LOS → 物理光口信号丢失 + → 该设备下联 SMD-SBRS-EN1-Z20HS，状态 Unack + → 建议：立即检查光纤/光模块 + +🟠 Major 告警详情 +1. [BKL-UNB-AN1-ZM3SP] 光功率越限 → 接收光功率过低 +2. [MDN-PMKN-EN1-Z20HS] 激光器温度越限 +3. [PAD-KBU-AN1-ZM8S] CRC误码越限（物理口+聚合口） +4. RSVP LSP BFD Session Down ×3条 +5. OSPF邻居中断 ×2条 + +🔄 关联分析发现 +- PBR-RPKU 与 PBR-PYSK 之间：OSPF邻居中断 + CPSBP告警 + → 推测该链路存在物理层问题 +- GRO 站点多条 RSVP LSP BFD Down + → 可能为 GRO 节点设备问题或出局光缆中断 + +💡 根因推荐 +1. 优先处理 SMD-LIIR-EN1-Z20HS 的 LOS（Critical） +2. 检查 PBR-RPKU ↔ PBR-PYSK 链路光功率和光模块 +3. 排查 GRO 站点汇聚设备状态 +``` + +### 示例2：配置检查 +``` +用户: "帮我检查这两份配置文件" +助手: +🔍 配置检查报告 +1. OSPF 配置检查 ✅ + - Area 一致性：匹配 + - Hello/Dead 间隔：一致 + - 网络类型：一致 + +2. 接口配置检查 ⚠️ + - [GE0/0/1] MTU 不匹配：本端 1500，对端 9000 + - [GE0/0/2] 描述缺失 + +3. BGP 配置检查 ❌ + - AS 号不匹配：本端 AS65001，对端 AS65002 +``` + +## 数据来源说明 + +本技能的知识库基于以下标准构建： +- 华为 NE40E/ME60 系列告警手册 +- 中兴 ZXR10 系列告警与配置规范 +- 3GPP 管理面标准（IRP/Solution） +- ITU-T 光传输标准 +- RFC 相关协议标准 + +## 局限性与注意事项 +- 本技能不直接连接设备执行命令，不做配置变更操作 +- 分析结果基于提供的日志/配置数据，用户需确认数据准确性 +- 推荐操作为参考建议，重大操作需人工复核 diff --git a/runtime/worker.py b/runtime/worker.py index 3e060267..32709a72 100644 --- a/runtime/worker.py +++ b/runtime/worker.py @@ -11,6 +11,7 @@ from oclaw.runtime.agent_core_run import AgentCoreRunInput, run_agent_core from oclaw.runtime.memory_stage import build_memory_context from oclaw.runtime.relay_pointer import build_acp_relay_result, validate_relay_share_envelope from oclaw.runtime.types import StandardMessage +from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload _LOCK = threading.Lock() _THREAD: threading.Thread | None = None @@ -120,11 +121,9 @@ def _maybe_generate_title_on_third_round(*, store: Any, msg: StandardMessage, mo else "仅基于以下用户正文生成简短会话标题。请使用对话内容主体语言命名。" "只返回标题文本，不要引号，不要markdown，最多18个字。" ) - resp = model.chat( - [{"role": "system", "content": sys}, {"role": "user", "content": body}], - [], - on_token=None, - ) + messages = [{"role": "system", "content": sys}, {"role": "user", "content": body}] + ensure_no_tool_or_embedded_image_payload(messages=messages, path="worker.auto_title") + resp = model.chat(messages, [], on_token=None) title = str(getattr(resp, "content", "") or "").strip().replace("\n", " ") title = title.strip("\"'` ").strip() if not title: diff --git a/tests/test_media_redact.py b/tests/test_media_redact.py new file mode 100644 index 00000000..e8698ded --- /dev/null +++ b/tests/test_media_redact.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import base64 + +from oclaw.runtime.chat.media_redact import ingest_embedded_image_blobs_as_refs, redact_embedded_image_blobs + + +def test_redact_nested_mcp_image_block() -> None: + big = "/9j/" + "a" * 800 + obj = { + "ok": True, + "result": {"content": [{"type": "image", "mime": "image/jpeg", "data": big}]}, + } + out = redact_embedded_image_blobs(obj) + assert out["result"]["content"][0].get("_image_payload_redacted") is True + assert "data" not in out["result"]["content"][0] + assert isinstance(out["result"]["content"][0].get("_redacted_payload_chars"), int) + + +def test_redact_keeps_small_data_field() -> None: + obj = {"type": "image", "mime": "image/png", "data": "abc"} + assert redact_embedded_image_blobs(obj) == obj + + +def test_ingest_embedded_image_blob_as_ref(tmp_path) -> None: + raw = base64.b64encode(b"png-bytes").decode("ascii") + obj = {"result": {"content": [{"type": "image", "mime": "image/png", "data": raw, "name": "x.png"}]}} + out, refs = ingest_embedded_image_blobs_as_refs(obj, root_dir=str(tmp_path), filename_prefix="unit") + block = out["result"]["content"][0] + assert block["type"] == "image_ref" + assert str(block.get("attachment_id") or "") + assert "data" not in block + assert refs and refs[0]["attachment_id"] == block["attachment_id"] + + +def test_ingest_embedded_binary_blob_as_binary_ref(tmp_path) -> None: + raw = base64.b64encode(b"%PDF-1.4-fake").decode("ascii") + obj = {"result": {"content": [{"type": "file", "mime": "application/pdf", "base64": raw, "name": "a.pdf"}]}} + out, refs = ingest_embedded_image_blobs_as_refs(obj, root_dir=str(tmp_path), filename_prefix="unit") + block = out["result"]["content"][0] + assert block["type"] == "binary_ref" + assert str(block.get("attachment_id") or "") + assert "base64" not in block + assert refs and refs[0]["attachment_id"] == block["attachment_id"] diff --git a/tests/test_model_path_audit.py b/tests/test_model_path_audit.py new file mode 100644 index 00000000..91cca02d --- /dev/null +++ b/tests/test_model_path_audit.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload + + +def test_audit_accepts_plain_system_user_messages() -> None: + ensure_no_tool_or_embedded_image_payload( + path="unit.ok", + messages=[ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "hello"}, + ], + ) + + +def test_audit_degrades_tool_role() -> None: + msgs = [ + {"role": "assistant", "content": "a"}, + {"role": "tool", "content": '{"ok":true}'}, + ] + ensure_no_tool_or_embedded_image_payload(path="unit.tool", messages=msgs) + assert msgs[1]["role"] == "assistant" + assert "omitted" in str(msgs[1]["content"] or "") + + +def test_audit_degrades_embedded_image_payload() -> None: + msgs = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "x"}, + {"type": "input_image", "image_base64": "a" * 600, "mime": "image/png"}, + ], + } + ] + ensure_no_tool_or_embedded_image_payload(path="unit.image", messages=msgs) + c = msgs[0]["content"] + assert isinstance(c, list) + assert any(isinstance(x, dict) and str(x.get("type") or "") == "text" and "omitted" in str(x.get("text") or "") for x in c) + + +def test_audit_degrades_large_base64_like_plain_text() -> None: + payload = "A" * 700 + msgs = [{"role": "user", "content": payload}] + ensure_no_tool_or_embedded_image_payload(path="unit.b64_text", messages=msgs) + assert "omitted" in str(msgs[0]["content"] or "") diff --git a/tests/test_oclaw_tool_result_guard.py b/tests/test_oclaw_tool_result_guard.py index adcee740..88a46a92 100644 --- a/tests/test_oclaw_tool_result_guard.py +++ b/tests/test_oclaw_tool_result_guard.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from pathlib import Path from oclaw.runtime.direct_loop import _OCLAW_TOOL_RESULT_HARD_CAP_CHARS, _build_model_context @@ -78,3 +79,42 @@ def test_oclaw_tool_result_context_guard_skips_active_turn_tool_messages(tmp_pat raw = str(tool_msgs[-1].get("content") or "") assert "_tool_result_guarded" not in raw + +def test_guard_redacts_mcp_nested_image_for_non_active_turn(tmp_path: Path) -> None: + store = SqliteStore(str(tmp_path / "ops.sqlite")) + sess = store.create_session("t") + past_turn = "turn-old" + blob = "/9j/" + "a" * 1200 + body = {"ok": True, "result": {"content": [{"type": "image", "mime": "image/jpeg", "data": blob}]}} + store.add_message( + session_id=sess.id, + role="assistant", + content="", + tool_calls=[{"id": "c_hist", "name": "mcp", "arguments": {}}], + turn_uuid=past_turn, + ) + store.add_message( + session_id=sess.id, + role="tool", + content=json.dumps(body, ensure_ascii=False), + tool_calls={"tool_call_id": "c_hist", "name": "mcp", "assistant_message_id": 1}, + turn_uuid=past_turn, + ) + msgs = _build_model_context( + store=store, + session_id=sess.id, + max_messages=50, + system_prompt="sys", + model=RuleBasedChatModel(), + lang="zh", + memory_context=None, + trace_id="t1", + parent_span_id=None, + active_turn_uuid="different-active-turn", + ) + tm = next(m for m in msgs if m.get("role") == "tool") + inner = json.loads(str(tm.get("content") or "")) + block = inner["result"]["content"][0] + assert block.get("_image_payload_redacted") is True + assert "data" not in block + diff --git a/tests/test_tool_loop_guard.py b/tests/test_tool_loop_guard.py index cc2e66d1..042f50f8 100644 --- a/tests/test_tool_loop_guard.py +++ b/tests/test_tool_loop_guard.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import json from pathlib import Path @@ -89,6 +90,94 @@ def test_repeated_tool_results_are_compacted_in_history(tmp_path: Path) -> None: assert "audit_note" in payloads[2] +def test_tool_result_image_blob_persisted_as_attachment_ref(tmp_path: Path) -> None: + store = SqliteStore(str(tmp_path / "img.sqlite")) + sess = store.create_session("t") + + def _handler(_args): + raw = base64.b64encode(b"\x89PNGtest-image").decode("ascii") + return { + "ok": True, + "result": {"content": [{"type": "image", "mime": "image/png", "data": raw, "name": "mcp.png"}]}, + } + + reg = ToolRegistry( + [ + ToolSpec( + name="mcp_image_tool", + description="returns image payload", + parameters={"type": "object", "properties": {}}, + handler=_handler, + read_only=True, + ) + ] + ) + tool_uses = [LLMToolCall(id="c1", name="mcp_image_tool", arguments={})] + ToolExecutor().execute_tool_uses( + ctx=ToolExecutionContext(store=store, tools=reg, session_id=sess.id, turn_uuid="turn-img"), + assistant_msg_id=1, + tool_uses=tool_uses, + ) + rows = store.get_messages(session_id=sess.id, limit=20) + tool_rows = [m for m in rows if str(getattr(m, "role", "") or "") == "tool"] + assert len(tool_rows) == 1 + payload = json.loads(str(getattr(tool_rows[0], "content", "") or "{}")) + blk = (((payload.get("result") or {}).get("content") or [{}])[0]) if isinstance(payload, dict) else {} + assert isinstance(blk, dict) + assert str(blk.get("type") or "") == "image_ref" + assert str(blk.get("attachment_id") or "") + assert "data" not in blk + atts = json.loads(str(getattr(tool_rows[0], "attachments", "") or "[]")) + assert isinstance(atts, list) and atts + assert str((atts[0] or {}).get("type") or "") == "image_ref" + assert str((atts[0] or {}).get("attachment_id") or "") + logs = store.get_tool_logs(sess.id, limit=10) + assert logs and isinstance(logs[0], dict) + body = logs[0].get("result") or {} + blk2 = (((body.get("result") or {}).get("content") or [{}])[0]) if isinstance(body, dict) else {} + assert isinstance(blk2, dict) + assert str(blk2.get("type") or "") == "image_ref" + assert "data" not in blk2 + + +def test_tool_result_non_image_base64_persisted_as_binary_ref(tmp_path: Path) -> None: + store = SqliteStore(str(tmp_path / "bin.sqlite")) + sess = store.create_session("t") + + def _handler(_args): + raw = base64.b64encode(b"PK\x03\x04fake-zip-bytes").decode("ascii") + return {"ok": True, "result": {"content": [{"type": "file", "mime": "application/zip", "base64": raw, "name": "a.zip"}]}} + + reg = ToolRegistry( + [ + ToolSpec( + name="mcp_file_tool", + description="returns file payload", + parameters={"type": "object", "properties": {}}, + handler=_handler, + read_only=True, + ) + ] + ) + ToolExecutor().execute_tool_uses( + ctx=ToolExecutionContext(store=store, tools=reg, session_id=sess.id, turn_uuid="turn-bin"), + assistant_msg_id=1, + tool_uses=[LLMToolCall(id="c1", name="mcp_file_tool", arguments={})], + ) + rows = store.get_messages(session_id=sess.id, limit=20) + tool_rows = [m for m in rows if str(getattr(m, "role", "") or "") == "tool"] + assert len(tool_rows) == 1 + payload = json.loads(str(getattr(tool_rows[0], "content", "") or "{}")) + blk = (((payload.get("result") or {}).get("content") or [{}])[0]) if isinstance(payload, dict) else {} + assert isinstance(blk, dict) + assert str(blk.get("type") or "") == "binary_ref" + assert str(blk.get("attachment_id") or "") + assert "base64" not in blk + atts = json.loads(str(getattr(tool_rows[0], "attachments", "") or "[]")) + assert isinstance(atts, list) and atts + assert str((atts[0] or {}).get("type") or "") == "binary_ref" + + def test_repeated_non_sql_tools_are_not_compacted(tmp_path: Path) -> None: store = SqliteStore(str(tmp_path / "g3.sqlite")) sess = store.create_session("t") @@ -429,6 +518,11 @@ def test_tool_result_image_payload_persisted_as_attachments(tmp_path: Path) -> N tool_rows = [m for m in rows if str(getattr(m, "role", "") or "") == "tool"] assert len(tool_rows) == 1 attachments = json.loads(str(getattr(tool_rows[0], "attachments", "") or "[]")) - assert any(str(a.get("type") or "") == "image" and str(a.get("data") or "") == "YWJj" for a in attachments) + assert any(str(a.get("type") or "") == "image_ref" and str(a.get("attachment_id") or "") for a in attachments) assert any(str(a.get("type") or "") == "image_url" and str(a.get("url") or "").endswith("/a.png") for a in attachments) + body = json.loads(str(getattr(tool_rows[0], "content", "") or "{}")) + content_items = ((body.get("result") or {}).get("content") or []) if isinstance(body, dict) else [] + image_block = next((x for x in content_items if isinstance(x, dict) and str(x.get("type") or "") == "image_ref"), {}) + assert str(image_block.get("attachment_id") or "") + assert "data" not in image_block diff --git a/tests/test_tool_pairing_messages.py b/tests/test_tool_pairing_messages.py index f83b693c..6c9f0cd1 100644 --- a/tests/test_tool_pairing_messages.py +++ b/tests/test_tool_pairing_messages.py @@ -138,7 +138,13 @@ def test_signature_metadata_not_replayed_by_default_for_non_whitelist_model() -> ensure_ascii=False, ), event_type="tool_call", - ) + ), + _Msg( + "tool", + json.dumps({"ok": True}), + tool_calls=json.dumps({"tool_call_id": "call_1", "name": "t"}, ensure_ascii=False), + event_type="tool_result", + ), ] msgs = build_llm_messages(store_messages=rows, system_prompt="s", model=model, lang="zh") assistant = [m for m in msgs if m.get("role") == "assistant"][0] @@ -158,7 +164,13 @@ def test_signature_metadata_can_be_forced_on_via_env(monkeypatch) -> None: ensure_ascii=False, ), event_type="tool_call", - ) + ), + _Msg( + "tool", + json.dumps({"ok": True}), + tool_calls=json.dumps({"tool_call_id": "call_1", "name": "t"}, ensure_ascii=False), + event_type="tool_result", + ), ] msgs = build_llm_messages(store_messages=rows, system_prompt="s", model=model, lang="zh") assistant = [m for m in msgs if m.get("role") == "assistant"][0] diff --git a/tests/test_tool_runtime_attachments.py b/tests/test_tool_runtime_attachments.py new file mode 100644 index 00000000..140c0a33 --- /dev/null +++ b/tests/test_tool_runtime_attachments.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from oclaw.runtime.chat.tool_runtime import _attachments_from_tool_result + + +def test_attachments_from_tool_result_preserves_non_image_ref_types() -> None: + result = { + "attachments": [ + { + "type": "text_ref", + "attachment_id": "att-text-1", + "mime": "text/plain", + "name": "a.txt", + "bytes": 12, + }, + { + "attachment_id": "att-video-1", + "mime": "video/mp4", + "name": "a.mp4", + "bytes": 1024, + }, + ] + } + + out = _attachments_from_tool_result(result) + by_id = {str(x.get("attachment_id")): x for x in out} + + assert by_id["att-text-1"]["type"] == "text_ref" + assert by_id["att-video-1"]["type"] == "video_ref" +