mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-08 23:33:16 +08:00
Refactor media payload handling to persist base64 blobs as attachment refs and keep non-turn model paths safe by degrading risky payloads.
This preserves multimodal fidelity for the latest user turn while preventing historical/tool replay bloat, and adds admin UI support for referenced attachment preview/download flows. Made-with: Cursor
This commit is contained in:
parent
3d27a01879
commit
6cfaff06f6
20 changed files with 1626 additions and 101 deletions
|
|
@ -877,6 +877,22 @@ def include_chat_routes(router: APIRouter, *, resolve_auth: Callable[[SqliteStor
|
|||
"messages": [_serialize_message(m) for m in msgs],
|
||||
}
|
||||
|
||||
@chat.delete("/sessions/{session_id}/messages/{message_id}")
|
||||
def api_chat_delete_message(
|
||||
session_id: str,
|
||||
message_id: int,
|
||||
authorization: str | None = Header(default=None),
|
||||
) -> dict[str, Any]:
|
||||
store = SqliteStore(db_path())
|
||||
ctx = resolve_auth(store, authorization)
|
||||
sess = _resolve_chat_session(store, ctx, session_id)
|
||||
if not sess:
|
||||
raise HTTPException(status_code=404, detail="session_not_found")
|
||||
ok = store.delete_message(session_id=str(session_id), message_id=int(message_id))
|
||||
if not ok:
|
||||
raise HTTPException(status_code=404, detail="message_not_found")
|
||||
return {"ok": True, "deleted": 1}
|
||||
|
||||
@chat.get("/sessions/{session_id}/wiki-events")
|
||||
def api_chat_wiki_events(
|
||||
session_id: str,
|
||||
|
|
|
|||
|
|
@ -243,6 +243,68 @@
|
|||
font-size: 12px;
|
||||
opacity: 0.9;
|
||||
}
|
||||
.chat-att-ref {
|
||||
min-width: 180px;
|
||||
max-width: 260px;
|
||||
border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.14));
|
||||
border-radius: 8px;
|
||||
background: rgba(255, 255, 255, 0.03);
|
||||
padding: 8px 10px;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
.chat-att-ref__name {
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
line-height: 1.35;
|
||||
margin-bottom: 4px;
|
||||
word-break: break-all;
|
||||
}
|
||||
.chat-att-ref__meta {
|
||||
font-size: 11px;
|
||||
line-height: 1.35;
|
||||
opacity: 0.75;
|
||||
word-break: break-all;
|
||||
}
|
||||
.chat-att-ref__link {
|
||||
display: inline-block;
|
||||
margin-top: 6px;
|
||||
font-size: 12px;
|
||||
color: var(--ds-accent, #5eb3ff);
|
||||
text-decoration: none;
|
||||
}
|
||||
.chat-att-ref__link:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
.chat-att-ref__btn {
|
||||
margin-top: 6px;
|
||||
border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.14));
|
||||
background: rgba(255, 255, 255, 0.04);
|
||||
color: inherit;
|
||||
border-radius: 6px;
|
||||
font-size: 12px;
|
||||
line-height: 1.2;
|
||||
padding: 4px 8px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.chat-att-ref__btn:hover {
|
||||
background: rgba(255, 255, 255, 0.08);
|
||||
}
|
||||
.chat-att-ref__preview {
|
||||
margin: 6px 0 0;
|
||||
padding: 8px;
|
||||
border-radius: 6px;
|
||||
border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.14));
|
||||
background: rgba(0, 0, 0, 0.26);
|
||||
max-height: 180px;
|
||||
overflow: auto;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
font-size: 11px;
|
||||
line-height: 1.35;
|
||||
}
|
||||
.chat-att-ref__preview:empty {
|
||||
display: none;
|
||||
}
|
||||
.chat-composer-shell {
|
||||
border-radius: 12px;
|
||||
border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.12));
|
||||
|
|
@ -484,13 +546,19 @@
|
|||
white-space: pre-wrap;
|
||||
}
|
||||
</style>
|
||||
<!-- Keep markdown helpers non-blocking. chat.js already falls back when they are unavailable. -->
|
||||
<script async src="https://cdn.jsdelivr.net/npm/marked@11.1.1/marked.min.js" crossorigin="anonymous"></script>
|
||||
<script async src="https://cdn.jsdelivr.net/npm/dompurify@3.0.8/dist/purify.min.js" crossorigin="anonymous"></script>
|
||||
<!-- Load markdown/mermaid helpers before chat.js (avoid raw ``` fences render). -->
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/marked@11.1.1/marked.min.js" crossorigin="anonymous"></script>
|
||||
<script defer src="https://cdn.jsdelivr.net/npm/dompurify@3.0.8/dist/purify.min.js" crossorigin="anonymous"></script>
|
||||
<script
|
||||
defer
|
||||
src="https://cdn.jsdelivr.net/npm/mermaid@10.9.1/dist/mermaid.min.js"
|
||||
crossorigin="anonymous"
|
||||
onload="(function(){try{ if (window.__oclawHydrateMermaidAll) window.__oclawHydrateMermaidAll(); }catch(e){}})()"
|
||||
></script>
|
||||
<!-- Cache-bust for desktop webview: avoid stale chat.js -->
|
||||
<script
|
||||
defer
|
||||
src="/admin/assets/chat.js?v=20260425-4"
|
||||
src="/admin/assets/chat.js?v=20260428-2"
|
||||
onerror="(function(){var s=document.getElementById('chat-boot-splash');if(s){s.querySelector('.chat-boot-splash__title span:last-child').textContent='无法加载 chat.js';s.querySelector('.chat-boot-splash__muted').textContent='请确认网关已启动且 /admin/assets/chat.js 可访问。';}})()"
|
||||
></script>
|
||||
</head>
|
||||
|
|
|
|||
|
|
@ -21,6 +21,11 @@ const I18N = {
|
|||
"chat.sessionMenu": "会话操作",
|
||||
"chat.rename": "重命名",
|
||||
"chat.delete": "删除",
|
||||
"chat.copy": "复制",
|
||||
"chat.deleteMessage": "删除消息",
|
||||
"chat.deleteMessageConfirm": "删除这条消息?",
|
||||
"chat.copyOk": "已复制",
|
||||
"chat.copyFail": "复制失败",
|
||||
"chat.deleteConfirm": "删除此会话?",
|
||||
"chat.exportMd": "导出 Markdown",
|
||||
"chat.exportJson": "导出 JSON",
|
||||
|
|
@ -138,6 +143,11 @@ const I18N = {
|
|||
"chat.specialistImageShort": "图像",
|
||||
"chat.specialistMemoryShort": "记忆",
|
||||
"chat.specialistManagerSelfShort": "全能者",
|
||||
"chat.attachment.download": "下载",
|
||||
"chat.attachment.preview": "预览",
|
||||
"chat.attachment.previewLoading": "加载中…",
|
||||
"chat.attachment.previewError": "预览失败",
|
||||
"chat.attachment.previewEmpty": "(空内容)",
|
||||
},
|
||||
en: {
|
||||
"chat.pageTitle": "oliver",
|
||||
|
|
@ -157,6 +167,11 @@ const I18N = {
|
|||
"chat.sessionMenu": "Session actions",
|
||||
"chat.rename": "Rename",
|
||||
"chat.delete": "Delete",
|
||||
"chat.copy": "Copy",
|
||||
"chat.deleteMessage": "Delete message",
|
||||
"chat.deleteMessageConfirm": "Delete this message?",
|
||||
"chat.copyOk": "Copied",
|
||||
"chat.copyFail": "Copy failed",
|
||||
"chat.deleteConfirm": "Delete this session?",
|
||||
"chat.exportMd": "Export Markdown",
|
||||
"chat.exportJson": "Export JSON",
|
||||
|
|
@ -274,6 +289,11 @@ const I18N = {
|
|||
"chat.specialistImageShort": "Image",
|
||||
"chat.specialistMemoryShort": "Memory",
|
||||
"chat.specialistManagerSelfShort": "Manager",
|
||||
"chat.attachment.download": "Download",
|
||||
"chat.attachment.preview": "Preview",
|
||||
"chat.attachment.previewLoading": "Loading…",
|
||||
"chat.attachment.previewError": "Preview failed",
|
||||
"chat.attachment.previewEmpty": "(empty)",
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -490,8 +510,10 @@ function _buildRenderRows(msgs) {
|
|||
timestamp: (m && m.timestamp) != null ? m.timestamp : "",
|
||||
attachments: null,
|
||||
_items: [],
|
||||
_message_ids: [],
|
||||
};
|
||||
}
|
||||
if (m && m.id != null) agg._message_ids.push(m.id);
|
||||
if (role === "assistant") {
|
||||
if (eventType === "reasoning") {
|
||||
if (String(content || "").trim()) {
|
||||
|
|
@ -1035,6 +1057,134 @@ function openChatImageLightbox(src, alt) {
|
|||
document.body.appendChild(backdrop);
|
||||
}
|
||||
|
||||
function openChatMermaidLightbox(svg) {
|
||||
const raw = String(svg || "").trim();
|
||||
if (!raw) return;
|
||||
closeChatImageLightbox();
|
||||
_chatLightboxPrevOverflow = document.body.style.overflow;
|
||||
document.body.style.overflow = "hidden";
|
||||
_chatLightboxKeyHandler = (ev) => {
|
||||
if (ev.key === "Escape") closeChatImageLightbox();
|
||||
};
|
||||
document.addEventListener("keydown", _chatLightboxKeyHandler);
|
||||
|
||||
const backdrop = el("div", {
|
||||
class: "chat-img-lightbox",
|
||||
role: "dialog",
|
||||
"aria-modal": "true",
|
||||
"aria-label": "Mermaid diagram viewer",
|
||||
});
|
||||
const inner = el("div", { class: "chat-img-lightbox__inner" });
|
||||
let scale = 1.0;
|
||||
const clamp = (v) => Math.max(0.2, Math.min(3.0, Number(v || 1)));
|
||||
const applyScale = () => {
|
||||
scale = clamp(scale);
|
||||
viewport.style.transform = `scale(${scale})`;
|
||||
zoomText.textContent = `${Math.round(scale * 100)}%`;
|
||||
};
|
||||
const closeBtn = el("button", {
|
||||
type: "button",
|
||||
class: "chat-img-lightbox__close",
|
||||
text: "×",
|
||||
"aria-label": t("chat.imageViewerClose"),
|
||||
onclick: (e) => {
|
||||
e.stopPropagation();
|
||||
closeChatImageLightbox();
|
||||
},
|
||||
});
|
||||
const toolbar = el("div", { class: "chat-mermaid-lightbox__toolbar" });
|
||||
const btnMinus = el("button", {
|
||||
type: "button",
|
||||
class: "chat-mermaid-lightbox__btn",
|
||||
text: "−",
|
||||
onclick: (e) => {
|
||||
e.stopPropagation();
|
||||
scale = clamp(scale - 0.1);
|
||||
applyScale();
|
||||
},
|
||||
});
|
||||
const btnPlus = el("button", {
|
||||
type: "button",
|
||||
class: "chat-mermaid-lightbox__btn",
|
||||
text: "+",
|
||||
onclick: (e) => {
|
||||
e.stopPropagation();
|
||||
scale = clamp(scale + 0.1);
|
||||
applyScale();
|
||||
},
|
||||
});
|
||||
const btnReset = el("button", {
|
||||
type: "button",
|
||||
class: "chat-mermaid-lightbox__btn",
|
||||
text: "100%",
|
||||
onclick: (e) => {
|
||||
e.stopPropagation();
|
||||
scale = 1.0;
|
||||
applyScale();
|
||||
},
|
||||
});
|
||||
const zoomText = el("span", { class: "chat-mermaid-lightbox__zoom", text: "100%" });
|
||||
toolbar.appendChild(btnMinus);
|
||||
toolbar.appendChild(btnReset);
|
||||
toolbar.appendChild(btnPlus);
|
||||
toolbar.appendChild(zoomText);
|
||||
|
||||
const wrap = el("div", { class: "chat-mermaid-lightbox__svg" });
|
||||
const viewport = el("div", { class: "chat-mermaid-lightbox__viewport" });
|
||||
viewport.innerHTML = raw;
|
||||
wrap.appendChild(viewport);
|
||||
// Drag-to-pan (scroll) inside the zoomable viewport.
|
||||
let dragging = false;
|
||||
let dragStartX = 0;
|
||||
let dragStartY = 0;
|
||||
let dragScrollLeft = 0;
|
||||
let dragScrollTop = 0;
|
||||
const onDragStart = (e) => {
|
||||
// Ignore drags started on toolbar/buttons.
|
||||
if (e && e.target && e.target.closest && e.target.closest(".chat-mermaid-lightbox__toolbar")) return;
|
||||
dragging = true;
|
||||
wrap.classList.add("chat-mermaid-lightbox__svg--dragging");
|
||||
dragStartX = Number(e.clientX || 0);
|
||||
dragStartY = Number(e.clientY || 0);
|
||||
dragScrollLeft = wrap.scrollLeft;
|
||||
dragScrollTop = wrap.scrollTop;
|
||||
};
|
||||
const onDragMove = (e) => {
|
||||
if (!dragging) return;
|
||||
const x = Number(e.clientX || 0);
|
||||
const y = Number(e.clientY || 0);
|
||||
wrap.scrollLeft = dragScrollLeft - (x - dragStartX);
|
||||
wrap.scrollTop = dragScrollTop - (y - dragStartY);
|
||||
};
|
||||
const onDragEnd = () => {
|
||||
dragging = false;
|
||||
wrap.classList.remove("chat-mermaid-lightbox__svg--dragging");
|
||||
};
|
||||
wrap.addEventListener("mousedown", (e) => onDragStart(e));
|
||||
window.addEventListener("mousemove", (e) => onDragMove(e));
|
||||
window.addEventListener("mouseup", () => onDragEnd());
|
||||
wrap.addEventListener(
|
||||
"wheel",
|
||||
(e) => {
|
||||
e.preventDefault();
|
||||
const dy = Number(e.deltaY || 0);
|
||||
const step = dy > 0 ? -0.08 : 0.08;
|
||||
scale = clamp(scale + step);
|
||||
applyScale();
|
||||
},
|
||||
{ passive: false },
|
||||
);
|
||||
inner.appendChild(closeBtn);
|
||||
inner.appendChild(toolbar);
|
||||
inner.appendChild(wrap);
|
||||
backdrop.appendChild(inner);
|
||||
backdrop.addEventListener("click", (e) => {
|
||||
if (e.target === backdrop) closeChatImageLightbox();
|
||||
});
|
||||
document.body.appendChild(backdrop);
|
||||
applyScale();
|
||||
}
|
||||
|
||||
function bindChatImageViewer(messagesEl) {
|
||||
messagesEl.addEventListener("click", (ev) => {
|
||||
const img = ev.target && ev.target.closest && ev.target.closest("img");
|
||||
|
|
@ -1047,6 +1197,16 @@ function bindChatImageViewer(messagesEl) {
|
|||
});
|
||||
}
|
||||
|
||||
function bindChatMermaidViewer(messagesEl) {
|
||||
messagesEl.addEventListener("click", (ev) => {
|
||||
const svg = ev.target && ev.target.closest && ev.target.closest(".mermaid svg");
|
||||
if (!svg || !messagesEl.contains(svg)) return;
|
||||
ev.preventDefault();
|
||||
ev.stopPropagation();
|
||||
openChatMermaidLightbox(svg.outerHTML || "");
|
||||
});
|
||||
}
|
||||
|
||||
/** Keep markdown output safe but allow images (USE_PROFILES html strips img in many DOMPurify builds). */
|
||||
function renderMarkdownHtml(src) {
|
||||
const raw = String(src ?? "");
|
||||
|
|
@ -1068,6 +1228,103 @@ function renderMarkdownHtml(src) {
|
|||
return `<div class="chat-msg__plain">${escapeHtml(raw).replace(/\n/g, "<br/>")}</div>`;
|
||||
}
|
||||
|
||||
let _mermaidBootstrapped = false;
|
||||
let _mermaidRetryTimer = null;
|
||||
window.__oclawHydrateMermaidAll = () => {
|
||||
try {
|
||||
const root = document.getElementById("app") || document.body;
|
||||
hydrateMermaidIn(root);
|
||||
} catch (_) {}
|
||||
};
|
||||
function hydrateMermaidIn(root) {
|
||||
const host = root && root.querySelectorAll ? root : null;
|
||||
if (!host) return;
|
||||
const codeNodes = host.querySelectorAll("pre > code.language-mermaid, pre > code.lang-mermaid");
|
||||
for (const code of codeNodes) {
|
||||
const pre = code.parentElement;
|
||||
if (!pre || !pre.parentElement) continue;
|
||||
const txt = String(code.textContent || "").trim();
|
||||
if (!txt) continue;
|
||||
const box = document.createElement("div");
|
||||
box.className = "mermaid";
|
||||
box.setAttribute("data-mermaid-raw", txt);
|
||||
box.textContent = txt;
|
||||
pre.parentElement.replaceChild(box, pre);
|
||||
}
|
||||
const _attachFallback = (node, errText = "") => {
|
||||
if (!node || node.querySelector(".mermaid-fallback")) return;
|
||||
const raw = String(node.getAttribute("data-mermaid-raw") || "").trim() || String(node.textContent || "").trim();
|
||||
const msg = String(errText || "").trim();
|
||||
node.innerHTML = `<div class="mermaid-fallback">${msg ? `<div class="mermaid-fallback__err">${escapeHtml(msg)}</div>` : ""}<pre>${escapeHtml(raw)}</pre></div>`;
|
||||
};
|
||||
if (typeof mermaid === "undefined") {
|
||||
if (_mermaidRetryTimer != null) return;
|
||||
_mermaidRetryTimer = setTimeout(() => {
|
||||
_mermaidRetryTimer = null;
|
||||
try {
|
||||
window.__oclawHydrateMermaidAll();
|
||||
} catch (_) {}
|
||||
}, 350);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (!_mermaidBootstrapped && typeof mermaid.initialize === "function") {
|
||||
mermaid.initialize({
|
||||
startOnLoad: false,
|
||||
securityLevel: "loose",
|
||||
theme: "base",
|
||||
themeVariables: {
|
||||
background: "transparent",
|
||||
primaryColor: "#1f2937",
|
||||
primaryBorderColor: "#94a3b8",
|
||||
primaryTextColor: "#e5e7eb",
|
||||
lineColor: "#94a3b8",
|
||||
textColor: "#e5e7eb",
|
||||
fontFamily: "Inter, system-ui, -apple-system, Segoe UI, Roboto, sans-serif",
|
||||
},
|
||||
});
|
||||
_mermaidBootstrapped = true;
|
||||
}
|
||||
const nodes = Array.from(host.querySelectorAll(".mermaid")).filter((n) => n && n.isConnected);
|
||||
if (!nodes.length) return;
|
||||
// Mermaid skips already processed nodes. In streaming / rerender scenarios, DOM can be replaced.
|
||||
// Remove the marker so Mermaid treats these nodes as fresh.
|
||||
for (const n of nodes) {
|
||||
try {
|
||||
n.removeAttribute("data-processed");
|
||||
} catch (_) {}
|
||||
}
|
||||
if (typeof mermaid.run === "function") {
|
||||
Promise.resolve(mermaid.run({ nodes }))
|
||||
.then(() => {
|
||||
for (const n of nodes) {
|
||||
if (!n || !n.isConnected) continue;
|
||||
if (!n.querySelector("svg")) _attachFallback(n, "Mermaid render failed (no svg output)");
|
||||
}
|
||||
})
|
||||
.catch((e) => {
|
||||
const msg = String((e && e.message) || e || "Mermaid render failed");
|
||||
for (const n of nodes) _attachFallback(n, msg);
|
||||
});
|
||||
} else if (typeof mermaid.init === "function") {
|
||||
try {
|
||||
mermaid.init(undefined, nodes);
|
||||
for (const n of nodes) {
|
||||
if (!n || !n.isConnected) continue;
|
||||
if (!n.querySelector("svg")) _attachFallback(n, "Mermaid render failed (no svg output)");
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = String((e && e.message) || e || "Mermaid render failed");
|
||||
for (const n of nodes) _attachFallback(n, msg);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
const msg = String((e && e.message) || e || "Mermaid render failed");
|
||||
const nodes = host.querySelectorAll(".mermaid");
|
||||
for (const n of nodes) _attachFallback(n, msg);
|
||||
}
|
||||
}
|
||||
|
||||
const RE_REDACTED_THINKING = new RegExp("<redacted_thinking>\\s*([\\s\\S]*?)\\s*</redacted_thinking>", "i");
|
||||
const RE_THINK_TAG = new RegExp("<think\\s*>\\s*([\\s\\S]*?)\\s*</think\\s*>", "i");
|
||||
|
||||
|
|
@ -1611,6 +1868,7 @@ async function buildMessageBubble(role, content, tsIso) {
|
|||
}
|
||||
|
||||
const _blobUrlCache = new Map();
|
||||
const _attachmentTextPreviewCache = new Map();
|
||||
|
||||
async function fetchAttachmentBlobUrl(attachmentId) {
|
||||
const aid = String(attachmentId || "").trim();
|
||||
|
|
@ -1627,6 +1885,22 @@ async function fetchAttachmentBlobUrl(attachmentId) {
|
|||
return url;
|
||||
}
|
||||
|
||||
async function fetchAttachmentTextPreview(attachmentId, maxChars = 1800) {
|
||||
const aid = String(attachmentId || "").trim();
|
||||
if (!aid) return "";
|
||||
if (_attachmentTextPreviewCache.has(aid)) return _attachmentTextPreviewCache.get(aid) || "";
|
||||
const token = localStorage.getItem(AUTH_TOKEN_KEY) || "";
|
||||
const res = await fetch(`/admin/api/chat/attachments/${encodeURIComponent(aid)}`, {
|
||||
headers: token ? { authorization: `Bearer ${token}` } : {},
|
||||
});
|
||||
if (!res.ok) throw new Error(`preview_http_${res.status}`);
|
||||
const blob = await res.blob();
|
||||
const txt = String(await blob.text());
|
||||
const out = txt.length > maxChars ? `${txt.slice(0, maxChars)}\n…` : txt;
|
||||
_attachmentTextPreviewCache.set(aid, out);
|
||||
return out;
|
||||
}
|
||||
|
||||
function parseAttachments(raw) {
|
||||
if (raw == null || raw === "") return [];
|
||||
if (Array.isArray(raw)) return raw.filter((x) => x && typeof x === "object");
|
||||
|
|
@ -1651,6 +1925,77 @@ async function renderAttachmentsEl(raw) {
|
|||
const list = parseAttachments(raw);
|
||||
if (!list.length) return null;
|
||||
const wrap = el("div", { class: "chat-att-wrap" });
|
||||
const isTextLikeMime = (mime) => {
|
||||
const m = String(mime || "").trim().toLowerCase();
|
||||
if (!m) return false;
|
||||
if (m.startsWith("text/")) return true;
|
||||
return (
|
||||
m.includes("json") ||
|
||||
m.includes("xml") ||
|
||||
m.includes("yaml") ||
|
||||
m.includes("yml") ||
|
||||
m.includes("csv") ||
|
||||
m.includes("javascript") ||
|
||||
m.includes("typescript")
|
||||
);
|
||||
};
|
||||
const buildRefCard = async (att, typ) => {
|
||||
const aid = String(att.attachment_id || att.attachmentId || "").trim();
|
||||
const mime = String(att.mime || att.mime_type || "application/octet-stream").trim();
|
||||
const name = String(att.name || `${typ || "attachment"}`).trim();
|
||||
const bytes = Number(att.bytes || 0);
|
||||
const sizeLabel = bytes > 0 ? `${Math.round((bytes / 1024) * 10) / 10} KB` : "";
|
||||
const card = el("div", { class: "chat-att-ref" });
|
||||
card.appendChild(el("div", { class: "chat-att-ref__name", text: name }));
|
||||
card.appendChild(el("div", { class: "chat-att-ref__meta", text: `${typ} · ${mime}${sizeLabel ? ` · ${sizeLabel}` : ""}` }));
|
||||
if (aid) card.appendChild(el("div", { class: "chat-att-ref__meta", text: `id: ${aid.slice(0, 16)}...` }));
|
||||
if (aid) {
|
||||
const url = await fetchAttachmentBlobUrl(aid);
|
||||
if (url) {
|
||||
card.appendChild(
|
||||
el("a", {
|
||||
class: "chat-att-ref__link",
|
||||
href: url,
|
||||
target: "_blank",
|
||||
rel: "noopener noreferrer",
|
||||
download: name || undefined,
|
||||
text: t("chat.attachment.download"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
const canPreviewText = !!aid && (String(typ || "") === "text_ref" || isTextLikeMime(mime));
|
||||
if (canPreviewText) {
|
||||
const preview = el("button", {
|
||||
type: "button",
|
||||
class: "chat-att-ref__btn",
|
||||
text: t("chat.attachment.preview"),
|
||||
});
|
||||
const pre = el("pre", { class: "chat-att-ref__preview" });
|
||||
preview.addEventListener("click", async () => {
|
||||
if (!pre.hidden) {
|
||||
pre.hidden = true;
|
||||
preview.textContent = t("chat.attachment.preview");
|
||||
return;
|
||||
}
|
||||
preview.disabled = true;
|
||||
preview.textContent = t("chat.attachment.previewLoading");
|
||||
try {
|
||||
const txt = await fetchAttachmentTextPreview(aid);
|
||||
pre.textContent = txt || t("chat.attachment.previewEmpty");
|
||||
} catch (_) {
|
||||
pre.textContent = t("chat.attachment.previewError");
|
||||
} finally {
|
||||
pre.hidden = false;
|
||||
preview.disabled = false;
|
||||
preview.textContent = t("chat.attachment.preview");
|
||||
}
|
||||
});
|
||||
card.appendChild(preview);
|
||||
card.appendChild(pre);
|
||||
}
|
||||
return card;
|
||||
};
|
||||
for (const att of list) {
|
||||
if (!att || typeof att !== "object") continue;
|
||||
const typ = String(att.type || "");
|
||||
|
|
@ -1683,6 +2028,8 @@ async function renderAttachmentsEl(raw) {
|
|||
const src = String(att.url || att.image_url || "").trim();
|
||||
const img = el("img", { class: "chat-att-img", src, alt: "" });
|
||||
wrap.appendChild(img);
|
||||
} else if (typ === "video_ref" || typ === "text_ref" || typ === "binary_ref") {
|
||||
wrap.appendChild(await buildRefCard(att, typ));
|
||||
} else if (typ === "image" || typ === "input_image") {
|
||||
const b64 = att.image_base64 || att.data;
|
||||
const mime = String(att.mime || "image/jpeg");
|
||||
|
|
@ -1693,13 +2040,15 @@ async function renderAttachmentsEl(raw) {
|
|||
wrap.appendChild(el("span", { class: "chat-att-chip", text: String(att.name || "image") }));
|
||||
}
|
||||
} else {
|
||||
wrap.appendChild(el("span", { class: "chat-att-chip", text: `📄 ${String(att.name || "file")}` }));
|
||||
const maybeAid = String(att.attachment_id || att.attachmentId || "").trim();
|
||||
if (maybeAid) wrap.appendChild(await buildRefCard(att, typ || "attachment_ref"));
|
||||
else wrap.appendChild(el("span", { class: "chat-att-chip", text: `📄 ${String(att.name || "file")}` }));
|
||||
}
|
||||
}
|
||||
return wrap.children.length ? wrap : null;
|
||||
}
|
||||
|
||||
async function appendMessageRow(messagesEl, m) {
|
||||
async function appendMessageRow(messagesEl, m, options = {}) {
|
||||
const role = String(m.role || "");
|
||||
const content = String(m.content || "");
|
||||
const ts = m.timestamp != null ? m.timestamp : "";
|
||||
|
|
@ -1714,7 +2063,67 @@ async function appendMessageRow(messagesEl, m) {
|
|||
if (innerBubble) innerBubble.appendChild(att);
|
||||
else bubble.appendChild(att);
|
||||
}
|
||||
const colNode = bubble.querySelector(".chat-msg-col");
|
||||
const innerBubble = bubble.querySelector(".chat-msg-col .chat-msg");
|
||||
if (innerBubble) {
|
||||
const copyText = (() => {
|
||||
const v = String(innerBubble.innerText || "").trim();
|
||||
return v;
|
||||
})();
|
||||
const ids = Array.isArray(m._message_ids) ? m._message_ids.filter((x) => x != null) : (m.id != null ? [m.id] : []);
|
||||
const canDelete = ids.length === 1 && typeof options.onDeleteMessage === "function";
|
||||
if (copyText || canDelete) {
|
||||
const bar = el("div", { class: "chat-msg__actions" });
|
||||
if (copyText) {
|
||||
bar.appendChild(
|
||||
el("button", {
|
||||
type: "button",
|
||||
class: "chat-msg__action-btn",
|
||||
text: "⧉",
|
||||
title: t("chat.copy"),
|
||||
"aria-label": t("chat.copy"),
|
||||
onclick: async (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
try {
|
||||
await navigator.clipboard.writeText(copyText);
|
||||
if (typeof options.onActionStatus === "function") options.onActionStatus(t("chat.copyOk"));
|
||||
} catch (_) {
|
||||
if (typeof options.onActionStatus === "function") options.onActionStatus(t("chat.copyFail"));
|
||||
}
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (canDelete) {
|
||||
bar.appendChild(
|
||||
el("button", {
|
||||
type: "button",
|
||||
class: "chat-msg__action-btn chat-msg__action-btn--danger",
|
||||
text: "🗑",
|
||||
title: t("chat.deleteMessage"),
|
||||
"aria-label": t("chat.deleteMessage"),
|
||||
onclick: async (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
let ok = false;
|
||||
if (typeof options.onConfirm === "function") {
|
||||
ok = await options.onConfirm(t("chat.deleteMessageConfirm"));
|
||||
} else {
|
||||
ok = window.confirm(t("chat.deleteMessageConfirm"));
|
||||
}
|
||||
if (!ok) return;
|
||||
await options.onDeleteMessage(ids[0]);
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (colNode) colNode.appendChild(bar);
|
||||
else innerBubble.appendChild(bar);
|
||||
}
|
||||
}
|
||||
messagesEl.appendChild(bubble);
|
||||
hydrateMermaidIn(bubble);
|
||||
}
|
||||
|
||||
function mount(node) {
|
||||
|
|
@ -1917,6 +2326,7 @@ async function renderChatUi() {
|
|||
const loadMoreWrap = el("div", { class: "chat-load-more" });
|
||||
const messagesEl = el("div", { class: "chat-messages" });
|
||||
bindChatImageViewer(messagesEl);
|
||||
bindChatMermaidViewer(messagesEl);
|
||||
const AUTO_SCROLL_BOTTOM_GAP_PX = 56;
|
||||
let shouldFollowMessages = true;
|
||||
const isNearBottom = () => {
|
||||
|
|
@ -2471,6 +2881,22 @@ async function renderChatUi() {
|
|||
return Number.isFinite(n) ? n : d;
|
||||
}
|
||||
|
||||
const handleDeleteMessage = async (messageId) => {
|
||||
const sid = String(activeId || "").trim();
|
||||
const mid = parseInt(String(messageId || 0), 10);
|
||||
if (!sid || !Number.isFinite(mid) || mid <= 0) return;
|
||||
await apiDelete(`/admin/api/chat/sessions/${encodeURIComponent(sid)}/messages/${mid}`);
|
||||
await loadMessagesForActive();
|
||||
};
|
||||
|
||||
const rowRenderOptions = {
|
||||
onDeleteMessage: handleDeleteMessage,
|
||||
onConfirm: confirmChatAction,
|
||||
onActionStatus: (msg) => {
|
||||
statusBar.textContent = String(msg || "");
|
||||
},
|
||||
};
|
||||
|
||||
const loadMessagesForActive = async () => {
|
||||
loadMessagesForActive._rid = (loadMessagesForActive._rid || 0) + 1;
|
||||
const rid = loadMessagesForActive._rid;
|
||||
|
|
@ -2501,7 +2927,7 @@ async function renderChatUi() {
|
|||
messagesEl.appendChild(el("div", { class: "muted", text: t("chat.empty") }));
|
||||
} else {
|
||||
for (const m of renderRows) {
|
||||
await appendMessageRow(messagesEl, m);
|
||||
await appendMessageRow(messagesEl, m, rowRenderOptions);
|
||||
}
|
||||
}
|
||||
statusBar.textContent = "";
|
||||
|
|
@ -2917,6 +3343,7 @@ async function renderChatUi() {
|
|||
let streamDisplayShown = "";
|
||||
let streamTextBuffer = "";
|
||||
const streamStitcher = createStreamStitcher();
|
||||
let streamMermaidTimerId = null;
|
||||
let perCharNewlineMode = false;
|
||||
let perCharNewlineScore = 0;
|
||||
let toolSeq = 0;
|
||||
|
|
@ -3178,6 +3605,25 @@ ${autoLimit ? `<div style="margin-top:8px;"><span class="muted">auto-added claus
|
|||
}
|
||||
scrollMessagesToBottom();
|
||||
};
|
||||
const _maybeScheduleStreamMermaidHydrate = () => {
|
||||
// Mermaid render is expensive and can crash if DOM is re-written mid-run.
|
||||
// For streaming, only hydrate when we see a complete mermaid fence block.
|
||||
const bubble = streamBubble;
|
||||
if (!bubble) return;
|
||||
const md = bubble.querySelector(".chat-msg__md");
|
||||
if (!md) return;
|
||||
const raw = String(streamDisplayShown || "");
|
||||
if (!raw.includes("```mermaid")) return;
|
||||
// Require a closed fence to avoid parsing partial streams.
|
||||
if (!/```mermaid[\s\S]*?\n```/m.test(raw)) return;
|
||||
if (streamMermaidTimerId != null) clearTimeout(streamMermaidTimerId);
|
||||
streamMermaidTimerId = setTimeout(() => {
|
||||
streamMermaidTimerId = null;
|
||||
try {
|
||||
hydrateMermaidIn(md);
|
||||
} catch (_) {}
|
||||
}, 180);
|
||||
};
|
||||
const renderStreamComposite = () => {
|
||||
if (renderPending) return;
|
||||
renderPending = true;
|
||||
|
|
@ -3185,6 +3631,7 @@ ${autoLimit ? `<div style="margin-top:8px;"><span class="muted">auto-added claus
|
|||
renderPending = false;
|
||||
renderRafId = null;
|
||||
_renderStreamCompositeNow();
|
||||
_maybeScheduleStreamMermaidHydrate();
|
||||
});
|
||||
};
|
||||
const _scheduleTypingTick = () => {
|
||||
|
|
@ -3270,7 +3717,7 @@ ${autoLimit ? `<div style="margin-top:8px;"><span class="muted">auto-added claus
|
|||
const last = rows && rows.length ? rows[rows.length - 1] : null;
|
||||
if (last) {
|
||||
if (streamRow && streamRow.parentNode) streamRow.remove();
|
||||
await appendMessageRow(messagesEl, last);
|
||||
await appendMessageRow(messagesEl, last, rowRenderOptions);
|
||||
scrollMessagesToBottom(true);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -3282,7 +3729,7 @@ ${autoLimit ? `<div style="margin-top:8px;"><span class="muted">auto-added claus
|
|||
role: "assistant",
|
||||
content: decodeEscapedNewlines(t0),
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
}, rowRenderOptions);
|
||||
scrollMessagesToBottom(true);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -3423,6 +3870,10 @@ ${autoLimit ? `<div style="margin-top:8px;"><span class="muted">auto-added claus
|
|||
turnFinalized = true;
|
||||
turnStreamedEnough = streamedEnough;
|
||||
chatStreamSegments = [];
|
||||
if (streamMermaidTimerId != null) {
|
||||
clearTimeout(streamMermaidTimerId);
|
||||
streamMermaidTimerId = null;
|
||||
}
|
||||
// Avoid end-of-turn flash: only reload history when stream had no usable content.
|
||||
if (!streamedEnough) {
|
||||
setTimeout(() => {
|
||||
|
|
@ -3444,6 +3895,10 @@ ${autoLimit ? `<div style="margin-top:8px;"><span class="muted">auto-added claus
|
|||
streamTextBuffer = "";
|
||||
chatRunId = null;
|
||||
chatStreamSegments = [];
|
||||
if (streamMermaidTimerId != null) {
|
||||
clearTimeout(streamMermaidTimerId);
|
||||
streamMermaidTimerId = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (state === "error") {
|
||||
|
|
@ -3456,6 +3911,10 @@ ${autoLimit ? `<div style="margin-top:8px;"><span class="muted">auto-added claus
|
|||
streamTextBuffer = "";
|
||||
chatRunId = null;
|
||||
chatStreamSegments = [];
|
||||
if (streamMermaidTimerId != null) {
|
||||
clearTimeout(streamMermaidTimerId);
|
||||
streamMermaidTimerId = null;
|
||||
}
|
||||
statusBar.textContent = `${t("chat.error")}: ${String(payload.errorMessage || "chat error")}`;
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -576,6 +576,190 @@ body.theme-ds-body .card {
|
|||
border: 1px solid var(--ds-border, rgba(255, 255, 255, 0.08));
|
||||
}
|
||||
|
||||
.chat-msg__actions {
|
||||
margin-top: 6px;
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
justify-content: flex-start;
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transition: opacity 120ms ease;
|
||||
}
|
||||
|
||||
.chat-msg-col:hover .chat-msg__actions {
|
||||
opacity: 1;
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
.chat-msg__action-btn {
|
||||
border: 1px solid rgba(148, 163, 184, 0.25);
|
||||
background: transparent;
|
||||
color: rgba(203, 213, 225, 0.92);
|
||||
border-radius: 999px;
|
||||
font-size: 15px;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 0;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.chat-msg__action-btn:hover {
|
||||
background: rgba(30, 41, 59, 0.45);
|
||||
}
|
||||
|
||||
.chat-msg__action-btn--danger {
|
||||
border-color: rgba(248, 113, 113, 0.45);
|
||||
color: #fca5a5;
|
||||
}
|
||||
|
||||
.chat-msg__md .mermaid {
|
||||
display: block;
|
||||
width: 100%;
|
||||
overflow: auto;
|
||||
padding: 8px;
|
||||
border-radius: 10px;
|
||||
border: 1px solid rgba(148, 163, 184, 0.35);
|
||||
background: rgba(15, 23, 42, 0.35);
|
||||
}
|
||||
|
||||
.chat-msg__md .mermaid svg {
|
||||
display: block;
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.chat-img-lightbox {
|
||||
position: fixed;
|
||||
inset: 0;
|
||||
z-index: 9999;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 18px;
|
||||
background: rgba(0, 0, 0, 0.78);
|
||||
backdrop-filter: blur(2px);
|
||||
}
|
||||
|
||||
.chat-img-lightbox__inner {
|
||||
position: relative;
|
||||
width: min(96vw, 1400px);
|
||||
max-height: 92vh;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border-radius: 14px;
|
||||
border: 1px solid rgba(148, 163, 184, 0.28);
|
||||
background: rgba(2, 6, 23, 0.55);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.chat-img-lightbox__close {
|
||||
position: absolute;
|
||||
top: 10px;
|
||||
right: 10px;
|
||||
width: 36px;
|
||||
height: 36px;
|
||||
border-radius: 10px;
|
||||
border: 1px solid rgba(148, 163, 184, 0.35);
|
||||
background: rgba(15, 23, 42, 0.85);
|
||||
color: rgba(255, 255, 255, 0.9);
|
||||
cursor: pointer;
|
||||
font-size: 22px;
|
||||
line-height: 32px;
|
||||
}
|
||||
|
||||
.chat-img-lightbox__close:hover {
|
||||
background: rgba(30, 41, 59, 0.9);
|
||||
}
|
||||
|
||||
.chat-img-lightbox__img {
|
||||
max-width: 96vw;
|
||||
max-height: 92vh;
|
||||
width: auto;
|
||||
height: auto;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__svg {
|
||||
width: min(96vw, 1400px);
|
||||
max-height: 92vh;
|
||||
overflow: auto;
|
||||
padding: 14px;
|
||||
cursor: grab;
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__svg--dragging {
|
||||
cursor: grabbing;
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__viewport {
|
||||
transform-origin: 0 0;
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__toolbar {
|
||||
position: absolute;
|
||||
left: 10px;
|
||||
top: 10px;
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
padding: 6px 8px;
|
||||
border-radius: 12px;
|
||||
border: 1px solid rgba(148, 163, 184, 0.35);
|
||||
background: rgba(15, 23, 42, 0.85);
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__btn {
|
||||
width: 40px;
|
||||
height: 32px;
|
||||
border-radius: 10px;
|
||||
border: 1px solid rgba(148, 163, 184, 0.3);
|
||||
background: rgba(2, 6, 23, 0.45);
|
||||
color: rgba(255, 255, 255, 0.92);
|
||||
cursor: pointer;
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__btn:hover {
|
||||
background: rgba(30, 41, 59, 0.75);
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__zoom {
|
||||
color: rgba(226, 232, 240, 0.9);
|
||||
font-size: 12px;
|
||||
padding-left: 2px;
|
||||
}
|
||||
|
||||
.chat-mermaid-lightbox__svg svg {
|
||||
width: max(980px, 96vw);
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
}
|
||||
|
||||
.chat-msg__md .mermaid .mermaid-fallback {
|
||||
padding: 8px;
|
||||
border-radius: 8px;
|
||||
background: rgba(2, 6, 23, 0.55);
|
||||
}
|
||||
|
||||
.chat-msg__md .mermaid .mermaid-fallback__err {
|
||||
color: #fca5a5;
|
||||
margin-bottom: 6px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.chat-msg__md .mermaid .mermaid-fallback pre {
|
||||
margin: 0;
|
||||
white-space: pre-wrap;
|
||||
color: #e5e7eb;
|
||||
}
|
||||
|
||||
.chat-msg--tool {
|
||||
align-self: flex-start;
|
||||
font-family: ui-monospace, monospace;
|
||||
|
|
|
|||
|
|
@ -1613,6 +1613,29 @@ class SqliteStore:
|
|||
conn.execute("DELETE FROM chat_session WHERE id = ?", (str(session_id),))
|
||||
return True
|
||||
|
||||
def delete_message(self, *, session_id: str, message_id: int) -> bool:
|
||||
sid = str(session_id or "").strip()
|
||||
mid = int(message_id or 0)
|
||||
if not sid or mid <= 0:
|
||||
return False
|
||||
with self._connect() as conn:
|
||||
cur = conn.execute(
|
||||
"DELETE FROM chat_message WHERE session_id = ? AND id = ?",
|
||||
(sid, mid),
|
||||
)
|
||||
if int(cur.rowcount or 0) <= 0:
|
||||
return False
|
||||
last_row = conn.execute(
|
||||
"SELECT MAX(timestamp) AS ts FROM chat_message WHERE session_id = ?",
|
||||
(sid,),
|
||||
).fetchone()
|
||||
last_ts = str((last_row["ts"] if last_row else "") or "").strip() or None
|
||||
conn.execute(
|
||||
"UPDATE chat_session SET last_message_at = ? WHERE id = ?",
|
||||
(last_ts, sid),
|
||||
)
|
||||
return True
|
||||
|
||||
def add_message(
|
||||
self,
|
||||
session_id: str,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import re
|
|||
from typing import Any
|
||||
|
||||
from oclaw.platform.llm.chat_models import _normalize_image_b64_payload, gemini_openai_compat_client, ChatModel
|
||||
from oclaw.runtime.chat.media_redact import redact_embedded_image_blobs
|
||||
from oclaw.runtime.chat.tool_runtime import tool_llm_message_max_chars, truncate_tool_result_for_llm_messages
|
||||
from oclaw.prompts import render_prompt
|
||||
from oclaw.platform.files.attachment_assets import attachment_id_to_data_url
|
||||
|
|
@ -196,8 +197,18 @@ def build_llm_messages(
|
|||
model: ChatModel,
|
||||
lang: str,
|
||||
tool_context_truncate_enabled: bool = True,
|
||||
active_turn_uuid: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""把 DB 中的消息序列转换为 LLM messages。"""
|
||||
"""把 DB 中的消息序列转换为 LLM messages。
|
||||
|
||||
When ``active_turn_uuid`` matches a tool/user row ``turn_uuid``, that turn is treated as the
|
||||
in-flight MCP turn: tool JSON is not stripped of nested image payloads here (see also
|
||||
:func:`~oclaw.runtime.direct_loop._guard_tool_results_for_llm_context`). Omit or leave empty
|
||||
to apply image-blob stripping for every tool row (safe default for callers without turn context).
|
||||
|
||||
Only the **last** user message may expand attachments into native multimodal ``input_image``;
|
||||
older user attachments are replayed as text metadata only.
|
||||
"""
|
||||
out: list[dict[str, Any]] = [{"role": "system", "content": (system_prompt or "").strip()}]
|
||||
thinking_mode_enabled = bool(getattr(model, "thinking_mode_enabled", False))
|
||||
allow_signature_replay = _allow_reasoning_signature_replay(model)
|
||||
|
|
@ -239,6 +250,15 @@ def build_llm_messages(
|
|||
seen_tool_ids.add(tcid)
|
||||
tool_ids_after.append(set(seen_tool_ids))
|
||||
tool_ids_after.reverse()
|
||||
|
||||
last_user_msg_idx = -1
|
||||
for _ui, _um in enumerate(store_messages or []):
|
||||
if str(getattr(_um, "role", "") or "") != "user":
|
||||
continue
|
||||
if str(getattr(_um, "event_type", "") or "").strip().lower() == "reasoning":
|
||||
continue
|
||||
last_user_msg_idx = _ui
|
||||
|
||||
def _attach_reasoning_content(row: dict[str, Any], m: Any) -> dict[str, Any]:
|
||||
if not thinking_mode_enabled:
|
||||
return row
|
||||
|
|
@ -291,34 +311,53 @@ def build_llm_messages(
|
|||
if not isinstance(att, dict):
|
||||
continue
|
||||
att_type = att.get("type")
|
||||
expand_user_image_for_model = bool(i == last_user_msg_idx)
|
||||
if att_type in ("image", "input_image"):
|
||||
b64 = _normalize_image_b64_payload(att.get("image_base64") or att.get("data"))
|
||||
if not b64:
|
||||
continue
|
||||
content_list.append(
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_base64": b64,
|
||||
"mime": att.get("mime") or "image/jpeg",
|
||||
}
|
||||
)
|
||||
if expand_user_image_for_model:
|
||||
b64 = _normalize_image_b64_payload(att.get("image_base64") or att.get("data"))
|
||||
if not b64:
|
||||
continue
|
||||
content_list.append(
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_base64": b64,
|
||||
"mime": att.get("mime") or "image/jpeg",
|
||||
}
|
||||
)
|
||||
else:
|
||||
name = str(att.get("name") or "image")
|
||||
mime = str(att.get("mime") or "image/jpeg")
|
||||
hs = "(historical attachment; pixels not replayed into model)"
|
||||
hs_zh = "(历史附件;不向模型回放像素)"
|
||||
hint = hs_zh if not str(lang or "").startswith("en") else hs
|
||||
meta_line = f"- name={name} mime={mime} {hint}"
|
||||
content_list.append(
|
||||
{
|
||||
"type": "text",
|
||||
"text": render_prompt(
|
||||
"tools/image_attachment_meta.md",
|
||||
variables={"meta_line": meta_line},
|
||||
strict=True,
|
||||
),
|
||||
}
|
||||
)
|
||||
elif att_type == "image_ref":
|
||||
# Prefer actual image bytes so multi-agent/image specialist can truly "see" history images.
|
||||
name = str(att.get("name") or "image")
|
||||
mime = str(att.get("mime") or "image/jpeg")
|
||||
aid = str(att.get("attachment_id") or "")
|
||||
data_url = attachment_id_to_data_url(aid, mime=mime) if aid else ""
|
||||
if data_url:
|
||||
if ";base64," in data_url:
|
||||
b64 = data_url.split(";base64,", 1)[1]
|
||||
content_list.append(
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_base64": b64,
|
||||
"mime": mime,
|
||||
}
|
||||
)
|
||||
continue
|
||||
if expand_user_image_for_model:
|
||||
data_url = attachment_id_to_data_url(aid, mime=mime) if aid else ""
|
||||
if data_url:
|
||||
if ";base64," in data_url:
|
||||
b64 = data_url.split(";base64,", 1)[1]
|
||||
content_list.append(
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_base64": b64,
|
||||
"mime": mime,
|
||||
}
|
||||
)
|
||||
continue
|
||||
w = att.get("width")
|
||||
h = att.get("height")
|
||||
sz = att.get("bytes")
|
||||
|
|
@ -423,7 +462,7 @@ def build_llm_messages(
|
|||
aid = str(_fid or "").strip()
|
||||
except Exception:
|
||||
aid = ""
|
||||
if aid and mime.startswith("image/"):
|
||||
if expand_user_image_for_model and aid and mime.startswith("image/"):
|
||||
data_url = attachment_id_to_data_url(aid, mime=mime)
|
||||
if data_url and ";base64," in data_url:
|
||||
b64 = data_url.split(";base64,", 1)[1]
|
||||
|
|
@ -575,6 +614,15 @@ def build_llm_messages(
|
|||
)
|
||||
continue
|
||||
raw_tc_content = getattr(m, "content", "") or ""
|
||||
_tun = str(getattr(m, "turn_uuid", "") or "").strip()
|
||||
_aus = str(active_turn_uuid or "").strip()
|
||||
if (not _aus) or (_tun != _aus):
|
||||
try:
|
||||
_p = json.loads(raw_tc_content)
|
||||
_p2 = redact_embedded_image_blobs(_p)
|
||||
raw_tc_content = json.dumps(_p2, ensure_ascii=False, default=str)
|
||||
except Exception:
|
||||
pass
|
||||
tool_content_out = raw_tc_content
|
||||
cap = tool_llm_message_max_chars()
|
||||
if str(tool_call_id) in historical_tool_ids:
|
||||
|
|
|
|||
198
runtime/chat/media_redact.py
Normal file
198
runtime/chat/media_redact.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
"""Strip/ingest embedded binary payloads from tool/MCP-shaped JSON.
|
||||
|
||||
Persistence is untouched; callers use copies when building model context."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Any
|
||||
from oclaw.platform.files.attachment_assets import AttachmentAssetStore
|
||||
|
||||
_IMAGE_CONTENT_TYPES = frozenset({"image", "input_image"})
|
||||
_BASE64_PAYLOAD_KEYS = ("data", "image_base64", "base64", "content_base64", "body_base64")
|
||||
# Below this length we keep values (tiny icons / markers).
|
||||
_MIN_B64_CHARS = 200
|
||||
|
||||
|
||||
def redact_embedded_image_blobs(obj: Any) -> Any:
|
||||
"""Deep-copy-ish transform: replace large base64 payloads with metadata placeholders."""
|
||||
if isinstance(obj, dict):
|
||||
return _redact_dict(obj)
|
||||
if isinstance(obj, list):
|
||||
return [redact_embedded_image_blobs(x) for x in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _looks_like_large_payload(s: str) -> bool:
|
||||
t = str(s or "").strip()
|
||||
if len(t) < _MIN_B64_CHARS:
|
||||
return False
|
||||
allowed = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r-_")
|
||||
if not t[: min(512, len(t))]:
|
||||
return False
|
||||
noise = sum(1 for ch in t[: min(2000, len(t))] if ch not in allowed)
|
||||
return noise <= max(2, len(t[: min(2000, len(t))]) // 200)
|
||||
|
||||
|
||||
def _redact_dict(d: dict[str, Any]) -> dict[str, Any]:
|
||||
typ = str(d.get("type") or "").strip().lower()
|
||||
payload_keys = [k for k in _BASE64_PAYLOAD_KEYS if isinstance(d.get(k), str) and _looks_like_large_payload(str(d.get(k) or ""))]
|
||||
if payload_keys:
|
||||
plen = max(len(str(d.get(k) or "")) for k in payload_keys)
|
||||
dup: dict[str, Any] = {}
|
||||
for k, v in d.items():
|
||||
if k in payload_keys:
|
||||
continue
|
||||
if isinstance(v, dict):
|
||||
dup[k] = _redact_dict(v)
|
||||
elif isinstance(v, list):
|
||||
dup[k] = [redact_embedded_image_blobs(x) for x in v]
|
||||
else:
|
||||
dup[k] = v
|
||||
is_image = typ in _IMAGE_CONTENT_TYPES
|
||||
dup["_image_payload_redacted" if is_image else "_binary_payload_redacted"] = True
|
||||
dup["_redacted_payload_chars"] = int(plen)
|
||||
dup["_redacted_payload_keys"] = list(payload_keys)
|
||||
return dup
|
||||
|
||||
out: dict[str, Any] = {}
|
||||
for k, v in d.items():
|
||||
if isinstance(v, dict):
|
||||
out[k] = _redact_dict(v)
|
||||
elif isinstance(v, list):
|
||||
out[k] = [redact_embedded_image_blobs(x) for x in v]
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
|
||||
def ingest_embedded_image_blobs_as_refs(
|
||||
obj: Any,
|
||||
*,
|
||||
root_dir: str | None = None,
|
||||
filename_prefix: str = "tool-image",
|
||||
) -> tuple[Any, list[dict[str, Any]]]:
|
||||
"""Persist nested base64 blobs and replace them with attachment refs.
|
||||
|
||||
Returns transformed object and newly created attachment refs.
|
||||
"""
|
||||
store = AttachmentAssetStore(root_dir=root_dir) if root_dir else AttachmentAssetStore()
|
||||
refs: list[dict[str, Any]] = []
|
||||
|
||||
def _ingest(node: Any, idx_seed: list[int]) -> Any:
|
||||
if isinstance(node, list):
|
||||
return [_ingest(x, idx_seed) for x in node]
|
||||
if not isinstance(node, dict):
|
||||
return node
|
||||
typ = str(node.get("type") or "").strip().lower()
|
||||
raw = _pick_base64_payload(node)
|
||||
if raw:
|
||||
blob = _decode_image_bytes(raw)
|
||||
if blob:
|
||||
idx_seed[0] += 1
|
||||
mime = str(node.get("mime") or node.get("mime_type") or "image/png").strip() or "image/png"
|
||||
ext = _filename_ext_for_mime(mime)
|
||||
name = str(node.get("name") or f"{filename_prefix}-{idx_seed[0]}{ext}").strip()
|
||||
meta = store.save_bytes(
|
||||
blob,
|
||||
filename=name,
|
||||
mime=mime,
|
||||
width=_safe_int(node.get("width")),
|
||||
height=_safe_int(node.get("height")),
|
||||
)
|
||||
ref_type = _ref_type_for_mime(mime, typ)
|
||||
ref = {
|
||||
"type": ref_type,
|
||||
"attachment_id": meta.attachment_id,
|
||||
"name": meta.name,
|
||||
"mime": meta.mime,
|
||||
"bytes": meta.bytes,
|
||||
"width": meta.width,
|
||||
"height": meta.height,
|
||||
}
|
||||
refs.append(ref)
|
||||
return ref
|
||||
redacted = _redact_dict(node)
|
||||
redacted["type"] = _ref_type_for_mime(
|
||||
str(node.get("mime") or node.get("mime_type") or "application/octet-stream"),
|
||||
typ,
|
||||
)
|
||||
redacted.setdefault("name", str(node.get("name") or "attachment"))
|
||||
redacted.setdefault("mime", str(node.get("mime") or node.get("mime_type") or "application/octet-stream"))
|
||||
return redacted
|
||||
out: dict[str, Any] = {}
|
||||
for k, v in node.items():
|
||||
out[k] = _ingest(v, idx_seed)
|
||||
return out
|
||||
|
||||
transformed = _ingest(obj, [0])
|
||||
uniq: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for r in refs:
|
||||
aid = str(r.get("attachment_id") or "").strip()
|
||||
if not aid or aid in seen:
|
||||
continue
|
||||
seen.add(aid)
|
||||
uniq.append(r)
|
||||
return transformed, uniq
|
||||
|
||||
|
||||
def _decode_image_bytes(raw: Any) -> bytes:
|
||||
s = str(raw or "").strip()
|
||||
if not s:
|
||||
return b""
|
||||
if s.startswith("data:") and ";base64," in s:
|
||||
s = s.split(";base64,", 1)[1]
|
||||
try:
|
||||
return base64.b64decode(s.encode("ascii"), validate=False)
|
||||
except Exception:
|
||||
return b""
|
||||
|
||||
|
||||
def _pick_base64_payload(node: dict[str, Any]) -> str:
|
||||
for k in _BASE64_PAYLOAD_KEYS:
|
||||
v = node.get(k)
|
||||
if isinstance(v, str) and str(v).strip():
|
||||
return v
|
||||
return ""
|
||||
|
||||
|
||||
def _ref_type_for_mime(mime: str, typ: str = "") -> str:
|
||||
m = str(mime or "").strip().lower()
|
||||
t = str(typ or "").strip().lower()
|
||||
if t in _IMAGE_CONTENT_TYPES or m.startswith("image/"):
|
||||
return "image_ref"
|
||||
if m.startswith("video/"):
|
||||
return "video_ref"
|
||||
if m.startswith("text/"):
|
||||
return "text_ref"
|
||||
return "binary_ref"
|
||||
|
||||
|
||||
def _filename_ext_for_mime(mime: str) -> str:
|
||||
m = str(mime or "").strip().lower()
|
||||
if m == "image/png":
|
||||
return ".png"
|
||||
if m in {"image/jpeg", "image/jpg"}:
|
||||
return ".jpg"
|
||||
if m == "image/webp":
|
||||
return ".webp"
|
||||
if m == "image/gif":
|
||||
return ".gif"
|
||||
if m == "video/mp4":
|
||||
return ".mp4"
|
||||
if m == "text/plain":
|
||||
return ".txt"
|
||||
return ".bin"
|
||||
|
||||
|
||||
def _safe_int(raw: Any) -> int | None:
|
||||
try:
|
||||
if raw is None:
|
||||
return None
|
||||
return int(raw)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
__all__ = ["redact_embedded_image_blobs", "ingest_embedded_image_blobs_as_refs"]
|
||||
87
runtime/chat/model_path_audit.py
Normal file
87
runtime/chat/model_path_audit.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
_MIN_B64_CHARS = 200
|
||||
|
||||
|
||||
def ensure_no_tool_or_embedded_image_payload(*, messages: list[dict[str, Any]], path: str) -> None:
|
||||
"""Guard non-turn model paths and degrade in place instead of raising.
|
||||
|
||||
- `role=tool` is downgraded to assistant text summary.
|
||||
- Embedded image/base64 payloads are replaced with safe text placeholders.
|
||||
"""
|
||||
for m in messages or []:
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
role = str(m.get("role") or "").strip().lower()
|
||||
if role == "tool":
|
||||
m["role"] = "assistant"
|
||||
m["content"] = f"[model_path_audit:{path}] tool payload omitted"
|
||||
continue
|
||||
content = m.get("content")
|
||||
if _contains_embedded_image_payload(content):
|
||||
m["content"] = _sanitize_content(content, path=path)
|
||||
|
||||
|
||||
def _contains_embedded_image_payload(obj: Any) -> bool:
|
||||
if isinstance(obj, str):
|
||||
return _contains_large_base64_like_text(obj)
|
||||
if isinstance(obj, list):
|
||||
return any(_contains_embedded_image_payload(x) for x in obj)
|
||||
if not isinstance(obj, dict):
|
||||
return False
|
||||
typ = str(obj.get("type") or "").strip().lower()
|
||||
if typ in {"image", "input_image"}:
|
||||
for k in ("data", "image_base64"):
|
||||
v = obj.get(k)
|
||||
if isinstance(v, str) and len(v.strip()) >= _MIN_B64_CHARS:
|
||||
return True
|
||||
for v in obj.values():
|
||||
if _contains_embedded_image_payload(v):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _contains_large_base64_like_text(text: str) -> bool:
|
||||
s = str(text or "").strip()
|
||||
if len(s) < _MIN_B64_CHARS:
|
||||
return False
|
||||
if s.startswith("data:") and ";base64," in s:
|
||||
s = s.split(";base64,", 1)[1]
|
||||
head = s[: min(4096, len(s))]
|
||||
if len(head) < _MIN_B64_CHARS:
|
||||
return False
|
||||
allowed = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n\r-_")
|
||||
noise = sum(1 for ch in head if ch not in allowed)
|
||||
# Similar heuristic to media redaction: mostly base64 alphabet over a long span.
|
||||
return noise <= max(4, len(head) // 200)
|
||||
|
||||
|
||||
def _sanitize_content(content: Any, *, path: str) -> Any:
|
||||
if isinstance(content, str):
|
||||
if _contains_large_base64_like_text(content):
|
||||
return f"[model_path_audit:{path}] base64 payload omitted"
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
out: list[Any] = []
|
||||
for item in content:
|
||||
if isinstance(item, dict):
|
||||
typ = str(item.get("type") or "").strip().lower()
|
||||
if typ in {"image", "input_image"}:
|
||||
out.append({"type": "text", "text": f"[model_path_audit:{path}] image payload omitted"})
|
||||
continue
|
||||
out.append(_sanitize_content(item, path=path))
|
||||
return out
|
||||
if isinstance(content, dict):
|
||||
out: dict[str, Any] = {}
|
||||
for k, v in content.items():
|
||||
if str(k) in {"data", "image_base64"} and isinstance(v, str) and _contains_large_base64_like_text(v):
|
||||
out[k] = f"[model_path_audit:{path}] payload omitted"
|
||||
continue
|
||||
out[k] = _sanitize_content(v, path=path)
|
||||
return out
|
||||
return content
|
||||
|
||||
|
||||
__all__ = ["ensure_no_tool_or_embedded_image_payload"]
|
||||
|
|
@ -21,6 +21,7 @@ from oclaw.platform.persistence.sqlite_store import SqliteStore
|
|||
from oclaw.runtime.tools.base import ToolRegistry
|
||||
from oclaw.platform.llm.chat_models import LLMToolCall
|
||||
from oclaw.runtime.tools.tool_validation import validate_tool_arguments
|
||||
from oclaw.runtime.chat.media_redact import ingest_embedded_image_blobs_as_refs
|
||||
from oclaw.runtime.tools.experts.workspace.workspace_base import (
|
||||
workspace_path_access_scope,
|
||||
workspace_write_namespace_scope,
|
||||
|
|
@ -58,13 +59,15 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]:
|
|||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
aid = str(result.get("attachment_id") or "").strip()
|
||||
root_mime = str(result.get("mime") or "").strip()
|
||||
if aid:
|
||||
ref_type = _ref_type_for_mime(root_mime)
|
||||
out.append(
|
||||
{
|
||||
"type": "image_ref",
|
||||
"type": ref_type,
|
||||
"attachment_id": aid,
|
||||
"name": str(result.get("name") or "generated-image"),
|
||||
"mime": str(result.get("mime") or "image/png"),
|
||||
"mime": root_mime or "application/octet-stream",
|
||||
"bytes": result.get("bytes"),
|
||||
"width": result.get("width"),
|
||||
"height": result.get("height"),
|
||||
|
|
@ -91,12 +94,16 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]:
|
|||
continue
|
||||
r_aid = str(r.get("attachment_id") or "").strip()
|
||||
if r_aid:
|
||||
r_typ = str(r.get("type") or "").strip().lower()
|
||||
r_mime = str(r.get("mime_type") or r.get("mime") or "").strip()
|
||||
if r_typ not in {"image_ref", "video_ref", "text_ref", "binary_ref"}:
|
||||
r_typ = _ref_type_for_mime(r_mime)
|
||||
out.append(
|
||||
{
|
||||
"type": "image_ref",
|
||||
"type": r_typ,
|
||||
"attachment_id": r_aid,
|
||||
"name": str(r.get("name") or "generated-image"),
|
||||
"mime": str(r.get("mime") or "image/png"),
|
||||
"mime": r_mime or "application/octet-stream",
|
||||
"bytes": r.get("bytes"),
|
||||
"width": r.get("width"),
|
||||
"height": r.get("height"),
|
||||
|
|
@ -110,15 +117,18 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]:
|
|||
if not isinstance(item, dict):
|
||||
continue
|
||||
typ = str(item.get("type") or "").strip().lower()
|
||||
if typ in {"image", "input_image"}:
|
||||
b64 = item.get("image_base64") or item.get("data")
|
||||
if isinstance(b64, str) and b64.strip():
|
||||
if typ in {"image_ref", "video_ref", "text_ref", "binary_ref"}:
|
||||
a_id = str(item.get("attachment_id") or "").strip()
|
||||
if a_id:
|
||||
out.append(
|
||||
{
|
||||
"type": "image",
|
||||
"data": b64.strip(),
|
||||
"mime": str(item.get("mime_type") or item.get("mime") or "image/png"),
|
||||
"name": str(item.get("name") or "tool-image"),
|
||||
"type": typ,
|
||||
"attachment_id": a_id,
|
||||
"mime": str(item.get("mime_type") or item.get("mime") or "application/octet-stream"),
|
||||
"name": str(item.get("name") or "tool-attachment"),
|
||||
"bytes": item.get("bytes"),
|
||||
"width": item.get("width"),
|
||||
"height": item.get("height"),
|
||||
}
|
||||
)
|
||||
elif typ == "image_url":
|
||||
|
|
@ -132,7 +142,7 @@ def _attachments_from_tool_result(result: Any) -> list[dict[str, Any]]:
|
|||
a.get("attachment_id")
|
||||
or a.get("pointer_uri")
|
||||
or a.get("url")
|
||||
or (f"b64:{a.get('mime')}:{len(str(a.get('data') or ''))}" if a.get("data") else "")
|
||||
or ""
|
||||
).strip()
|
||||
if not k or k in seen:
|
||||
continue
|
||||
|
|
@ -828,6 +838,10 @@ class ToolExecutor:
|
|||
)
|
||||
result, duration_ms = results_by_id[tc.id]
|
||||
result = normalize_tool_result(result)
|
||||
persisted_result, ingested_refs = ingest_embedded_image_blobs_as_refs(
|
||||
result,
|
||||
filename_prefix=f"{str(tc.name or 'tool')}-{str(tc.id or '')}",
|
||||
)
|
||||
logger.info(
|
||||
"tool_runtime tool session=%s name=%s duration_ms=%d ok=%s",
|
||||
ctx.session_id[:12],
|
||||
|
|
@ -840,7 +854,7 @@ class ToolExecutor:
|
|||
session_id=ctx.session_id,
|
||||
tool_name=tc.name,
|
||||
args=tc.arguments,
|
||||
result=result,
|
||||
result=persisted_result,
|
||||
specialist=ctx.specialist,
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
|
|
@ -849,7 +863,7 @@ class ToolExecutor:
|
|||
# until the turn finishes, so current-round model context remains lossless.
|
||||
t_trunc = time.perf_counter()
|
||||
observed_rows_this_call = int(_estimate_observed_rows(result))
|
||||
result_for_llm = dict(result or {})
|
||||
result_for_llm = dict(persisted_result or {})
|
||||
if tc.name in _SQL_REPLAY_COMPACT_TOOL_NAMES:
|
||||
current = int(local_turn_tool_name_counts.get(tc.name, 0))
|
||||
current_rows = int(local_turn_tool_observed_rows.get(tc.name, 0))
|
||||
|
|
@ -870,7 +884,7 @@ class ToolExecutor:
|
|||
role="tool",
|
||||
content=tool_content,
|
||||
tool_calls={"tool_call_id": tc.id, "name": tc.name, "assistant_message_id": assistant_msg_id},
|
||||
attachments=_attachments_from_tool_result(result) or None,
|
||||
attachments=(_merge_attachments(_attachments_from_tool_result(persisted_result), ingested_refs) or None),
|
||||
turn_uuid=ctx.turn_uuid,
|
||||
event_type="tool_result",
|
||||
event_payload={"tool_name": tc.name, "observed_rows": int(observed_rows_this_call)},
|
||||
|
|
@ -971,3 +985,29 @@ __all__ = [
|
|||
"truncate_tool_result_for_llm_messages",
|
||||
"compact_turn_tool_messages_for_storage",
|
||||
]
|
||||
|
||||
|
||||
def _merge_attachments(*parts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for part in parts:
|
||||
for a in part or []:
|
||||
if not isinstance(a, dict):
|
||||
continue
|
||||
k = str(a.get("attachment_id") or a.get("pointer_uri") or a.get("url") or "").strip()
|
||||
if not k or k in seen:
|
||||
continue
|
||||
seen.add(k)
|
||||
out.append(a)
|
||||
return out
|
||||
|
||||
|
||||
def _ref_type_for_mime(mime: str) -> str:
|
||||
m = str(mime or "").strip().lower()
|
||||
if m.startswith("image/"):
|
||||
return "image_ref"
|
||||
if m.startswith("video/"):
|
||||
return "video_ref"
|
||||
if m.startswith("text/"):
|
||||
return "text_ref"
|
||||
return "binary_ref"
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from types import SimpleNamespace
|
|||
from typing import Any, Callable, Optional
|
||||
|
||||
from oclaw.runtime.chat.agent_messages import build_llm_messages
|
||||
from oclaw.runtime.chat.media_redact import redact_embedded_image_blobs
|
||||
from oclaw.runtime.chat.tool_runtime import ToolExecutionConfig
|
||||
from oclaw.runtime.chat.turn_types import TurnRunOutcome
|
||||
from oclaw.runtime.skill_executor import SkillExecutionContext, SkillExecutor
|
||||
|
|
@ -269,6 +270,21 @@ def _json_dumps_safe(obj: Any) -> str:
|
|||
return json.dumps({"ok": False, "error": "not_json_serializable"}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _tool_message_with_content(m: Any, content: str, *, sid: str = "") -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id=getattr(m, "id", 0),
|
||||
session_id=str(getattr(m, "session_id", None) or sid or ""),
|
||||
role="tool",
|
||||
content=content,
|
||||
tool_calls=getattr(m, "tool_calls", None),
|
||||
timestamp=getattr(m, "timestamp", ""),
|
||||
attachments=getattr(m, "attachments", None),
|
||||
turn_uuid=getattr(m, "turn_uuid", None),
|
||||
event_type=getattr(m, "event_type", None),
|
||||
event_payload=getattr(m, "event_payload", None),
|
||||
)
|
||||
|
||||
|
||||
def _split_reasoning_and_body(text: str, *, explicit_reasoning: str | None = None) -> tuple[list[str], str]:
|
||||
explicit = str(explicit_reasoning or "").strip()
|
||||
raw = str(text or "")
|
||||
|
|
@ -317,6 +333,12 @@ def _guard_tool_results_for_llm_context(
|
|||
out.append(m)
|
||||
continue
|
||||
raw = str(getattr(m, "content", "") or "")
|
||||
try:
|
||||
_parsed0 = json.loads(raw)
|
||||
_parsed1 = redact_embedded_image_blobs(_parsed0)
|
||||
raw = _json_dumps_safe(_parsed1)
|
||||
except Exception:
|
||||
pass
|
||||
# Best-effort parse tool JSON for image-query specific guard and overflow metadata.
|
||||
ok = None
|
||||
error_code = ""
|
||||
|
|
@ -350,17 +372,7 @@ def _guard_tool_results_for_llm_context(
|
|||
"图片分析结果在上下文回放中已截断,请缩小 query_image_attachment 的问题范围。"
|
||||
)
|
||||
guarded = _json_dumps_safe(guarded_obj)
|
||||
out.append(
|
||||
SimpleNamespace(
|
||||
id=getattr(m, "id", 0),
|
||||
session_id=getattr(m, "session_id", session_id),
|
||||
role="tool",
|
||||
content=guarded,
|
||||
tool_calls=getattr(m, "tool_calls", None),
|
||||
timestamp=getattr(m, "timestamp", ""),
|
||||
attachments=getattr(m, "attachments", None),
|
||||
)
|
||||
)
|
||||
out.append(_tool_message_with_content(m, guarded, sid=session_id))
|
||||
continue
|
||||
# Guard video transcript replay similarly (usually long).
|
||||
if str(obj.get("task") or "").strip().lower() == "transcript" and has_attachment_id and len(text) > video_cap:
|
||||
|
|
@ -371,20 +383,10 @@ def _guard_tool_results_for_llm_context(
|
|||
guarded_obj["video_result_original_chars"] = len(text)
|
||||
guarded_obj["video_result_replay_cap_chars"] = video_cap
|
||||
guarded = _json_dumps_safe(guarded_obj)
|
||||
out.append(
|
||||
SimpleNamespace(
|
||||
id=getattr(m, "id", 0),
|
||||
session_id=getattr(m, "session_id", session_id),
|
||||
role="tool",
|
||||
content=guarded,
|
||||
tool_calls=getattr(m, "tool_calls", None),
|
||||
timestamp=getattr(m, "timestamp", ""),
|
||||
attachments=getattr(m, "attachments", None),
|
||||
)
|
||||
)
|
||||
out.append(_tool_message_with_content(m, guarded, sid=session_id))
|
||||
continue
|
||||
if len(raw) <= cap:
|
||||
out.append(m)
|
||||
out.append(_tool_message_with_content(m, raw, sid=session_id))
|
||||
continue
|
||||
preview = raw[: max(1, min(4000, cap - 400))] + "\n...<tool_result_guard_truncated>"
|
||||
guarded_obj = {
|
||||
|
|
@ -402,17 +404,7 @@ def _guard_tool_results_for_llm_context(
|
|||
),
|
||||
}
|
||||
guarded = _json_dumps_safe(guarded_obj)
|
||||
out.append(
|
||||
SimpleNamespace(
|
||||
id=getattr(m, "id", 0),
|
||||
session_id=getattr(m, "session_id", session_id),
|
||||
role="tool",
|
||||
content=guarded,
|
||||
tool_calls=getattr(m, "tool_calls", None),
|
||||
timestamp=getattr(m, "timestamp", ""),
|
||||
attachments=getattr(m, "attachments", None),
|
||||
)
|
||||
)
|
||||
out.append(_tool_message_with_content(m, guarded, sid=session_id))
|
||||
if trace_id:
|
||||
_emit_direct_loop_trace(
|
||||
store=store,
|
||||
|
|
@ -631,6 +623,7 @@ def _build_model_context(
|
|||
model=model,
|
||||
lang=lang,
|
||||
tool_context_truncate_enabled=tool_context_truncate_enabled,
|
||||
active_turn_uuid=active_turn_uuid,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from oclaw.runtime.router import decide_route
|
|||
from oclaw.runtime.worker import ensure_worker_started
|
||||
from oclaw.runtime.orchestration.trace import new_span_id, new_trace_id
|
||||
from oclaw.runtime.chat.tool_runtime import compact_turn_tool_messages_for_storage
|
||||
from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload
|
||||
|
||||
_OC_STAGE_BY_EVENT: dict[str, str] = {
|
||||
"gateway_received": "ingress",
|
||||
|
|
@ -203,11 +204,9 @@ class OclawGateway:
|
|||
else "仅基于以下用户正文生成简短会话标题。请使用对话内容主体语言命名。"
|
||||
"只返回标题文本,不要引号,不要markdown,最多18个字。"
|
||||
)
|
||||
resp = model.chat(
|
||||
[{"role": "system", "content": sys}, {"role": "user", "content": body}],
|
||||
[],
|
||||
on_token=None,
|
||||
)
|
||||
messages = [{"role": "system", "content": sys}, {"role": "user", "content": body}]
|
||||
ensure_no_tool_or_embedded_image_payload(messages=messages, path="gateway.auto_title")
|
||||
resp = model.chat(messages, [], on_token=None)
|
||||
title = str(getattr(resp, "content", "") or "").strip().replace("\n", " ")
|
||||
title = title.strip("\"'` ").strip()
|
||||
if not title:
|
||||
|
|
@ -317,6 +316,7 @@ class OclawGateway:
|
|||
),
|
||||
},
|
||||
]
|
||||
ensure_no_tool_or_embedded_image_payload(messages=messages, path="gateway.manager_select")
|
||||
resp = model.chat(messages, [], on_token=None)
|
||||
obj = self._parse_json_object(str(getattr(resp, "content", "") or ""))
|
||||
if not isinstance(obj, dict):
|
||||
|
|
@ -404,11 +404,9 @@ class OclawGateway:
|
|||
f"专家结果:\n{str(specialist_reply or '').strip()}\n\n"
|
||||
"要求:保持简洁、准确,不要暴露内部流程。"
|
||||
)
|
||||
resp = model.chat(
|
||||
[{"role": "system", "content": manager_context}, {"role": "user", "content": user_text}],
|
||||
[],
|
||||
on_token=on_token,
|
||||
)
|
||||
messages = [{"role": "system", "content": manager_context}, {"role": "user", "content": user_text}]
|
||||
ensure_no_tool_or_embedded_image_payload(messages=messages, path="gateway.manager_finalize")
|
||||
resp = model.chat(messages, [], on_token=on_token)
|
||||
final_text = str(getattr(resp, "content", "") or "").strip()
|
||||
return final_text or str(specialist_reply or "")
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import Any
|
|||
from oclaw.runtime.types import StandardMessage
|
||||
from oclaw.runtime.types import normalize_interaction_mode, normalize_requested_specialist
|
||||
from oclaw.prompts.loader import render_runtime_prompt
|
||||
from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -86,6 +87,7 @@ def _decide_llm_json(msg: StandardMessage, *, model: Any | None) -> RouterDecisi
|
|||
},
|
||||
{"role": "user", "content": user_block},
|
||||
]
|
||||
ensure_no_tool_or_embedded_image_payload(messages=messages, path="router.llm_json")
|
||||
resp = model.chat(messages, [], on_token=None)
|
||||
raw = str(getattr(resp, "content", "") or "")
|
||||
obj = _parse_router_json_object(raw)
|
||||
|
|
|
|||
144
runtime/skills/aiops-telecom/SKILL.md
Normal file
144
runtime/skills/aiops-telecom/SKILL.md
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
# AIOps-电信数通智能运维技能包
|
||||
|
||||
## 概述
|
||||
面向电信/数通网络的 AI 智能运维技能包。支持告警日志智能分析、配置错漏检查、故障模式学习与根因推荐。适用于华为、中兴、Cisco、Juniper 等主流数通设备。
|
||||
|
||||
## 能力说明
|
||||
|
||||
### 1. 告警日志智能分析
|
||||
|
||||
#### 1.1 输入格式支持
|
||||
- Excel/CSV 格式的告警清单(如网管导出的告警报表)
|
||||
- 纯文本格式的 syslog/告警日志
|
||||
- 设备 CLI 输出的告警信息
|
||||
|
||||
#### 1.2 分析维度
|
||||
|
||||
| 维度 | 说明 |
|
||||
|------|------|
|
||||
| **告警分级统计** | Critical / Major / Warning 级别分布及占比 |
|
||||
| **设备维度聚合** | 按设备(ME/NE)统计告警数量 Top N |
|
||||
| **告警类型归类** | 物理层/链路层/网络层/路由层/安全层分类 |
|
||||
| **时间维度分析** | 告警爆发时间窗口、频次趋势 |
|
||||
| **关联分析** | 同设备/同链路的多层告警关联,推测根因 |
|
||||
| **影响评估** | 评估受影响业务(L3VPN/Tunnel/Ethernet)范围 |
|
||||
|
||||
#### 1.3 告警类型知识库
|
||||
|
||||
##### 物理层告警
|
||||
| 告警名称 | 级别 | 含义 | 常见根因 | 推荐操作 |
|
||||
|----------|------|------|---------|---------|
|
||||
| Ethernet Physical LOS | Critical | 光口信号丢失 | 光纤断/松、光模块故障、对端设备断电 | 检查光纤链路、光功率、更换光模块 |
|
||||
| Ethernet Physical Laser Temperature | Major | 激光器温度越限 | 光模块老化、设备散热不良、环境温度过高 | 检查设备风扇/温度、更换光模块 |
|
||||
| Ethernet Physical Input Optical Power | Major | 接收光功率越限 | 光纤衰减过大、接口脏污、对端发光异常 | 清洁光纤接头、检查光功率预算 |
|
||||
| Ethernet Physical CRC Error | Major | CRC 误码越限 | 光纤质量差、接头污染、电磁干扰 | 清洁光纤、检查物理链路、更换尾纤 |
|
||||
| SmartGroup RX CRC Error | Major | 聚合口 CRC 误码 | 聚合成员链路质量问题 | 排查各成员链路、更换问题链路 |
|
||||
|
||||
##### 路由/MPLS层告警
|
||||
| 告警名称 | 级别 | 含义 | 常见根因 | 推荐操作 |
|
||||
|----------|------|------|---------|---------|
|
||||
| OSPF Neighbor Down | Major | OSPF 邻居中断 | 物理链路故障、Hello 超时、配置不匹配 | 检查邻居间链路、验证 OSPF 配置参数 |
|
||||
| RSVP LSP BFD Session Down | Major | MPLS TE隧道 BFD 检测失败 | 隧道途经链路故障、节点故障 | 检查隧道路径、确认中间节点状态 |
|
||||
| BGP Peer Flap | Major | BGP 邻居震荡 | 链路不稳、Keepalive 超时、策略变更 | 检查 BGP 配置、链路稳定性 |
|
||||
|
||||
##### 安全/控制面告警
|
||||
| 告警名称 | 级别 | 含义 | 常见根因 | 推荐操作 |
|
||||
|----------|------|------|---------|---------|
|
||||
| CPSBP 超阈值 | Warning | 控制面报文速率超阈值 | 攻击流量、广播风暴、配置过低 | 分析流量源、调整 CP 保护阈值 |
|
||||
| ARP IP冲突 | Warning | 检测到 IP 地址冲突 | 私接设备、IP 配置错误 | 定位冲突源、释放/更改 IP 地址 |
|
||||
|
||||
### 2. 配置错漏检查
|
||||
|
||||
#### 2.1 支持检查项
|
||||
|
||||
| 检查类别 | 典型检查项 |
|
||||
|----------|-----------|
|
||||
| **接口配置** | VLAN 配置一致性、MTU 匹配、描述规范、端口模式(access/trunk) |
|
||||
| **路由协议** | OSPF area 一致性、BGP AS 号核对、Route-map 逻辑、邻居配置完整性 |
|
||||
| **MPLS/TE** | LSP 配置完整性、隧道保护策略、FRR 配置 |
|
||||
| **安全策略** | ACL 规则匹配、控制面保护(CoPP)、端口安全、MAC 漂移检测 |
|
||||
| **高可用** | VRRP/HSRP 配置一致性、BFD 联动、链路聚合(LACP)配置 |
|
||||
| **QoS** | 队列策略、带宽限制、优先级映射 |
|
||||
|
||||
#### 2.2 分析模式
|
||||
- **配置比对**:新旧配置 diff,快速定位变更点
|
||||
- **合规检查**:基于标准模板检查配置是否符合规范
|
||||
- **逻辑验证**:检查配置逻辑矛盾(如 ACL 冗余/冲突、路由黑洞)
|
||||
|
||||
### 3. 故障模式学习 (self-learning)
|
||||
|
||||
系统会记录每次分析过程中的:
|
||||
- 告警 → 修复措施 对应关系
|
||||
- 配置错误 → 正确配置 修正方案
|
||||
- 经用户确认的根因分析结论
|
||||
|
||||
这些 learnings 会存入 `.learnings/` 目录,在后续分析中自动参考,越用越准确。
|
||||
|
||||
## 使用示例
|
||||
|
||||
### 示例1:告警日志分析
|
||||
```
|
||||
用户: "分析这份告警日志"
|
||||
助手: 按以下格式输出分析报告:
|
||||
|
||||
📊 告警概览
|
||||
- 总告警数:24条
|
||||
- Critical:2条 (8.3%) ⛔
|
||||
- Major:9条 (37.5%) ⚠️
|
||||
- Warning:13条 (54.2%) ⚡
|
||||
|
||||
🔴 Critical 告警详情
|
||||
1. [SMD-LIIR-EN1-Z20HS] ETPI LOS → 物理光口信号丢失
|
||||
→ 该设备下联 SMD-SBRS-EN1-Z20HS,状态 Unack
|
||||
→ 建议:立即检查光纤/光模块
|
||||
|
||||
🟠 Major 告警详情
|
||||
1. [BKL-UNB-AN1-ZM3SP] 光功率越限 → 接收光功率过低
|
||||
2. [MDN-PMKN-EN1-Z20HS] 激光器温度越限
|
||||
3. [PAD-KBU-AN1-ZM8S] CRC误码越限(物理口+聚合口)
|
||||
4. RSVP LSP BFD Session Down ×3条
|
||||
5. OSPF邻居中断 ×2条
|
||||
|
||||
🔄 关联分析发现
|
||||
- PBR-RPKU 与 PBR-PYSK 之间:OSPF邻居中断 + CPSBP告警
|
||||
→ 推测该链路存在物理层问题
|
||||
- GRO 站点多条 RSVP LSP BFD Down
|
||||
→ 可能为 GRO 节点设备问题或出局光缆中断
|
||||
|
||||
💡 根因推荐
|
||||
1. 优先处理 SMD-LIIR-EN1-Z20HS 的 LOS(Critical)
|
||||
2. 检查 PBR-RPKU ↔ PBR-PYSK 链路光功率和光模块
|
||||
3. 排查 GRO 站点汇聚设备状态
|
||||
```
|
||||
|
||||
### 示例2:配置检查
|
||||
```
|
||||
用户: "帮我检查这两份配置文件"
|
||||
助手:
|
||||
🔍 配置检查报告
|
||||
1. OSPF 配置检查 ✅
|
||||
- Area 一致性:匹配
|
||||
- Hello/Dead 间隔:一致
|
||||
- 网络类型:一致
|
||||
|
||||
2. 接口配置检查 ⚠️
|
||||
- [GE0/0/1] MTU 不匹配:本端 1500,对端 9000
|
||||
- [GE0/0/2] 描述缺失
|
||||
|
||||
3. BGP 配置检查 ❌
|
||||
- AS 号不匹配:本端 AS65001,对端 AS65002
|
||||
```
|
||||
|
||||
## 数据来源说明
|
||||
|
||||
本技能的知识库基于以下标准构建:
|
||||
- 华为 NE40E/ME60 系列告警手册
|
||||
- 中兴 ZXR10 系列告警与配置规范
|
||||
- 3GPP 管理面标准(IRP/Solution)
|
||||
- ITU-T 光传输标准
|
||||
- RFC 相关协议标准
|
||||
|
||||
## 局限性与注意事项
|
||||
- 本技能不直接连接设备执行命令,不做配置变更操作
|
||||
- 分析结果基于提供的日志/配置数据,用户需确认数据准确性
|
||||
- 推荐操作为参考建议,重大操作需人工复核
|
||||
|
|
@ -11,6 +11,7 @@ from oclaw.runtime.agent_core_run import AgentCoreRunInput, run_agent_core
|
|||
from oclaw.runtime.memory_stage import build_memory_context
|
||||
from oclaw.runtime.relay_pointer import build_acp_relay_result, validate_relay_share_envelope
|
||||
from oclaw.runtime.types import StandardMessage
|
||||
from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
_THREAD: threading.Thread | None = None
|
||||
|
|
@ -120,11 +121,9 @@ def _maybe_generate_title_on_third_round(*, store: Any, msg: StandardMessage, mo
|
|||
else "仅基于以下用户正文生成简短会话标题。请使用对话内容主体语言命名。"
|
||||
"只返回标题文本,不要引号,不要markdown,最多18个字。"
|
||||
)
|
||||
resp = model.chat(
|
||||
[{"role": "system", "content": sys}, {"role": "user", "content": body}],
|
||||
[],
|
||||
on_token=None,
|
||||
)
|
||||
messages = [{"role": "system", "content": sys}, {"role": "user", "content": body}]
|
||||
ensure_no_tool_or_embedded_image_payload(messages=messages, path="worker.auto_title")
|
||||
resp = model.chat(messages, [], on_token=None)
|
||||
title = str(getattr(resp, "content", "") or "").strip().replace("\n", " ")
|
||||
title = title.strip("\"'` ").strip()
|
||||
if not title:
|
||||
|
|
|
|||
44
tests/test_media_redact.py
Normal file
44
tests/test_media_redact.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
from oclaw.runtime.chat.media_redact import ingest_embedded_image_blobs_as_refs, redact_embedded_image_blobs
|
||||
|
||||
|
||||
def test_redact_nested_mcp_image_block() -> None:
|
||||
big = "/9j/" + "a" * 800
|
||||
obj = {
|
||||
"ok": True,
|
||||
"result": {"content": [{"type": "image", "mime": "image/jpeg", "data": big}]},
|
||||
}
|
||||
out = redact_embedded_image_blobs(obj)
|
||||
assert out["result"]["content"][0].get("_image_payload_redacted") is True
|
||||
assert "data" not in out["result"]["content"][0]
|
||||
assert isinstance(out["result"]["content"][0].get("_redacted_payload_chars"), int)
|
||||
|
||||
|
||||
def test_redact_keeps_small_data_field() -> None:
|
||||
obj = {"type": "image", "mime": "image/png", "data": "abc"}
|
||||
assert redact_embedded_image_blobs(obj) == obj
|
||||
|
||||
|
||||
def test_ingest_embedded_image_blob_as_ref(tmp_path) -> None:
|
||||
raw = base64.b64encode(b"png-bytes").decode("ascii")
|
||||
obj = {"result": {"content": [{"type": "image", "mime": "image/png", "data": raw, "name": "x.png"}]}}
|
||||
out, refs = ingest_embedded_image_blobs_as_refs(obj, root_dir=str(tmp_path), filename_prefix="unit")
|
||||
block = out["result"]["content"][0]
|
||||
assert block["type"] == "image_ref"
|
||||
assert str(block.get("attachment_id") or "")
|
||||
assert "data" not in block
|
||||
assert refs and refs[0]["attachment_id"] == block["attachment_id"]
|
||||
|
||||
|
||||
def test_ingest_embedded_binary_blob_as_binary_ref(tmp_path) -> None:
|
||||
raw = base64.b64encode(b"%PDF-1.4-fake").decode("ascii")
|
||||
obj = {"result": {"content": [{"type": "file", "mime": "application/pdf", "base64": raw, "name": "a.pdf"}]}}
|
||||
out, refs = ingest_embedded_image_blobs_as_refs(obj, root_dir=str(tmp_path), filename_prefix="unit")
|
||||
block = out["result"]["content"][0]
|
||||
assert block["type"] == "binary_ref"
|
||||
assert str(block.get("attachment_id") or "")
|
||||
assert "base64" not in block
|
||||
assert refs and refs[0]["attachment_id"] == block["attachment_id"]
|
||||
46
tests/test_model_path_audit.py
Normal file
46
tests/test_model_path_audit.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from oclaw.runtime.chat.model_path_audit import ensure_no_tool_or_embedded_image_payload
|
||||
|
||||
|
||||
def test_audit_accepts_plain_system_user_messages() -> None:
|
||||
ensure_no_tool_or_embedded_image_payload(
|
||||
path="unit.ok",
|
||||
messages=[
|
||||
{"role": "system", "content": "sys"},
|
||||
{"role": "user", "content": "hello"},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_audit_degrades_tool_role() -> None:
|
||||
msgs = [
|
||||
{"role": "assistant", "content": "a"},
|
||||
{"role": "tool", "content": '{"ok":true}'},
|
||||
]
|
||||
ensure_no_tool_or_embedded_image_payload(path="unit.tool", messages=msgs)
|
||||
assert msgs[1]["role"] == "assistant"
|
||||
assert "omitted" in str(msgs[1]["content"] or "")
|
||||
|
||||
|
||||
def test_audit_degrades_embedded_image_payload() -> None:
|
||||
msgs = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "x"},
|
||||
{"type": "input_image", "image_base64": "a" * 600, "mime": "image/png"},
|
||||
],
|
||||
}
|
||||
]
|
||||
ensure_no_tool_or_embedded_image_payload(path="unit.image", messages=msgs)
|
||||
c = msgs[0]["content"]
|
||||
assert isinstance(c, list)
|
||||
assert any(isinstance(x, dict) and str(x.get("type") or "") == "text" and "omitted" in str(x.get("text") or "") for x in c)
|
||||
|
||||
|
||||
def test_audit_degrades_large_base64_like_plain_text() -> None:
|
||||
payload = "A" * 700
|
||||
msgs = [{"role": "user", "content": payload}]
|
||||
ensure_no_tool_or_embedded_image_payload(path="unit.b64_text", messages=msgs)
|
||||
assert "omitted" in str(msgs[0]["content"] or "")
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from oclaw.runtime.direct_loop import _OCLAW_TOOL_RESULT_HARD_CAP_CHARS, _build_model_context
|
||||
|
|
@ -78,3 +79,42 @@ def test_oclaw_tool_result_context_guard_skips_active_turn_tool_messages(tmp_pat
|
|||
raw = str(tool_msgs[-1].get("content") or "")
|
||||
assert "_tool_result_guarded" not in raw
|
||||
|
||||
|
||||
def test_guard_redacts_mcp_nested_image_for_non_active_turn(tmp_path: Path) -> None:
|
||||
store = SqliteStore(str(tmp_path / "ops.sqlite"))
|
||||
sess = store.create_session("t")
|
||||
past_turn = "turn-old"
|
||||
blob = "/9j/" + "a" * 1200
|
||||
body = {"ok": True, "result": {"content": [{"type": "image", "mime": "image/jpeg", "data": blob}]}}
|
||||
store.add_message(
|
||||
session_id=sess.id,
|
||||
role="assistant",
|
||||
content="",
|
||||
tool_calls=[{"id": "c_hist", "name": "mcp", "arguments": {}}],
|
||||
turn_uuid=past_turn,
|
||||
)
|
||||
store.add_message(
|
||||
session_id=sess.id,
|
||||
role="tool",
|
||||
content=json.dumps(body, ensure_ascii=False),
|
||||
tool_calls={"tool_call_id": "c_hist", "name": "mcp", "assistant_message_id": 1},
|
||||
turn_uuid=past_turn,
|
||||
)
|
||||
msgs = _build_model_context(
|
||||
store=store,
|
||||
session_id=sess.id,
|
||||
max_messages=50,
|
||||
system_prompt="sys",
|
||||
model=RuleBasedChatModel(),
|
||||
lang="zh",
|
||||
memory_context=None,
|
||||
trace_id="t1",
|
||||
parent_span_id=None,
|
||||
active_turn_uuid="different-active-turn",
|
||||
)
|
||||
tm = next(m for m in msgs if m.get("role") == "tool")
|
||||
inner = json.loads(str(tm.get("content") or ""))
|
||||
block = inner["result"]["content"][0]
|
||||
assert block.get("_image_payload_redacted") is True
|
||||
assert "data" not in block
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -89,6 +90,94 @@ def test_repeated_tool_results_are_compacted_in_history(tmp_path: Path) -> None:
|
|||
assert "audit_note" in payloads[2]
|
||||
|
||||
|
||||
def test_tool_result_image_blob_persisted_as_attachment_ref(tmp_path: Path) -> None:
|
||||
store = SqliteStore(str(tmp_path / "img.sqlite"))
|
||||
sess = store.create_session("t")
|
||||
|
||||
def _handler(_args):
|
||||
raw = base64.b64encode(b"\x89PNGtest-image").decode("ascii")
|
||||
return {
|
||||
"ok": True,
|
||||
"result": {"content": [{"type": "image", "mime": "image/png", "data": raw, "name": "mcp.png"}]},
|
||||
}
|
||||
|
||||
reg = ToolRegistry(
|
||||
[
|
||||
ToolSpec(
|
||||
name="mcp_image_tool",
|
||||
description="returns image payload",
|
||||
parameters={"type": "object", "properties": {}},
|
||||
handler=_handler,
|
||||
read_only=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
tool_uses = [LLMToolCall(id="c1", name="mcp_image_tool", arguments={})]
|
||||
ToolExecutor().execute_tool_uses(
|
||||
ctx=ToolExecutionContext(store=store, tools=reg, session_id=sess.id, turn_uuid="turn-img"),
|
||||
assistant_msg_id=1,
|
||||
tool_uses=tool_uses,
|
||||
)
|
||||
rows = store.get_messages(session_id=sess.id, limit=20)
|
||||
tool_rows = [m for m in rows if str(getattr(m, "role", "") or "") == "tool"]
|
||||
assert len(tool_rows) == 1
|
||||
payload = json.loads(str(getattr(tool_rows[0], "content", "") or "{}"))
|
||||
blk = (((payload.get("result") or {}).get("content") or [{}])[0]) if isinstance(payload, dict) else {}
|
||||
assert isinstance(blk, dict)
|
||||
assert str(blk.get("type") or "") == "image_ref"
|
||||
assert str(blk.get("attachment_id") or "")
|
||||
assert "data" not in blk
|
||||
atts = json.loads(str(getattr(tool_rows[0], "attachments", "") or "[]"))
|
||||
assert isinstance(atts, list) and atts
|
||||
assert str((atts[0] or {}).get("type") or "") == "image_ref"
|
||||
assert str((atts[0] or {}).get("attachment_id") or "")
|
||||
logs = store.get_tool_logs(sess.id, limit=10)
|
||||
assert logs and isinstance(logs[0], dict)
|
||||
body = logs[0].get("result") or {}
|
||||
blk2 = (((body.get("result") or {}).get("content") or [{}])[0]) if isinstance(body, dict) else {}
|
||||
assert isinstance(blk2, dict)
|
||||
assert str(blk2.get("type") or "") == "image_ref"
|
||||
assert "data" not in blk2
|
||||
|
||||
|
||||
def test_tool_result_non_image_base64_persisted_as_binary_ref(tmp_path: Path) -> None:
|
||||
store = SqliteStore(str(tmp_path / "bin.sqlite"))
|
||||
sess = store.create_session("t")
|
||||
|
||||
def _handler(_args):
|
||||
raw = base64.b64encode(b"PK\x03\x04fake-zip-bytes").decode("ascii")
|
||||
return {"ok": True, "result": {"content": [{"type": "file", "mime": "application/zip", "base64": raw, "name": "a.zip"}]}}
|
||||
|
||||
reg = ToolRegistry(
|
||||
[
|
||||
ToolSpec(
|
||||
name="mcp_file_tool",
|
||||
description="returns file payload",
|
||||
parameters={"type": "object", "properties": {}},
|
||||
handler=_handler,
|
||||
read_only=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
ToolExecutor().execute_tool_uses(
|
||||
ctx=ToolExecutionContext(store=store, tools=reg, session_id=sess.id, turn_uuid="turn-bin"),
|
||||
assistant_msg_id=1,
|
||||
tool_uses=[LLMToolCall(id="c1", name="mcp_file_tool", arguments={})],
|
||||
)
|
||||
rows = store.get_messages(session_id=sess.id, limit=20)
|
||||
tool_rows = [m for m in rows if str(getattr(m, "role", "") or "") == "tool"]
|
||||
assert len(tool_rows) == 1
|
||||
payload = json.loads(str(getattr(tool_rows[0], "content", "") or "{}"))
|
||||
blk = (((payload.get("result") or {}).get("content") or [{}])[0]) if isinstance(payload, dict) else {}
|
||||
assert isinstance(blk, dict)
|
||||
assert str(blk.get("type") or "") == "binary_ref"
|
||||
assert str(blk.get("attachment_id") or "")
|
||||
assert "base64" not in blk
|
||||
atts = json.loads(str(getattr(tool_rows[0], "attachments", "") or "[]"))
|
||||
assert isinstance(atts, list) and atts
|
||||
assert str((atts[0] or {}).get("type") or "") == "binary_ref"
|
||||
|
||||
|
||||
def test_repeated_non_sql_tools_are_not_compacted(tmp_path: Path) -> None:
|
||||
store = SqliteStore(str(tmp_path / "g3.sqlite"))
|
||||
sess = store.create_session("t")
|
||||
|
|
@ -429,6 +518,11 @@ def test_tool_result_image_payload_persisted_as_attachments(tmp_path: Path) -> N
|
|||
tool_rows = [m for m in rows if str(getattr(m, "role", "") or "") == "tool"]
|
||||
assert len(tool_rows) == 1
|
||||
attachments = json.loads(str(getattr(tool_rows[0], "attachments", "") or "[]"))
|
||||
assert any(str(a.get("type") or "") == "image" and str(a.get("data") or "") == "YWJj" for a in attachments)
|
||||
assert any(str(a.get("type") or "") == "image_ref" and str(a.get("attachment_id") or "") for a in attachments)
|
||||
assert any(str(a.get("type") or "") == "image_url" and str(a.get("url") or "").endswith("/a.png") for a in attachments)
|
||||
body = json.loads(str(getattr(tool_rows[0], "content", "") or "{}"))
|
||||
content_items = ((body.get("result") or {}).get("content") or []) if isinstance(body, dict) else []
|
||||
image_block = next((x for x in content_items if isinstance(x, dict) and str(x.get("type") or "") == "image_ref"), {})
|
||||
assert str(image_block.get("attachment_id") or "")
|
||||
assert "data" not in image_block
|
||||
|
||||
|
|
|
|||
|
|
@ -138,7 +138,13 @@ def test_signature_metadata_not_replayed_by_default_for_non_whitelist_model() ->
|
|||
ensure_ascii=False,
|
||||
),
|
||||
event_type="tool_call",
|
||||
)
|
||||
),
|
||||
_Msg(
|
||||
"tool",
|
||||
json.dumps({"ok": True}),
|
||||
tool_calls=json.dumps({"tool_call_id": "call_1", "name": "t"}, ensure_ascii=False),
|
||||
event_type="tool_result",
|
||||
),
|
||||
]
|
||||
msgs = build_llm_messages(store_messages=rows, system_prompt="s", model=model, lang="zh")
|
||||
assistant = [m for m in msgs if m.get("role") == "assistant"][0]
|
||||
|
|
@ -158,7 +164,13 @@ def test_signature_metadata_can_be_forced_on_via_env(monkeypatch) -> None:
|
|||
ensure_ascii=False,
|
||||
),
|
||||
event_type="tool_call",
|
||||
)
|
||||
),
|
||||
_Msg(
|
||||
"tool",
|
||||
json.dumps({"ok": True}),
|
||||
tool_calls=json.dumps({"tool_call_id": "call_1", "name": "t"}, ensure_ascii=False),
|
||||
event_type="tool_result",
|
||||
),
|
||||
]
|
||||
msgs = build_llm_messages(store_messages=rows, system_prompt="s", model=model, lang="zh")
|
||||
assistant = [m for m in msgs if m.get("role") == "assistant"][0]
|
||||
|
|
|
|||
30
tests/test_tool_runtime_attachments.py
Normal file
30
tests/test_tool_runtime_attachments.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from oclaw.runtime.chat.tool_runtime import _attachments_from_tool_result
|
||||
|
||||
|
||||
def test_attachments_from_tool_result_preserves_non_image_ref_types() -> None:
|
||||
result = {
|
||||
"attachments": [
|
||||
{
|
||||
"type": "text_ref",
|
||||
"attachment_id": "att-text-1",
|
||||
"mime": "text/plain",
|
||||
"name": "a.txt",
|
||||
"bytes": 12,
|
||||
},
|
||||
{
|
||||
"attachment_id": "att-video-1",
|
||||
"mime": "video/mp4",
|
||||
"name": "a.mp4",
|
||||
"bytes": 1024,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
out = _attachments_from_tool_result(result)
|
||||
by_id = {str(x.get("attachment_id")): x for x in out}
|
||||
|
||||
assert by_id["att-text-1"]["type"] == "text_ref"
|
||||
assert by_id["att-video-1"]["type"] == "video_ref"
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue