diff --git a/interfaces/admin/static/chat.js b/interfaces/admin/static/chat.js index c2e662dc..ac956296 100644 --- a/interfaces/admin/static/chat.js +++ b/interfaces/admin/static/chat.js @@ -416,7 +416,7 @@ function _appendAssistantTextSegments(inner, rawText, collapsedItems) { } } -function _buildAggregatedAssistantBubble(tsIso, items) { +async function _buildAggregatedAssistantBubble(tsIso, items) { const inner = el("div", { class: "chat-msg chat-msg--assistant chat-msg--rich" }); const collapsedItems = []; const _inlineImageHtmlFromAttachments = (raw) => { @@ -453,8 +453,10 @@ function _buildAggregatedAssistantBubble(tsIso, items) { if (adminChatShowToolOutput) collapsedItems.push({ title: _toolSummaryTitle("tool_call"), text }); } else if (kind === "tool_result") { if (adminChatShowToolOutput) collapsedItems.push({ title: _toolSummaryTitle("tool"), text }); - const inlineImages = _inlineImageHtmlFromAttachments(it && it.attachments); - if (inlineImages) inner.appendChild(el("div", { class: "chat-msg__md", html: inlineImages })); + // 支持两类附件：base64（image/input_image）和引用型（image_ref）。 + // image_ref 需要异步拉取 blob，因此改为走 renderAttachmentsEl()。 + const attsEl = await renderAttachmentsEl(it && it.attachments); + if (attsEl) inner.appendChild(attsEl); } else { inner.appendChild(el("div", { class: "chat-msg__md", html: renderMarkdownHtml(text) })); } @@ -2063,7 +2065,7 @@ async function appendMessageRow(messagesEl, m, options = {}) { const content = String(m.content || ""); const ts = m.timestamp != null ? m.timestamp : ""; const bubble = Array.isArray(m._items) - ? _buildAggregatedAssistantBubble(ts, m._items) + ? await _buildAggregatedAssistantBubble(ts, m._items) : await buildMessageBubble(role, content, ts); // For aggregated assistant bubbles, attachments should be rendered inline // at tool_result positions, not appended at bubble tail. diff --git a/runtime/application/gateway/inbound_service.py b/runtime/application/gateway/inbound_service.py index e8311506..04fa009f 100644 --- a/runtime/application/gateway/inbound_service.py +++ b/runtime/application/gateway/inbound_service.py @@ -319,6 +319,132 @@ def _parse_message_attachments(raw: Any) -> list[dict[str, Any]]: return [x for x in obj if isinstance(x, dict)] +def _first_attachment_id(atts: list[dict[str, Any]]) -> str: + for a in atts or []: + if not isinstance(a, dict): + continue + aid = str(a.get("attachment_id") or a.get("attachmentId") or "").strip().lower() + if aid: + return aid + return "" + + +def _collect_recent_tool_attachments(*, store: Any, session_id: str) -> list[dict[str, Any]]: + """Fallback for channel delivery: if assistant attachments are missing, reuse recent tool attachments. + + This is intentionally conservative: only returns attachments when they look like media or refs. + """ + sid = str(session_id or "").strip() + if not sid: + return [] + try: + rows = store.get_messages(session_id=sid, limit=80) + except Exception: + rows = [] + # Prefer newest tool attachments. + for row in reversed(list(rows or [])): + role = str(getattr(row, "role", "") or "").strip().lower() + if role != "tool": + continue + atts = _parse_message_attachments(getattr(row, "attachments", None)) + if not atts: + continue + # Only accept common media/ref shapes. + ok = False + for a in atts: + if not isinstance(a, dict): + continue + t = str(a.get("type") or "").strip().lower() + if t in {"image_ref", "video_ref", "binary_ref", "text_ref", "image", "input_image", "image_url"}: + ok = True + break + if ok: + return atts + return [] + + +def _maybe_add_media_path_for_wechat_reply(reply: dict[str, Any]) -> None: + """For wechat/weixin sidecar, prefer a local file path for media send.""" + try: + from oclaw.platform.files.attachment_assets import AttachmentAssetStore + except Exception: + return + if not isinstance(reply, dict): + return + if str(reply.get("media_path") or reply.get("mediaPath") or "").strip(): + return + if str(reply.get("media_url") or reply.get("mediaUrl") or "").strip(): + return + atts = reply.get("attachments") if isinstance(reply.get("attachments"), list) else [] + aid = _first_attachment_id([a for a in atts if isinstance(a, dict)]) + if not aid: + return + p = AttachmentAssetStore().get_local_path(aid) + if not p: + return + # use the existing runner field name + reply["media_path"] = str(p) + + +def _maybe_expand_reply_attachments_for_channel(reply: dict[str, Any]) -> None: + """For chat sidecar delivery, convert attachment refs to base64 payloads. + + The runner supports base64 keys like data_base64/media_base64/image_base64/data. + This avoids relying on sidecar being able to access gateway-local disk paths. + """ + if not isinstance(reply, dict): + return + raw = reply.get("attachments") + if not isinstance(raw, list) or not raw: + return + # If already has base64 payloads, keep as-is. + for a in raw: + if not isinstance(a, dict): + continue + if str( + a.get("data_base64") + or a.get("media_base64") + or a.get("image_base64") + or a.get("video_base64") + or a.get("audio_base64") + or a.get("data") + or "" + ).strip(): + return + # Expand first attachment ref only (avoid large payload). + a0 = next((a for a in raw if isinstance(a, dict)), None) + if not isinstance(a0, dict): + return + aid = str(a0.get("attachment_id") or a0.get("attachmentId") or "").strip().lower() + if not aid: + return + try: + import base64 + + from oclaw.platform.files.attachment_assets import AttachmentAssetStore + except Exception: + return + ast = AttachmentAssetStore() + blob, meta = ast.load_bytes(aid) + if not blob: + return + # Conservative cap to avoid blowing up channel payload. + max_bytes = 8 * 1024 * 1024 + if len(blob) > max_bytes: + return + mime = (meta.mime if meta else "") or str(a0.get("mime") or a0.get("mime_type") or "application/octet-stream") + name = (meta.name if meta else "") or str(a0.get("name") or "attachment") + b64 = base64.b64encode(blob).decode("ascii") + reply["attachments"] = [ + { + "name": name, + "mime": mime, + "media_type": mime, + "data_base64": b64, + } + ] + + def _collect_reply_attachments_from_history(*, store: Any, session_id: str, reply_text: str) -> list[dict[str, Any]]: sid = str(session_id or "").strip() if not sid: @@ -573,6 +699,9 @@ def process_inbound_payload(payload: dict[str, Any]) -> dict[str, Any]: if adapter is not None: replies = [adapter.format_outbound(OutboundMessage(external_chat_id=inbound.external_chat_id, text=reply))] else: + if not reply_attachments: + # If assistant didn't persist attachments, fall back to recent tool-produced media. + reply_attachments = _collect_recent_tool_attachments(store=store, session_id=str(session_id)) replies = [ { "channel": inbound.channel, @@ -582,6 +711,14 @@ def process_inbound_payload(payload: dict[str, Any]) -> dict[str, Any]: "metadata": {}, } ] + # For wechat/weixin sidecar delivery, add a local media_path when reply attachments refer to attachment_id. + for r in replies or []: + if not isinstance(r, dict): + continue + ch = str(r.get("channel") or inbound.channel or "").strip().lower() + if ch in {"wechat", "weixin", "whatsapp"}: + _maybe_expand_reply_attachments_for_channel(r) + _maybe_add_media_path_for_wechat_reply(r) out = {"ok": True, "replies": replies} return out diff --git a/tests/test_inbound_service_reply_suppress.py b/tests/test_inbound_service_reply_suppress.py index 60c4ef11..b0f14dd9 100644 --- a/tests/test_inbound_service_reply_suppress.py +++ b/tests/test_inbound_service_reply_suppress.py @@ -4,6 +4,9 @@ from dataclasses import dataclass from oclaw.runtime.application.gateway.inbound_service import ( _collect_reply_attachments_from_history, + _collect_recent_tool_attachments, + _maybe_add_media_path_for_wechat_reply, + _maybe_expand_reply_attachments_for_channel, _parse_message_attachments, _should_suppress_channel_reply, ) @@ -51,3 +54,75 @@ def test_collect_reply_attachments_prefers_matching_assistant_text() -> None: assert len(out) == 1 assert out[0].get("attachment_id") == "new" + +def test_collect_recent_tool_attachments_falls_back_to_tool_media() -> None: + rows = [ + _Row(role="assistant", content="x", attachments=None), + _Row(role="tool", content="{}", attachments='[{"type":"image_ref","attachment_id":"a1"}]'), + ] + out = _collect_recent_tool_attachments(store=_FakeStore(rows), session_id="s1") + assert len(out) == 1 + assert out[0].get("attachment_id") == "a1" + + +def test_maybe_add_media_path_for_wechat_reply_sets_media_path(monkeypatch) -> None: + # Avoid touching disk: stub AttachmentAssetStore.get_local_path. + from pathlib import Path + + def _fake_get_local_path(self, attachment_id: str): # noqa: ANN001 + assert attachment_id == "a1" + return Path("D:/tmp/fake.png") + + monkeypatch.setattr( + "oclaw.platform.files.attachment_assets.AttachmentAssetStore.get_local_path", + _fake_get_local_path, + ) + r = {"channel": "wechat", "text": "hi", "attachments": [{"type": "image_ref", "attachment_id": "a1"}]} + _maybe_add_media_path_for_wechat_reply(r) + assert r.get("media_path") in {"D:/tmp/fake.png", "D:\\tmp\\fake.png"} + + +def test_maybe_expand_reply_attachments_for_channel_converts_ref_to_base64(monkeypatch) -> None: + import base64 + + class _Meta: + mime = "image/png" + name = "x.png" + + def _fake_load_bytes(self, attachment_id: str): # noqa: ANN001 + assert attachment_id == "a1" + return b"abc", _Meta() + + monkeypatch.setattr( + "oclaw.platform.files.attachment_assets.AttachmentAssetStore.load_bytes", + _fake_load_bytes, + ) + r = {"channel": "wechat", "attachments": [{"type": "image_ref", "attachment_id": "a1"}]} + _maybe_expand_reply_attachments_for_channel(r) + out = r.get("attachments") + assert isinstance(out, list) and len(out) == 1 + assert out[0].get("data_base64") == base64.b64encode(b"abc").decode("ascii") + assert out[0].get("mime") == "image/png" + + +def test_maybe_expand_reply_attachments_for_channel_works_for_whatsapp(monkeypatch) -> None: + import base64 + + class _Meta: + mime = "image/png" + name = "wa.png" + + def _fake_load_bytes(self, attachment_id: str): # noqa: ANN001 + assert attachment_id == "wa1" + return b"wa", _Meta() + + monkeypatch.setattr( + "oclaw.platform.files.attachment_assets.AttachmentAssetStore.load_bytes", + _fake_load_bytes, + ) + r = {"channel": "whatsapp", "attachments": [{"type": "image_ref", "attachment_id": "wa1"}]} + _maybe_expand_reply_attachments_for_channel(r) + out = r.get("attachments") + assert isinstance(out, list) and len(out) == 1 + assert out[0].get("data_base64") == base64.b64encode(b"wa").decode("ascii") +