diff --git a/runtime/application/gateway/inbound_service.py b/runtime/application/gateway/inbound_service.py index 66477953..4be5ce81 100644 --- a/runtime/application/gateway/inbound_service.py +++ b/runtime/application/gateway/inbound_service.py @@ -473,6 +473,7 @@ def _channel_attachments_for_gateway(raw: list[dict[str, Any]]) -> list[dict[str from pathlib import Path from svc.files.attachment_assets import AttachmentAssetStore + from svc.files.file_attachments import expand_attachment_ref, process_file_data out: list[dict[str, Any]] = [] ast = AttachmentAssetStore() @@ -481,8 +482,7 @@ def _channel_attachments_for_gateway(raw: list[dict[str, Any]]) -> list[dict[str continue aid = str(a.get("attachment_id") or a.get("attachmentId") or "").strip().lower() if aid: - t = str(a.get("type") or "").strip().lower() or "binary_ref" - out.append({"type": t, "attachment_id": aid}) + out.extend(expand_attachment_ref(a)) continue lp = str(a.get("local_path") or a.get("media_path") or "").strip() if not lp: @@ -495,7 +495,15 @@ def _channel_attachments_for_gateway(raw: list[dict[str, Any]]) -> list[dict[str if not mime: mime = "application/octet-stream" try: - meta = ast.save_bytes(p.read_bytes(), filename=p.name, mime=mime) + data = p.read_bytes() + except Exception: + continue + got = process_file_data(p.name, data) + if got: + out.extend(got) + continue + try: + meta = ast.save_bytes(data, filename=p.name, mime=mime) except Exception: continue if kind == "image" or mime.startswith("image/"): @@ -503,7 +511,7 @@ def _channel_attachments_for_gateway(raw: list[dict[str, Any]]) -> list[dict[str elif kind == "video" or mime.startswith("video/"): out.append({"type": "video_ref", "attachment_id": meta.attachment_id}) else: - out.append({"type": "binary_ref", "attachment_id": meta.attachment_id}) + out.append({"type": "binary_ref", "attachment_id": meta.attachment_id, "name": meta.name, "mime": meta.mime}) return out @@ -548,10 +556,16 @@ def _rows_since_last_user_message(rows: list[Any]) -> list[Any]: return list(rows[last_user_idx + 1 :]) +_CHANNEL_DELIVERABLE_ATTACHMENT_TYPES = frozenset( + {"image_ref", "video_ref", "image", "input_image", "image_url"} +) + + def _collect_recent_tool_attachments(*, store: Any, session_id: str) -> list[dict[str, Any]]: """Fallback for channel delivery: reuse tool media produced during the current user turn only. Avoids re-sending images from earlier conversation turns when the latest assistant row has no attachments. + Only visual outbound media is eligible — not text_ref/binary_ref from read/query tools. """ sid = str(session_id or "").strip() if not sid: @@ -569,15 +583,16 @@ def _collect_recent_tool_attachments(*, store: Any, session_id: str) -> list[dic if not atts: continue ok = False + deliverable: list[dict[str, Any]] = [] for a in atts: if not isinstance(a, dict): continue t = str(a.get("type") or "").strip().lower() - if t in {"image_ref", "video_ref", "binary_ref", "text_ref", "image", "input_image", "image_url"}: + if t in _CHANNEL_DELIVERABLE_ATTACHMENT_TYPES: ok = True - break + deliverable.append(a) if ok: - return atts + return deliverable return [] diff --git a/runtime/chat/agent_messages.py b/runtime/chat/agent_messages.py index a0c9e6dc..f3ba61fa 100644 --- a/runtime/chat/agent_messages.py +++ b/runtime/chat/agent_messages.py @@ -17,7 +17,8 @@ from svc.llm.chat_models import _normalize_image_b64_payload, gemini_openai_comp from runtime.chat.media_redact import redact_embedded_image_blobs from runtime.chat.tool_runtime import tool_llm_message_max_chars, truncate_tool_result_for_llm_messages from runtime.prompt_templates import render_prompt -from svc.files.attachment_assets import attachment_id_to_data_url +from svc.files.attachment_assets import attachment_id_to_data_url, AttachmentAssetStore +from svc.files.file_attachments import expand_attachment_ref from runtime.relay_pointer import parse_pointer_uri logger = logging.getLogger(__name__) @@ -316,11 +317,24 @@ def build_llm_messages( except Exception: attachments = [] + expand_user_image_for_model = bool(i == last_user_msg_idx) + if expand_user_image_for_model: + expanded_atts: list[dict[str, Any]] = [] + for att in attachments or []: + if not isinstance(att, dict): + continue + if str(att.get("type") or "").strip().lower() == "binary_ref": + subs = expand_attachment_ref(att) + if subs and any(str(s.get("type") or "") != "binary_ref" for s in subs): + expanded_atts.extend(subs) + continue + expanded_atts.append(att) + attachments = expanded_atts + for att in attachments or []: if not isinstance(att, dict): continue att_type = att.get("type") - expand_user_image_for_model = bool(i == last_user_msg_idx) if att_type in ("image", "input_image"): if expand_user_image_for_model: b64 = _normalize_image_b64_payload(att.get("image_base64") or att.get("data")) @@ -459,6 +473,29 @@ def build_llm_messages( meta_line += f"\n- bytes: {sz}" meta_line += "\n- tools: query_video_attachment" content_list.append({"type": "text", "text": meta_line}) + elif att_type == "binary_ref": + aid = str(att.get("attachment_id") or "").strip() + name = str(att.get("name") or "file") + mime = str(att.get("mime") or "application/octet-stream") + sz = att.get("bytes") + if aid and (name == "file" or not mime or mime == "application/octet-stream"): + try: + meta = AttachmentAssetStore().get_meta(aid) + if meta: + if name == "file": + name = str(meta.name or name) + if not mime or mime == "application/octet-stream": + mime = str(meta.mime or mime) + if sz is None: + sz = meta.bytes + except Exception: + pass + meta_line = f"[BinaryAttachment]\n- name: {name}\n- mime: {mime}\n- attachment_id: {aid}" + if sz: + meta_line += f"\n- bytes: {sz}" + meta_line += "\n- tools: attachment_local_url" + meta_line += "\n- note: user uploaded a file; resolve or analyze it via attachment_id." + content_list.append({"type": "text", "text": meta_line}) elif att_type == "relay_pointer": p_uri = str(att.get("pointer_uri") or "").strip() if not p_uri: diff --git a/svc/files/file_attachments.py b/svc/files/file_attachments.py index f8831062..625202cd 100644 --- a/svc/files/file_attachments.py +++ b/svc/files/file_attachments.py @@ -373,6 +373,47 @@ def process_zip(zip_data: bytes, *, _depth: int = 0) -> list[dict[str, Any]]: ) +def expand_attachment_ref( + att: dict[str, Any], + *, + data: bytes | None = None, + name: str | None = None, +) -> list[dict[str, Any]]: + """Expand ``binary_ref`` (or raw bytes) into parsed attachment dicts for the agent.""" + if not isinstance(att, dict): + return [] + t = str(att.get("type") or "").strip().lower() + if t and t not in {"binary_ref"}: + return [att] + aid = str(att.get("attachment_id") or att.get("attachmentId") or "").strip().lower() + meta = None + if data is None and aid: + blob, meta = AttachmentAssetStore().load_bytes(aid) + if not blob: + return [att] if t == "binary_ref" else [] + data = blob + name = name or str(getattr(meta, "name", "") or "file") + if data is None: + return [att] if t == "binary_ref" else [] + fname = str(name or att.get("name") or getattr(meta, "name", "") or "file") + got = process_file_data(fname, data) + if got: + return got + if aid: + if meta is None: + meta = AttachmentAssetStore().get_meta(aid) + return [ + { + "type": "binary_ref", + "attachment_id": aid, + "name": fname, + "mime": str(att.get("mime") or getattr(meta, "mime", "") or "application/octet-stream"), + "bytes": att.get("bytes") if att.get("bytes") is not None else getattr(meta, "bytes", None), + } + ] + return [] + + def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[dict[str, Any]]: ext = name.split(".")[-1].lower() if "." in name else "" attachments: list[dict[str, Any]] = [] @@ -773,4 +814,4 @@ def process_file_data(name: str, data: bytes, *, _zip_depth: int = 0) -> list[di return attachments -__all__ = ["process_zip", "process_file_data", "clear_attachment_limits_cache"] +__all__ = ["process_zip", "process_file_data", "expand_attachment_ref", "clear_attachment_limits_cache"] diff --git a/tests/test_binary_ref_attachments.py b/tests/test_binary_ref_attachments.py new file mode 100644 index 00000000..5239a10d --- /dev/null +++ b/tests/test_binary_ref_attachments.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from runtime.application.gateway.inbound_service import _channel_attachments_for_gateway +from runtime.chat.agent_messages import build_llm_messages +from svc.files.attachment_assets import AttachmentAssetStore +from svc.files.file_attachments import expand_attachment_ref +from svc.llm.chat_models import RuleBasedChatModel + + +@pytest.fixture +def att_store(tmp_path, monkeypatch): + store = AttachmentAssetStore(root_dir=tmp_path / "att") + + def _factory(root_dir=None): + if root_dir is None: + return store + return AttachmentAssetStore(root_dir=root_dir) + + monkeypatch.setattr("svc.files.file_attachments.AttachmentAssetStore", _factory) + monkeypatch.setattr("runtime.chat.agent_messages.AttachmentAssetStore", _factory) + return store + + +def _msg_text(message: dict) -> str: + content = message.get("content") + if isinstance(content, list): + return "\n".join(str(x.get("text") or "") for x in content if isinstance(x, dict)) + return str(content or "") + + +def test_expand_attachment_ref_large_txt_becomes_text_ref(att_store) -> None: + text = ("line\n" * 5000).strip() + data = text.encode("utf-8") + meta = att_store.save_bytes(data, filename="sites.txt", mime="text/plain") + got = expand_attachment_ref({"type": "binary_ref", "attachment_id": meta.attachment_id}) + types = {str(x.get("type") or "") for x in got} + assert "text_ref" in types + assert "text" in types + + +def test_channel_attachments_for_gateway_parses_local_txt(tmp_path) -> None: + p = tmp_path / "Site_List.txt" + p.write_text("a\nb\nc\n", encoding="utf-8") + got = _channel_attachments_for_gateway([{"local_path": str(p), "kind": "document", "mime": "text/plain"}]) + assert got + assert any(str(x.get("type") or "") == "text" for x in got) + + +def test_build_llm_messages_binary_ref_meta_for_historical_turn(att_store) -> None: + data = b"hello attachment" + meta = att_store.save_bytes(data, filename="note.txt", mime="text/plain") + rows = [ + SimpleNamespace( + role="user", + event_type="user_text", + content="first", + attachments=[{"type": "binary_ref", "attachment_id": meta.attachment_id}], + tool_calls=None, + turn_uuid="t1", + ), + SimpleNamespace( + role="user", + event_type="user_text", + content="second", + attachments=None, + tool_calls=None, + turn_uuid="t2", + ), + ] + msgs = build_llm_messages(store_messages=rows, system_prompt="s", model=RuleBasedChatModel(), lang="zh") + first_user = next(m for m in msgs if m.get("role") == "user" and "first" in _msg_text(m)) + content = _msg_text(first_user) + assert "BinaryAttachment" in content + assert meta.attachment_id in content + + +def test_build_llm_messages_binary_ref_expands_on_last_turn(att_store) -> None: + text = ("row\n" * 4000).strip() + data = text.encode("utf-8") + meta = att_store.save_bytes(data, filename="Site_List.txt", mime="text/plain") + rows = [ + SimpleNamespace( + role="user", + event_type="user_text", + content="帮看一下有多少数据", + attachments=[{"type": "binary_ref", "attachment_id": meta.attachment_id}], + tool_calls=None, + turn_uuid="t1", + ), + ] + msgs = build_llm_messages(store_messages=rows, system_prompt="s", model=RuleBasedChatModel(), lang="zh") + user = msgs[-1] + content = _msg_text(user) + assert "LongTextAttachment" in content or "text_id" in content diff --git a/tests/test_inbound_service_reply_suppress.py b/tests/test_inbound_service_reply_suppress.py index 1bf0be28..dbed3938 100644 --- a/tests/test_inbound_service_reply_suppress.py +++ b/tests/test_inbound_service_reply_suppress.py @@ -174,6 +174,20 @@ def test_collect_recent_tool_attachments_ignores_media_from_prior_turn() -> None assert out == [] +def test_collect_recent_tool_attachments_ignores_text_ref_from_lookup_tools() -> None: + rows = [ + _Row(role="user", content="analyze file", attachments=None), + _Row( + role="tool", + content="{}", + attachments='[{"type":"text_ref","attachment_id":"user-upload","name":"Site_List.txt","mime":"text/plain"}]', + ), + _Row(role="assistant", content="15,687 rows", attachments=None), + ] + out = _collect_recent_tool_attachments(store=_FakeStore(rows), session_id="s1") + assert out == [] + + def test_maybe_add_media_path_for_wechat_reply_sets_media_path(monkeypatch) -> None: # Avoid touching disk: stub AttachmentAssetStore.get_local_path. from pathlib import Path