feat(chat): image specialist session image fallback and WS UX

- Resolve legacy multimodal inputs from session history when the user sends
  text-only (assistant images first, then user uploads); optional env toggles.
- Support relay_pointer image resolution in legacy lane collector.
- Align uvicorn WebSocket frame limit with MAX_PAYLOAD_BYTES; document
  OCLAW_UVICORN_WS_MAX_SIZE and compatible-mode env knobs.
- Admin chat: encode attachments after bubble preview; FileReader-based
  base64 for large files.
- Tests and docs for new behavior.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-05-10 11:25:21 +08:00
parent faeb067856
commit 396dba76ea
8 changed files with 368 additions and 28 deletions

View file

@ -1,5 +1,9 @@
from __future__ import annotations
import json
import pytest
from oclaw.platform.llm.image_http_common import (
dashscope_multimodal_http_ok,
dashscope_native_multimodal_url_from_compatible_base,
@ -8,10 +12,43 @@ from oclaw.platform.llm.image_http_common import (
from oclaw.platform.llm.image_http_common import build_extract_diag_empty
from oclaw.platform.llm.image_legacy_client import (
collect_legacy_lane_images_from_attachments,
collect_legacy_lane_images_with_session_fallback,
legacy_image_assistant_body_with_placeholder,
legacy_image_turn_bundle,
normalize_legacy_output_image_urls,
parse_message_attachments_json,
)
from oclaw.platform.llm.image_legacy_client import _http_content_blocks
from oclaw.platform.llm.image_legacy_client import _openai_compatible_vision_content
def test_openai_compatible_vision_has_type_per_part() -> None:
"""compatible-mode /chat/completions expects type on each content element."""
b = _openai_compatible_vision_content(
["https://example.invalid/a.png", "data:image/png;base64,abcd"],
"prompt",
)
assert len(b) == 3
assert b[0]["type"] == "image_url" and "url" in b[0]["image_url"]
assert b[1]["type"] == "image_url"
assert b[2]["type"] == "text" and b[2]["text"] == "prompt"
def test_http_content_blocks_multi_image_then_text() -> None:
"""DashScope samples: one user message, content = N × {"image": url} then {"text": ...}."""
b = _http_content_blocks(
[
"https://example.invalid/a.png",
"https://example.invalid/b.png",
],
"合成说明",
typed=False,
)
assert b == [
{"image": "https://example.invalid/a.png"},
{"image": "https://example.invalid/b.png"},
{"text": "合成说明"},
]
def test_normalize_legacy_output_dict_image_parts() -> None:
@ -246,3 +283,106 @@ def test_extract_text_and_images_openai_top_level_unchanged() -> None:
)
assert "hi" in text
assert images == ["https://example.invalid/a.jpg"]
def test_parse_message_attachments_json_string_list() -> None:
raw = json.dumps([{"type": "image_url", "url": "https://example.invalid/z.png"}])
got = parse_message_attachments_json(raw)
assert len(got) == 1 and got[0]["url"] == "https://example.invalid/z.png"
class _FakeHistMsg:
__slots__ = ("role", "attachments")
def __init__(self, role: str, attachments: object) -> None:
self.role = role
self.attachments = attachments
class _FakeHistStore:
def __init__(self, msgs: list[_FakeHistMsg]) -> None:
self._msgs = msgs
def get_messages(self, session_id: str, limit: int = 200) -> list[_FakeHistMsg]:
_ = session_id
return self._msgs[-limit:] if len(self._msgs) > limit else list(self._msgs)
def test_session_fallback_prefers_latest_assistant_images() -> None:
"""Newest assistant row with images wins over older user uploads."""
msgs = [
_FakeHistMsg(
"user",
json.dumps([{"type": "image_url", "url": "https://example.invalid/old-user.png"}]),
),
_FakeHistMsg(
"assistant",
json.dumps([{"type": "image_url", "url": "https://example.invalid/from-assistant.png"}]),
),
_FakeHistMsg("user", "null"),
]
store = _FakeHistStore(msgs)
imgs, src = collect_legacy_lane_images_with_session_fallback(
store=store,
session_id="s1",
attachments=[],
)
assert src == "assistant_history"
assert imgs == ["https://example.invalid/from-assistant.png"]
def test_session_fallback_user_history_when_no_assistant_images() -> None:
msgs = [
_FakeHistMsg(
"user",
json.dumps([{"type": "image_url", "url": "https://example.invalid/only-user.png"}]),
),
_FakeHistMsg("assistant", "[]"),
_FakeHistMsg("user", "null"),
]
store = _FakeHistStore(msgs)
imgs, src = collect_legacy_lane_images_with_session_fallback(
store=store,
session_id="s1",
attachments=[],
)
assert src == "user_history"
assert imgs == ["https://example.invalid/only-user.png"]
def test_session_fallback_disabled_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("AIA_IMAGE_SPECIALIST_SESSION_IMAGE_FALLBACK", "0")
msgs = [
_FakeHistMsg(
"assistant",
json.dumps([{"type": "image_url", "url": "https://example.invalid/a.png"}]),
),
_FakeHistMsg("user", "null"),
]
store = _FakeHistStore(msgs)
imgs, src = collect_legacy_lane_images_with_session_fallback(
store=store,
session_id="s1",
attachments=[],
)
assert src == ""
assert imgs == []
def test_session_fallback_current_attachments_skip_history(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("AIA_IMAGE_SPECIALIST_SESSION_IMAGE_FALLBACK", raising=False)
msgs = [
_FakeHistMsg(
"assistant",
json.dumps([{"type": "image_url", "url": "https://example.invalid/hist.png"}]),
),
_FakeHistMsg("user", "null"),
]
store = _FakeHistStore(msgs)
imgs, src = collect_legacy_lane_images_with_session_fallback(
store=store,
session_id="s1",
attachments=[{"type": "image_url", "url": "https://example.invalid/current.png"}],
)
assert src == "current"
assert imgs == ["https://example.invalid/current.png"]