mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-12 11:43:25 +08:00
feat(chat): image specialist session image fallback and WS UX
- Resolve legacy multimodal inputs from session history when the user sends text-only (assistant images first, then user uploads); optional env toggles. - Support relay_pointer image resolution in legacy lane collector. - Align uvicorn WebSocket frame limit with MAX_PAYLOAD_BYTES; document OCLAW_UVICORN_WS_MAX_SIZE and compatible-mode env knobs. - Admin chat: encode attachments after bubble preview; FileReader-based base64 for large files. - Tests and docs for new behavior. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
faeb067856
commit
396dba76ea
8 changed files with 368 additions and 28 deletions
|
|
@ -1,5 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from oclaw.platform.llm.image_http_common import (
|
||||
dashscope_multimodal_http_ok,
|
||||
dashscope_native_multimodal_url_from_compatible_base,
|
||||
|
|
@ -8,10 +12,43 @@ from oclaw.platform.llm.image_http_common import (
|
|||
from oclaw.platform.llm.image_http_common import build_extract_diag_empty
|
||||
from oclaw.platform.llm.image_legacy_client import (
|
||||
collect_legacy_lane_images_from_attachments,
|
||||
collect_legacy_lane_images_with_session_fallback,
|
||||
legacy_image_assistant_body_with_placeholder,
|
||||
legacy_image_turn_bundle,
|
||||
normalize_legacy_output_image_urls,
|
||||
parse_message_attachments_json,
|
||||
)
|
||||
from oclaw.platform.llm.image_legacy_client import _http_content_blocks
|
||||
from oclaw.platform.llm.image_legacy_client import _openai_compatible_vision_content
|
||||
|
||||
|
||||
def test_openai_compatible_vision_has_type_per_part() -> None:
|
||||
"""compatible-mode /chat/completions expects type on each content element."""
|
||||
b = _openai_compatible_vision_content(
|
||||
["https://example.invalid/a.png", "data:image/png;base64,abcd"],
|
||||
"prompt",
|
||||
)
|
||||
assert len(b) == 3
|
||||
assert b[0]["type"] == "image_url" and "url" in b[0]["image_url"]
|
||||
assert b[1]["type"] == "image_url"
|
||||
assert b[2]["type"] == "text" and b[2]["text"] == "prompt"
|
||||
|
||||
|
||||
def test_http_content_blocks_multi_image_then_text() -> None:
|
||||
"""DashScope samples: one user message, content = N × {"image": url} then {"text": ...}."""
|
||||
b = _http_content_blocks(
|
||||
[
|
||||
"https://example.invalid/a.png",
|
||||
"https://example.invalid/b.png",
|
||||
],
|
||||
"合成说明",
|
||||
typed=False,
|
||||
)
|
||||
assert b == [
|
||||
{"image": "https://example.invalid/a.png"},
|
||||
{"image": "https://example.invalid/b.png"},
|
||||
{"text": "合成说明"},
|
||||
]
|
||||
|
||||
|
||||
def test_normalize_legacy_output_dict_image_parts() -> None:
|
||||
|
|
@ -246,3 +283,106 @@ def test_extract_text_and_images_openai_top_level_unchanged() -> None:
|
|||
)
|
||||
assert "hi" in text
|
||||
assert images == ["https://example.invalid/a.jpg"]
|
||||
|
||||
|
||||
def test_parse_message_attachments_json_string_list() -> None:
|
||||
raw = json.dumps([{"type": "image_url", "url": "https://example.invalid/z.png"}])
|
||||
got = parse_message_attachments_json(raw)
|
||||
assert len(got) == 1 and got[0]["url"] == "https://example.invalid/z.png"
|
||||
|
||||
|
||||
class _FakeHistMsg:
|
||||
__slots__ = ("role", "attachments")
|
||||
|
||||
def __init__(self, role: str, attachments: object) -> None:
|
||||
self.role = role
|
||||
self.attachments = attachments
|
||||
|
||||
|
||||
class _FakeHistStore:
|
||||
def __init__(self, msgs: list[_FakeHistMsg]) -> None:
|
||||
self._msgs = msgs
|
||||
|
||||
def get_messages(self, session_id: str, limit: int = 200) -> list[_FakeHistMsg]:
|
||||
_ = session_id
|
||||
return self._msgs[-limit:] if len(self._msgs) > limit else list(self._msgs)
|
||||
|
||||
|
||||
def test_session_fallback_prefers_latest_assistant_images() -> None:
|
||||
"""Newest assistant row with images wins over older user uploads."""
|
||||
msgs = [
|
||||
_FakeHistMsg(
|
||||
"user",
|
||||
json.dumps([{"type": "image_url", "url": "https://example.invalid/old-user.png"}]),
|
||||
),
|
||||
_FakeHistMsg(
|
||||
"assistant",
|
||||
json.dumps([{"type": "image_url", "url": "https://example.invalid/from-assistant.png"}]),
|
||||
),
|
||||
_FakeHistMsg("user", "null"),
|
||||
]
|
||||
store = _FakeHistStore(msgs)
|
||||
imgs, src = collect_legacy_lane_images_with_session_fallback(
|
||||
store=store,
|
||||
session_id="s1",
|
||||
attachments=[],
|
||||
)
|
||||
assert src == "assistant_history"
|
||||
assert imgs == ["https://example.invalid/from-assistant.png"]
|
||||
|
||||
|
||||
def test_session_fallback_user_history_when_no_assistant_images() -> None:
|
||||
msgs = [
|
||||
_FakeHistMsg(
|
||||
"user",
|
||||
json.dumps([{"type": "image_url", "url": "https://example.invalid/only-user.png"}]),
|
||||
),
|
||||
_FakeHistMsg("assistant", "[]"),
|
||||
_FakeHistMsg("user", "null"),
|
||||
]
|
||||
store = _FakeHistStore(msgs)
|
||||
imgs, src = collect_legacy_lane_images_with_session_fallback(
|
||||
store=store,
|
||||
session_id="s1",
|
||||
attachments=[],
|
||||
)
|
||||
assert src == "user_history"
|
||||
assert imgs == ["https://example.invalid/only-user.png"]
|
||||
|
||||
|
||||
def test_session_fallback_disabled_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("AIA_IMAGE_SPECIALIST_SESSION_IMAGE_FALLBACK", "0")
|
||||
msgs = [
|
||||
_FakeHistMsg(
|
||||
"assistant",
|
||||
json.dumps([{"type": "image_url", "url": "https://example.invalid/a.png"}]),
|
||||
),
|
||||
_FakeHistMsg("user", "null"),
|
||||
]
|
||||
store = _FakeHistStore(msgs)
|
||||
imgs, src = collect_legacy_lane_images_with_session_fallback(
|
||||
store=store,
|
||||
session_id="s1",
|
||||
attachments=[],
|
||||
)
|
||||
assert src == ""
|
||||
assert imgs == []
|
||||
|
||||
|
||||
def test_session_fallback_current_attachments_skip_history(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("AIA_IMAGE_SPECIALIST_SESSION_IMAGE_FALLBACK", raising=False)
|
||||
msgs = [
|
||||
_FakeHistMsg(
|
||||
"assistant",
|
||||
json.dumps([{"type": "image_url", "url": "https://example.invalid/hist.png"}]),
|
||||
),
|
||||
_FakeHistMsg("user", "null"),
|
||||
]
|
||||
store = _FakeHistStore(msgs)
|
||||
imgs, src = collect_legacy_lane_images_with_session_fallback(
|
||||
store=store,
|
||||
session_id="s1",
|
||||
attachments=[{"type": "image_url", "url": "https://example.invalid/current.png"}],
|
||||
)
|
||||
assert src == "current"
|
||||
assert imgs == ["https://example.invalid/current.png"]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue