feat: multimodal image clients, workspaces, Tushare skill, test fixes

- Replace monolithic image_message_client with HTTP/OCR/legacy modules; tighten OpenAI transport + tool schemas for multimodal downgrade to OCR specialist path.
- Add image/stock workspace prompts (META/SOUL/ROLE_SYSTEM); register experts; tweak specialist agent/direct loop/query_image_attachment.
- Add bundled runtime/skills/tushare-finance (references, api_client, SKILL metadata).
- Document OCR-related env vars; admin chat tweaks; README; weixin_install Ensure-OfficialPluginRuntimeDeps helper.
- Tests: multimodal downgrade + OCR coverage, strict tool pairing in attachment replay guard, workspace contract skips _internal/_system dirs, router/trace/prompt guards.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-05-10 04:41:42 +08:00
parent 6307ade480
commit d1bcc4debe
266 changed files with 57024 additions and 839 deletions

View file

@ -275,6 +275,11 @@ def build_ops_agent(
)
# `default_registry` treats empty allow_tags + empty allow_tools as "no filter". Use an impossible
# tool name so the image specialist gets an empty tool surface (vision-only turns).
_IMAGE_SPECIALIST_TOOL_ALLOWLIST: tuple[str, ...] = ("__oclaw_image_specialist_no_tools__",)
def build_gateway_executor(
store: SqliteStore,
*,
@ -321,14 +326,17 @@ def build_gateway_executor(
path_policy_tenant_id=path_policy_tenant_id,
path_policy_user_id=path_policy_user_id,
)
tools = default_registry(
expert=expert_name_for_specialist(prof.name),
specialist=prof.name,
policy_session_id=policy_session_id,
path_policy_tenant_id=path_policy_tenant_id,
path_policy_user_id=path_policy_user_id,
store=store,
)
reg_kw: dict[str, Any] = {
"expert": expert_name_for_specialist(prof.name),
"specialist": prof.name,
"policy_session_id": policy_session_id,
"path_policy_tenant_id": path_policy_tenant_id,
"path_policy_user_id": path_policy_user_id,
"store": store,
}
if prof.name == "image":
reg_kw["allow_tools"] = list(_IMAGE_SPECIALIST_TOOL_ALLOWLIST)
tools = default_registry(**reg_kw)
return Agent(
store=store,
tools=tools,

View file

@ -15,7 +15,7 @@ from oclaw.runtime.chat.agent import GenerationInterrupted
from oclaw.runtime.agents.network_ops_agent import NetworkOpsAgent
from oclaw.platform.persistence.sqlite_store import SqliteStore
from oclaw.platform.files.attachment_assets import AttachmentAssetStore, attachment_id_to_data_url
from oclaw.platform.llm.image_message_client import send_image_messages
from oclaw.platform.llm.image_ocr_client import send_ocr_image_messages
from oclaw.runtime.tools import default_registry
from oclaw.runtime.agents.specialists import expert_name_for_specialist
@ -267,25 +267,22 @@ class SpecialistAgentRunner:
else:
_, chosen_model, _ = self._resolve_profile_and_model(step.specialist)
model_name = str(
os.getenv("AIA_IMAGE_MODEL")
or getattr(chosen_model, "model", None)
or ""
os.getenv("AIA_OCR_MODEL") or getattr(chosen_model, "model", None) or ""
).strip() or None
api_key = str(getattr(chosen_model, "api_key", "") or "").strip() or None
base_url = str(getattr(chosen_model, "base_url", "") or "").strip() or None
dashscope_api_key = api_key if base_url and "dashscope.aliyuncs.com" in base_url.lower() else None
dashscope_base_http_api_url = None
if base_url and "dashscope.aliyuncs.com" in base_url.lower():
# Normalize compatible-mode/v1 to native /api/v1 for DashScope SDK.
dashscope_base_http_api_url = str(base_url).replace("/compatible-mode/v1", "/api/v1")
resp = send_image_messages(
text_parts = [
str(x).strip()
for x in (step.objective, step.input_text, parent_task.user_text)
if str(x or "").strip()
]
user_text = "\n".join(text_parts) if text_parts else "请根据用户上传的图片作答:描述可见场景、物体与文字;不确定处请标明。"
resp = send_ocr_image_messages(
images=selected_images,
prompt=f"{step.objective}\n\n{step.input_text}\n\n{parent_task.user_text}",
prompt=user_text,
model=model_name,
api_key=api_key,
base_url=base_url,
dashscope_api_key=dashscope_api_key,
dashscope_base_http_api_url=dashscope_base_http_api_url,
)
image_debug_schema = str(resp.get("debug_used_schema") or "").strip()
dbg = resp.get("debug_used_debug")

View file

@ -37,10 +37,16 @@ SPECIALISTS: dict[SpecialistId, SpecialistConfig] = {
expert_name="memory",
default_tool_tags=None,
),
"image": SpecialistConfig(
specialist_id="image",
# Vision turns attach pixels in-message; gateway executor exposes no tools (see factory).
expert_name="image",
default_tool_tags=None,
),
}
def discover_specialist_ids() -> tuple[SpecialistId, ...]:
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory"))
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory", "image"))
return tuple(str(x.get("id") or "").strip().lower() for x in rows if str(x.get("id") or "").strip())