mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-09 04:40:45 +08:00
feat: multimodal image clients, workspaces, Tushare skill, test fixes
- Replace monolithic image_message_client with HTTP/OCR/legacy modules; tighten OpenAI transport + tool schemas for multimodal downgrade to OCR specialist path. - Add image/stock workspace prompts (META/SOUL/ROLE_SYSTEM); register experts; tweak specialist agent/direct loop/query_image_attachment. - Add bundled runtime/skills/tushare-finance (references, api_client, SKILL metadata). - Document OCR-related env vars; admin chat tweaks; README; weixin_install Ensure-OfficialPluginRuntimeDeps helper. - Tests: multimodal downgrade + OCR coverage, strict tool pairing in attachment replay guard, workspace contract skips _internal/_system dirs, router/trace/prompt guards. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
6307ade480
commit
d1bcc4debe
266 changed files with 57024 additions and 839 deletions
|
|
@ -275,6 +275,11 @@ def build_ops_agent(
|
|||
)
|
||||
|
||||
|
||||
# `default_registry` treats empty allow_tags + empty allow_tools as "no filter". Use an impossible
|
||||
# tool name so the image specialist gets an empty tool surface (vision-only turns).
|
||||
_IMAGE_SPECIALIST_TOOL_ALLOWLIST: tuple[str, ...] = ("__oclaw_image_specialist_no_tools__",)
|
||||
|
||||
|
||||
def build_gateway_executor(
|
||||
store: SqliteStore,
|
||||
*,
|
||||
|
|
@ -321,14 +326,17 @@ def build_gateway_executor(
|
|||
path_policy_tenant_id=path_policy_tenant_id,
|
||||
path_policy_user_id=path_policy_user_id,
|
||||
)
|
||||
tools = default_registry(
|
||||
expert=expert_name_for_specialist(prof.name),
|
||||
specialist=prof.name,
|
||||
policy_session_id=policy_session_id,
|
||||
path_policy_tenant_id=path_policy_tenant_id,
|
||||
path_policy_user_id=path_policy_user_id,
|
||||
store=store,
|
||||
)
|
||||
reg_kw: dict[str, Any] = {
|
||||
"expert": expert_name_for_specialist(prof.name),
|
||||
"specialist": prof.name,
|
||||
"policy_session_id": policy_session_id,
|
||||
"path_policy_tenant_id": path_policy_tenant_id,
|
||||
"path_policy_user_id": path_policy_user_id,
|
||||
"store": store,
|
||||
}
|
||||
if prof.name == "image":
|
||||
reg_kw["allow_tools"] = list(_IMAGE_SPECIALIST_TOOL_ALLOWLIST)
|
||||
tools = default_registry(**reg_kw)
|
||||
return Agent(
|
||||
store=store,
|
||||
tools=tools,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from oclaw.runtime.chat.agent import GenerationInterrupted
|
|||
from oclaw.runtime.agents.network_ops_agent import NetworkOpsAgent
|
||||
from oclaw.platform.persistence.sqlite_store import SqliteStore
|
||||
from oclaw.platform.files.attachment_assets import AttachmentAssetStore, attachment_id_to_data_url
|
||||
from oclaw.platform.llm.image_message_client import send_image_messages
|
||||
from oclaw.platform.llm.image_ocr_client import send_ocr_image_messages
|
||||
from oclaw.runtime.tools import default_registry
|
||||
from oclaw.runtime.agents.specialists import expert_name_for_specialist
|
||||
|
||||
|
|
@ -267,25 +267,22 @@ class SpecialistAgentRunner:
|
|||
else:
|
||||
_, chosen_model, _ = self._resolve_profile_and_model(step.specialist)
|
||||
model_name = str(
|
||||
os.getenv("AIA_IMAGE_MODEL")
|
||||
or getattr(chosen_model, "model", None)
|
||||
or ""
|
||||
os.getenv("AIA_OCR_MODEL") or getattr(chosen_model, "model", None) or ""
|
||||
).strip() or None
|
||||
api_key = str(getattr(chosen_model, "api_key", "") or "").strip() or None
|
||||
base_url = str(getattr(chosen_model, "base_url", "") or "").strip() or None
|
||||
dashscope_api_key = api_key if base_url and "dashscope.aliyuncs.com" in base_url.lower() else None
|
||||
dashscope_base_http_api_url = None
|
||||
if base_url and "dashscope.aliyuncs.com" in base_url.lower():
|
||||
# Normalize compatible-mode/v1 to native /api/v1 for DashScope SDK.
|
||||
dashscope_base_http_api_url = str(base_url).replace("/compatible-mode/v1", "/api/v1")
|
||||
resp = send_image_messages(
|
||||
text_parts = [
|
||||
str(x).strip()
|
||||
for x in (step.objective, step.input_text, parent_task.user_text)
|
||||
if str(x or "").strip()
|
||||
]
|
||||
user_text = "\n".join(text_parts) if text_parts else "请根据用户上传的图片作答:描述可见场景、物体与文字;不确定处请标明。"
|
||||
resp = send_ocr_image_messages(
|
||||
images=selected_images,
|
||||
prompt=f"{step.objective}\n\n{step.input_text}\n\n{parent_task.user_text}",
|
||||
prompt=user_text,
|
||||
model=model_name,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
dashscope_api_key=dashscope_api_key,
|
||||
dashscope_base_http_api_url=dashscope_base_http_api_url,
|
||||
)
|
||||
image_debug_schema = str(resp.get("debug_used_schema") or "").strip()
|
||||
dbg = resp.get("debug_used_debug")
|
||||
|
|
|
|||
|
|
@ -37,10 +37,16 @@ SPECIALISTS: dict[SpecialistId, SpecialistConfig] = {
|
|||
expert_name="memory",
|
||||
default_tool_tags=None,
|
||||
),
|
||||
"image": SpecialistConfig(
|
||||
specialist_id="image",
|
||||
# Vision turns attach pixels in-message; gateway executor exposes no tools (see factory).
|
||||
expert_name="image",
|
||||
default_tool_tags=None,
|
||||
),
|
||||
}
|
||||
|
||||
def discover_specialist_ids() -> tuple[SpecialistId, ...]:
|
||||
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory"))
|
||||
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory", "image"))
|
||||
return tuple(str(x.get("id") or "").strip().lower() for x in rows if str(x.get("id") or "").strip())
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue