feat(video): DashScope video specialist with correct i2v request bodies

- Add video_generation_client: async video-synthesis, t2v vs i2v (Wan 2.7 input.media first_frame vs legacy img_url).

- Coerce *-t2v* to *-i2v* when first frame present; AIA_VIDEO_I2V_INPUT_STYLE / I2V_MODEL overrides.

- Gateway/direct_loop early return for video specialist; workspace video + factory allowlist.

- Normalize WS and admin chat attachments (image_ref parity); mp4 attachment store roundtrip tests.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
oliver 2026-05-10 16:20:43 +08:00
parent 398d85ba06
commit 4335040a27
21 changed files with 1152 additions and 56 deletions

View file

@ -276,7 +276,7 @@ def build_ops_agent(
# `default_registry` treats empty allow_tags + empty allow_tools as "no filter". Use an impossible
# tool name so the image specialist gets an empty tool surface (vision-only turns).
# tool name so image/video specialists get an empty tool surface (dedicated HTTP lanes).
_IMAGE_SPECIALIST_TOOL_ALLOWLIST: tuple[str, ...] = ("__oclaw_image_specialist_no_tools__",)
@ -334,7 +334,7 @@ def build_gateway_executor(
"path_policy_user_id": path_policy_user_id,
"store": store,
}
if prof.name == "image":
if prof.name in {"image", "video"}:
reg_kw["allow_tools"] = list(_IMAGE_SPECIALIST_TOOL_ALLOWLIST)
tools = default_registry(**reg_kw)
return Agent(

View file

@ -16,10 +16,17 @@ from oclaw.platform.persistence.sqlite_store import SqliteStore
from oclaw.platform.llm.image_legacy_client import (
IMAGE_SPECIALIST_DEFAULT_PROMPT_ZH,
collect_legacy_lane_images_from_attachments,
collect_legacy_lane_images_with_session_fallback,
legacy_image_assistant_body_with_placeholder,
legacy_image_turn_bundle,
send_legacy_image_messages,
)
from oclaw.platform.llm.video_generation_client import (
VIDEO_SPECIALIST_DEFAULT_PROMPT_ZH,
legacy_video_assistant_body_with_placeholder,
legacy_video_turn_bundle,
send_video_generation_request,
)
from oclaw.runtime.tools import default_registry
from oclaw.runtime.agents.specialists import expert_name_for_specialist
@ -301,6 +308,51 @@ class SpecialistAgentRunner:
tool_traces=(),
notes="image_pipeline",
)
elif step.specialist == "video":
image_protocol = "video_generation.http"
_, chosen_model, _ = self._resolve_profile_and_model(step.specialist)
text_parts = [
str(x).strip()
for x in (step.objective, step.input_text, parent_task.user_text)
if str(x or "").strip()
]
user_text_v = "\n".join(text_parts) if text_parts else VIDEO_SPECIALIST_DEFAULT_PROMPT_ZH
policy_sid = str(policy_session_id or parent_task.session_id or session_id or "").strip()
v_frames, _v_src = collect_legacy_lane_images_with_session_fallback(
store=self.store,
session_id=policy_sid,
attachments=list(parent_task.attachments or []),
max_images=1,
)
v_frame = str(v_frames[0]).strip() if v_frames else None
resp_v = send_video_generation_request(
prompt=user_text_v,
model=str(getattr(chosen_model, "model", "") or "").strip() or None,
api_key=str(getattr(chosen_model, "api_key", "") or "").strip() or None,
base_url=str(getattr(chosen_model, "base_url", "") or "").strip() or None,
img_url=v_frame,
on_progress=on_progress,
should_stop=should_stop,
)
ok, output, produced_attachments = legacy_video_turn_bundle(resp_v)
output = legacy_video_assistant_body_with_placeholder(
lang=self.lang,
body_text=output,
produced=produced_attachments if ok else None,
)
self.store.add_message(
session_id=session_id,
role="assistant",
content=output,
attachments=(produced_attachments or None) if ok else None,
)
specialist_delivery = SpecialistDelivery(
specialist=step.specialist,
step_id=step.step_id,
answer_text=str(output or ""),
tool_traces=(),
notes="video_pipeline",
)
else:
agent = self._build_agent_for(
step.specialist,

View file

@ -43,10 +43,15 @@ SPECIALISTS: dict[SpecialistId, SpecialistConfig] = {
expert_name="image",
default_tool_tags=None,
),
"video": SpecialistConfig(
specialist_id="video",
expert_name="video",
default_tool_tags=None,
),
}
def discover_specialist_ids() -> tuple[SpecialistId, ...]:
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory", "image"))
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory", "image", "video"))
return tuple(str(x.get("id") or "").strip().lower() for x in rows if str(x.get("id") or "").strip())