mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-10 18:35:48 +08:00
feat(video): DashScope video specialist with correct i2v request bodies
- Add video_generation_client: async video-synthesis, t2v vs i2v (Wan 2.7 input.media first_frame vs legacy img_url). - Coerce *-t2v* to *-i2v* when first frame present; AIA_VIDEO_I2V_INPUT_STYLE / I2V_MODEL overrides. - Gateway/direct_loop early return for video specialist; workspace video + factory allowlist. - Normalize WS and admin chat attachments (image_ref parity); mp4 attachment store roundtrip tests. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
398d85ba06
commit
4335040a27
21 changed files with 1152 additions and 56 deletions
|
|
@ -276,7 +276,7 @@ def build_ops_agent(
|
|||
|
||||
|
||||
# `default_registry` treats empty allow_tags + empty allow_tools as "no filter". Use an impossible
|
||||
# tool name so the image specialist gets an empty tool surface (vision-only turns).
|
||||
# tool name so image/video specialists get an empty tool surface (dedicated HTTP lanes).
|
||||
_IMAGE_SPECIALIST_TOOL_ALLOWLIST: tuple[str, ...] = ("__oclaw_image_specialist_no_tools__",)
|
||||
|
||||
|
||||
|
|
@ -334,7 +334,7 @@ def build_gateway_executor(
|
|||
"path_policy_user_id": path_policy_user_id,
|
||||
"store": store,
|
||||
}
|
||||
if prof.name == "image":
|
||||
if prof.name in {"image", "video"}:
|
||||
reg_kw["allow_tools"] = list(_IMAGE_SPECIALIST_TOOL_ALLOWLIST)
|
||||
tools = default_registry(**reg_kw)
|
||||
return Agent(
|
||||
|
|
|
|||
|
|
@ -16,10 +16,17 @@ from oclaw.platform.persistence.sqlite_store import SqliteStore
|
|||
from oclaw.platform.llm.image_legacy_client import (
|
||||
IMAGE_SPECIALIST_DEFAULT_PROMPT_ZH,
|
||||
collect_legacy_lane_images_from_attachments,
|
||||
collect_legacy_lane_images_with_session_fallback,
|
||||
legacy_image_assistant_body_with_placeholder,
|
||||
legacy_image_turn_bundle,
|
||||
send_legacy_image_messages,
|
||||
)
|
||||
from oclaw.platform.llm.video_generation_client import (
|
||||
VIDEO_SPECIALIST_DEFAULT_PROMPT_ZH,
|
||||
legacy_video_assistant_body_with_placeholder,
|
||||
legacy_video_turn_bundle,
|
||||
send_video_generation_request,
|
||||
)
|
||||
from oclaw.runtime.tools import default_registry
|
||||
from oclaw.runtime.agents.specialists import expert_name_for_specialist
|
||||
|
||||
|
|
@ -301,6 +308,51 @@ class SpecialistAgentRunner:
|
|||
tool_traces=(),
|
||||
notes="image_pipeline",
|
||||
)
|
||||
elif step.specialist == "video":
|
||||
image_protocol = "video_generation.http"
|
||||
_, chosen_model, _ = self._resolve_profile_and_model(step.specialist)
|
||||
text_parts = [
|
||||
str(x).strip()
|
||||
for x in (step.objective, step.input_text, parent_task.user_text)
|
||||
if str(x or "").strip()
|
||||
]
|
||||
user_text_v = "\n".join(text_parts) if text_parts else VIDEO_SPECIALIST_DEFAULT_PROMPT_ZH
|
||||
policy_sid = str(policy_session_id or parent_task.session_id or session_id or "").strip()
|
||||
v_frames, _v_src = collect_legacy_lane_images_with_session_fallback(
|
||||
store=self.store,
|
||||
session_id=policy_sid,
|
||||
attachments=list(parent_task.attachments or []),
|
||||
max_images=1,
|
||||
)
|
||||
v_frame = str(v_frames[0]).strip() if v_frames else None
|
||||
resp_v = send_video_generation_request(
|
||||
prompt=user_text_v,
|
||||
model=str(getattr(chosen_model, "model", "") or "").strip() or None,
|
||||
api_key=str(getattr(chosen_model, "api_key", "") or "").strip() or None,
|
||||
base_url=str(getattr(chosen_model, "base_url", "") or "").strip() or None,
|
||||
img_url=v_frame,
|
||||
on_progress=on_progress,
|
||||
should_stop=should_stop,
|
||||
)
|
||||
ok, output, produced_attachments = legacy_video_turn_bundle(resp_v)
|
||||
output = legacy_video_assistant_body_with_placeholder(
|
||||
lang=self.lang,
|
||||
body_text=output,
|
||||
produced=produced_attachments if ok else None,
|
||||
)
|
||||
self.store.add_message(
|
||||
session_id=session_id,
|
||||
role="assistant",
|
||||
content=output,
|
||||
attachments=(produced_attachments or None) if ok else None,
|
||||
)
|
||||
specialist_delivery = SpecialistDelivery(
|
||||
specialist=step.specialist,
|
||||
step_id=step.step_id,
|
||||
answer_text=str(output or ""),
|
||||
tool_traces=(),
|
||||
notes="video_pipeline",
|
||||
)
|
||||
else:
|
||||
agent = self._build_agent_for(
|
||||
step.specialist,
|
||||
|
|
|
|||
|
|
@ -43,10 +43,15 @@ SPECIALISTS: dict[SpecialistId, SpecialistConfig] = {
|
|||
expert_name="image",
|
||||
default_tool_tags=None,
|
||||
),
|
||||
"video": SpecialistConfig(
|
||||
specialist_id="video",
|
||||
expert_name="video",
|
||||
default_tool_tags=None,
|
||||
),
|
||||
}
|
||||
|
||||
def discover_specialist_ids() -> tuple[SpecialistId, ...]:
|
||||
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory", "image"))
|
||||
rows = specialist_registry_snapshot(base_order=("generalist", "ops", "memory", "image", "video"))
|
||||
return tuple(str(x.get("id") or "").strip().lower() for x in rows if str(x.get("id") or "").strip())
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue