mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-17 06:11:01 +08:00
重构主控编排与运行时预热链路，统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为（含 wiki worker）、更新 Admin 可观测性与相关测试，降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
388
openclaw/extensions/openai/video-generation-provider.ts
Normal file
388
openclaw/extensions/openai/video-generation-provider.ts
Normal file
|
|
@ -0,0 +1,388 @@
|
|||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
createProviderOperationDeadline,
|
||||
fetchWithTimeout,
|
||||
postJsonRequest,
|
||||
resolveProviderOperationTimeoutMs,
|
||||
resolveProviderHttpRequestConfig,
|
||||
waitProviderOperationPollInterval,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
|
||||
import type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import { resolveConfiguredOpenAIBaseUrl, toOpenAIDataUrl } from "./shared.js";
|
||||
|
||||
const DEFAULT_OPENAI_VIDEO_BASE_URL = "https://api.openai.com/v1";
|
||||
const DEFAULT_OPENAI_VIDEO_MODEL = "sora-2";
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
const POLL_INTERVAL_MS = 2_500;
|
||||
const MAX_POLL_ATTEMPTS = 120;
|
||||
const OPENAI_VIDEO_SECONDS = [4, 8, 12] as const;
|
||||
const OPENAI_VIDEO_SIZES = ["720x1280", "1280x720", "1024x1792", "1792x1024"] as const;
|
||||
|
||||
type OpenAIVideoStatus = "queued" | "in_progress" | "completed" | "failed";
|
||||
|
||||
type OpenAIVideoResponse = {
|
||||
id?: string;
|
||||
model?: string;
|
||||
status?: OpenAIVideoStatus;
|
||||
prompt?: string | null;
|
||||
seconds?: string;
|
||||
size?: string;
|
||||
error?: {
|
||||
code?: string;
|
||||
message?: string;
|
||||
} | null;
|
||||
};
|
||||
|
||||
function toBlobBytes(buffer: Buffer): ArrayBuffer {
|
||||
const arrayBuffer = new ArrayBuffer(buffer.byteLength);
|
||||
new Uint8Array(arrayBuffer).set(buffer);
|
||||
return arrayBuffer;
|
||||
}
|
||||
|
||||
function resolveDurationSeconds(durationSeconds: number | undefined): "4" | "8" | "12" | undefined {
|
||||
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||
return undefined;
|
||||
}
|
||||
const rounded = Math.max(OPENAI_VIDEO_SECONDS[0], Math.round(durationSeconds));
|
||||
const nearest = OPENAI_VIDEO_SECONDS.reduce((best, current) =>
|
||||
Math.abs(current - rounded) < Math.abs(best - rounded) ? current : best,
|
||||
);
|
||||
return String(nearest) as "4" | "8" | "12";
|
||||
}
|
||||
|
||||
function resolveSize(params: {
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
resolution?: string;
|
||||
}): (typeof OPENAI_VIDEO_SIZES)[number] | undefined {
|
||||
const explicitSize = normalizeOptionalString(params.size);
|
||||
if (
|
||||
explicitSize &&
|
||||
OPENAI_VIDEO_SIZES.includes(explicitSize as (typeof OPENAI_VIDEO_SIZES)[number])
|
||||
) {
|
||||
return explicitSize as (typeof OPENAI_VIDEO_SIZES)[number];
|
||||
}
|
||||
switch (normalizeOptionalString(params.aspectRatio)) {
|
||||
case "9:16":
|
||||
return "720x1280";
|
||||
case "16:9":
|
||||
return "1280x720";
|
||||
case "4:7":
|
||||
return "1024x1792";
|
||||
case "7:4":
|
||||
return "1792x1024";
|
||||
default:
|
||||
break;
|
||||
}
|
||||
if (params.resolution === "1080P") {
|
||||
return "1792x1024";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveReferenceAsset(req: VideoGenerationRequest) {
|
||||
const allAssets = [...(req.inputImages ?? []), ...(req.inputVideos ?? [])];
|
||||
if (allAssets.length === 0) {
|
||||
return null;
|
||||
}
|
||||
if (allAssets.length > 1) {
|
||||
throw new Error("OpenAI video generation supports at most one reference image or video.");
|
||||
}
|
||||
const [asset] = allAssets;
|
||||
if (!asset?.buffer) {
|
||||
throw new Error(
|
||||
"OpenAI video generation currently requires local image/video uploads for reference assets.",
|
||||
);
|
||||
}
|
||||
const mimeType =
|
||||
normalizeOptionalString(asset.mimeType) ||
|
||||
((req.inputVideos?.length ?? 0) > 0 ? "video/mp4" : "image/png");
|
||||
const extension = mimeType.includes("video")
|
||||
? "mp4"
|
||||
: mimeType.includes("jpeg")
|
||||
? "jpg"
|
||||
: mimeType.includes("webp")
|
||||
? "webp"
|
||||
: "png";
|
||||
const fileName =
|
||||
normalizeOptionalString(asset.fileName) ||
|
||||
`${(req.inputVideos?.length ?? 0) > 0 ? "reference-video" : "reference-image"}.${extension}`;
|
||||
return new File([toBlobBytes(asset.buffer)], fileName, { type: mimeType });
|
||||
}
|
||||
|
||||
async function pollOpenAIVideo(params: {
|
||||
videoId: string;
|
||||
headers: Headers;
|
||||
timeoutMs?: number;
|
||||
baseUrl: string;
|
||||
fetchFn: typeof fetch;
|
||||
}): Promise<OpenAIVideoResponse> {
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: params.timeoutMs,
|
||||
label: `OpenAI video generation task ${params.videoId}`,
|
||||
});
|
||||
for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) {
|
||||
const response = await fetchWithTimeout(
|
||||
`${params.baseUrl}/videos/${params.videoId}`,
|
||||
{
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
},
|
||||
resolveProviderOperationTimeoutMs({ deadline, defaultTimeoutMs: DEFAULT_TIMEOUT_MS }),
|
||||
params.fetchFn,
|
||||
);
|
||||
await assertOkOrThrowHttpError(response, "OpenAI video status request failed");
|
||||
const payload = (await response.json()) as OpenAIVideoResponse;
|
||||
if (payload.status === "completed") {
|
||||
return payload;
|
||||
}
|
||||
if (payload.status === "failed") {
|
||||
throw new Error(
|
||||
normalizeOptionalString(payload.error?.message) || "OpenAI video generation failed",
|
||||
);
|
||||
}
|
||||
await waitProviderOperationPollInterval({ deadline, pollIntervalMs: POLL_INTERVAL_MS });
|
||||
}
|
||||
throw new Error(`OpenAI video generation task ${params.videoId} did not finish in time`);
|
||||
}
|
||||
|
||||
async function downloadOpenAIVideo(params: {
|
||||
videoId: string;
|
||||
headers: Headers;
|
||||
timeoutMs?: number;
|
||||
baseUrl: string;
|
||||
fetchFn: typeof fetch;
|
||||
}): Promise<GeneratedVideoAsset> {
|
||||
const url = new URL(`${params.baseUrl}/videos/${params.videoId}/content`);
|
||||
url.searchParams.set("variant", "video");
|
||||
const response = await fetchWithTimeout(
|
||||
url.toString(),
|
||||
{
|
||||
method: "GET",
|
||||
headers: new Headers({
|
||||
...Object.fromEntries(params.headers.entries()),
|
||||
Accept: "application/binary",
|
||||
}),
|
||||
},
|
||||
params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
|
||||
params.fetchFn,
|
||||
);
|
||||
await assertOkOrThrowHttpError(response, "OpenAI video download failed");
|
||||
const mimeType = normalizeOptionalString(response.headers.get("content-type")) ?? "video/mp4";
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return {
|
||||
buffer: Buffer.from(arrayBuffer),
|
||||
mimeType,
|
||||
fileName: `video-1.${mimeType.includes("webm") ? "webm" : "mp4"}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildOpenAIVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
id: "openai",
|
||||
label: "OpenAI",
|
||||
defaultModel: DEFAULT_OPENAI_VIDEO_MODEL,
|
||||
models: [DEFAULT_OPENAI_VIDEO_MODEL, "sora-2-pro"],
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "openai",
|
||||
agentDir,
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
maxDurationSeconds: 12,
|
||||
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||
supportsSize: true,
|
||||
sizes: OPENAI_VIDEO_SIZES,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
maxDurationSeconds: 12,
|
||||
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||
supportsSize: true,
|
||||
sizes: OPENAI_VIDEO_SIZES,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputVideos: 1,
|
||||
maxDurationSeconds: 12,
|
||||
supportedDurationSeconds: OPENAI_VIDEO_SECONDS,
|
||||
supportsSize: true,
|
||||
sizes: OPENAI_VIDEO_SIZES,
|
||||
},
|
||||
},
|
||||
async generateVideo(req) {
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "openai",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("OpenAI API key missing");
|
||||
}
|
||||
|
||||
const fetchFn = fetch;
|
||||
const deadline = createProviderOperationDeadline({
|
||||
timeoutMs: req.timeoutMs,
|
||||
label: "OpenAI video generation",
|
||||
});
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: resolveConfiguredOpenAIBaseUrl(req.cfg),
|
||||
defaultBaseUrl: DEFAULT_OPENAI_VIDEO_BASE_URL,
|
||||
allowPrivateNetwork: false,
|
||||
defaultHeaders: {
|
||||
Authorization: `Bearer ${auth.apiKey}`,
|
||||
},
|
||||
provider: "openai",
|
||||
capability: "video",
|
||||
transport: "http",
|
||||
});
|
||||
|
||||
const model = normalizeOptionalString(req.model) ?? DEFAULT_OPENAI_VIDEO_MODEL;
|
||||
const seconds = resolveDurationSeconds(req.durationSeconds);
|
||||
const size = resolveSize({
|
||||
size: req.size,
|
||||
aspectRatio: req.aspectRatio,
|
||||
resolution: req.resolution,
|
||||
});
|
||||
const inputImage = req.inputImages?.[0];
|
||||
const referenceAsset = resolveReferenceAsset(req);
|
||||
const requestUrl = `${baseUrl}/videos`;
|
||||
const requestResult = referenceAsset
|
||||
? inputImage?.buffer
|
||||
? await (() => {
|
||||
const jsonHeaders = new Headers(headers);
|
||||
jsonHeaders.set("Content-Type", "application/json");
|
||||
return postJsonRequest({
|
||||
url: requestUrl,
|
||||
headers: jsonHeaders,
|
||||
body: {
|
||||
prompt: req.prompt,
|
||||
model,
|
||||
...(seconds ? { seconds } : {}),
|
||||
...(size ? { size } : {}),
|
||||
input_reference: {
|
||||
image_url: toOpenAIDataUrl(
|
||||
inputImage.buffer,
|
||||
normalizeOptionalString(inputImage.mimeType) ?? "image/png",
|
||||
),
|
||||
},
|
||||
},
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
})()
|
||||
: await (() => {
|
||||
const form = new FormData();
|
||||
form.set("prompt", req.prompt);
|
||||
form.set("model", model);
|
||||
if (seconds) {
|
||||
form.set("seconds", seconds);
|
||||
}
|
||||
if (size) {
|
||||
form.set("size", size);
|
||||
}
|
||||
form.set("input_reference", referenceAsset);
|
||||
const multipartHeaders = new Headers(headers);
|
||||
multipartHeaders.delete("Content-Type");
|
||||
return fetchWithTimeout(
|
||||
requestUrl,
|
||||
{
|
||||
method: "POST",
|
||||
headers: multipartHeaders,
|
||||
body: form,
|
||||
},
|
||||
resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
).then((response) => ({
|
||||
response,
|
||||
release: async () => {},
|
||||
}));
|
||||
})()
|
||||
: await (() => {
|
||||
const jsonHeaders = new Headers(headers);
|
||||
jsonHeaders.set("Content-Type", "application/json");
|
||||
return postJsonRequest({
|
||||
url: requestUrl,
|
||||
headers: jsonHeaders,
|
||||
body: {
|
||||
prompt: req.prompt,
|
||||
model,
|
||||
...(seconds ? { seconds } : {}),
|
||||
...(size ? { size } : {}),
|
||||
},
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
fetchFn,
|
||||
allowPrivateNetwork,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
})();
|
||||
const { response, release } = requestResult;
|
||||
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "OpenAI video generation failed");
|
||||
const submitted = (await response.json()) as OpenAIVideoResponse;
|
||||
const videoId = normalizeOptionalString(submitted.id);
|
||||
if (!videoId) {
|
||||
throw new Error("OpenAI video generation response missing video id");
|
||||
}
|
||||
const completed = await pollOpenAIVideo({
|
||||
videoId,
|
||||
headers,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
baseUrl,
|
||||
fetchFn,
|
||||
});
|
||||
const video = await downloadOpenAIVideo({
|
||||
videoId,
|
||||
headers,
|
||||
timeoutMs: resolveProviderOperationTimeoutMs({
|
||||
deadline,
|
||||
defaultTimeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
}),
|
||||
baseUrl,
|
||||
fetchFn,
|
||||
});
|
||||
return {
|
||||
videos: [video],
|
||||
model: completed.model ?? submitted.model ?? model,
|
||||
metadata: {
|
||||
videoId,
|
||||
status: completed.status,
|
||||
seconds: completed.seconds ?? submitted.seconds,
|
||||
size: completed.size ?? submitted.size,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue