mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-13 09:30:45 +08:00
重构主控编排与运行时预热链路，统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为（含 wiki worker）、更新 Admin 可观测性与相关测试，降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
417
openclaw/extensions/fal/image-generation-provider.test.ts
Normal file
417
openclaw/extensions/fal/image-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuardMock: vi.fn(),
|
||||
}));
|
||||
|
||||
import {
|
||||
_setFalFetchGuardForTesting,
|
||||
buildFalImageGenerationProvider,
|
||||
} from "./image-generation-provider.js";
|
||||
|
||||
function expectFalJsonPost(params: { call: number; url: string; body: Record<string, unknown> }) {
|
||||
const request = fetchWithSsrFGuardMock.mock.calls[params.call - 1]?.[0];
|
||||
expect(request).toBeTruthy();
|
||||
expect(request?.url).toBe(params.url);
|
||||
expect(request?.auditContext).toBe("fal-image-generate");
|
||||
expect(request?.init?.method).toBe("POST");
|
||||
const headers = new Headers(request?.init?.headers);
|
||||
expect(headers.get("authorization")).toBe("Key fal-test-key");
|
||||
expect(headers.get("content-type")).toBe("application/json");
|
||||
expect(JSON.parse(String(request?.init?.body))).toEqual(params.body);
|
||||
}
|
||||
|
||||
describe("fal image-generation provider", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
_setFalFetchGuardForTesting(null);
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("generates image buffers from the fal sync API", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setFalFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
const releaseRequest = vi.fn(async () => {});
|
||||
const releaseDownload = vi.fn(async () => {});
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
images: [
|
||||
{
|
||||
url: "https://v3.fal.media/files/example/generated.png",
|
||||
content_type: "image/png",
|
||||
},
|
||||
],
|
||||
prompt: "draw a cat",
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: releaseRequest,
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("png-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: releaseDownload,
|
||||
});
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
const result = await provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "draw a cat",
|
||||
cfg: {},
|
||||
count: 2,
|
||||
size: "1536x1024",
|
||||
});
|
||||
|
||||
expectFalJsonPost({
|
||||
call: 1,
|
||||
url: "https://fal.run/fal-ai/flux/dev",
|
||||
body: {
|
||||
prompt: "draw a cat",
|
||||
image_size: { width: 1536, height: 1024 },
|
||||
num_images: 2,
|
||||
output_format: "png",
|
||||
},
|
||||
});
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
url: "https://v3.fal.media/files/example/generated.png",
|
||||
auditContext: "fal-image-download",
|
||||
policy: undefined,
|
||||
}),
|
||||
);
|
||||
expect(releaseRequest).toHaveBeenCalledTimes(1);
|
||||
expect(releaseDownload).toHaveBeenCalledTimes(1);
|
||||
expect(result).toEqual({
|
||||
images: [
|
||||
{
|
||||
buffer: Buffer.from("png-data"),
|
||||
mimeType: "image/png",
|
||||
fileName: "image-1.png",
|
||||
},
|
||||
],
|
||||
model: "fal-ai/flux/dev",
|
||||
metadata: { prompt: "draw a cat" },
|
||||
});
|
||||
});
|
||||
|
||||
it("uses image-to-image endpoint and data-uri input for edits", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setFalFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
images: [{ url: "https://v3.fal.media/files/example/edited.png" }],
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("edited-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "turn this into a noir poster",
|
||||
cfg: {},
|
||||
resolution: "2K",
|
||||
inputImages: [
|
||||
{
|
||||
buffer: Buffer.from("source-image"),
|
||||
mimeType: "image/jpeg",
|
||||
fileName: "source.jpg",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expectFalJsonPost({
|
||||
call: 1,
|
||||
url: "https://fal.run/fal-ai/flux/dev/image-to-image",
|
||||
body: {
|
||||
prompt: "turn this into a noir poster",
|
||||
image_size: { width: 2048, height: 2048 },
|
||||
num_images: 1,
|
||||
output_format: "png",
|
||||
image_url: `data:image/jpeg;base64,${Buffer.from("source-image").toString("base64")}`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("maps aspect ratio for text generation without forcing a square default", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setFalFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
images: [{ url: "https://v3.fal.media/files/example/wide.png" }],
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("wide-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "wide cinematic shot",
|
||||
cfg: {},
|
||||
aspectRatio: "16:9",
|
||||
});
|
||||
|
||||
expectFalJsonPost({
|
||||
call: 1,
|
||||
url: "https://fal.run/fal-ai/flux/dev",
|
||||
body: {
|
||||
prompt: "wide cinematic shot",
|
||||
image_size: "landscape_16_9",
|
||||
num_images: 1,
|
||||
output_format: "png",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("combines resolution and aspect ratio for text generation", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setFalFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
images: [{ url: "https://v3.fal.media/files/example/portrait.png" }],
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("portrait-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "portrait poster",
|
||||
cfg: {},
|
||||
resolution: "2K",
|
||||
aspectRatio: "9:16",
|
||||
});
|
||||
|
||||
expectFalJsonPost({
|
||||
call: 1,
|
||||
url: "https://fal.run/fal-ai/flux/dev",
|
||||
body: {
|
||||
prompt: "portrait poster",
|
||||
image_size: { width: 1152, height: 2048 },
|
||||
num_images: 1,
|
||||
output_format: "png",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects multi-image edit requests for now", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
await expect(
|
||||
provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "combine these",
|
||||
cfg: {},
|
||||
inputImages: [
|
||||
{ buffer: Buffer.from("one"), mimeType: "image/png" },
|
||||
{ buffer: Buffer.from("two"), mimeType: "image/png" },
|
||||
],
|
||||
}),
|
||||
).rejects.toThrow("at most one reference image");
|
||||
});
|
||||
|
||||
it("rejects aspect ratio overrides for the current edit endpoint", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
await expect(
|
||||
provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "make it widescreen",
|
||||
cfg: {},
|
||||
aspectRatio: "16:9",
|
||||
inputImages: [{ buffer: Buffer.from("one"), mimeType: "image/png" }],
|
||||
}),
|
||||
).rejects.toThrow("does not support aspectRatio overrides");
|
||||
});
|
||||
|
||||
it("blocks private-network image download URLs through the SSRF guard", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setFalFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
const blocked = new Error("Blocked: resolves to private/internal/special-use IP address");
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
images: [{ url: "http://169.254.169.254/latest/meta-data/iam/security-credentials/" }],
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockRejectedValueOnce(blocked);
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
await expect(
|
||||
provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "draw a cat",
|
||||
cfg: {},
|
||||
}),
|
||||
).rejects.toThrow(blocked.message);
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
url: "http://169.254.169.254/latest/meta-data/iam/security-credentials/",
|
||||
auditContext: "fal-image-download",
|
||||
policy: undefined,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not auto-whitelist trusted private relay hosts from a configured baseUrl", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setFalFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
images: [{ url: "http://media.relay.internal/files/generated.png" }],
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("png-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildFalImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "fal",
|
||||
model: "fal-ai/flux/dev",
|
||||
prompt: "draw a cat",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
fal: {
|
||||
baseUrl: "http://relay.internal:8080",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "http://relay.internal:8080/fal-ai/flux/dev",
|
||||
auditContext: "fal-image-generate",
|
||||
policy: undefined,
|
||||
}),
|
||||
);
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
url: "http://media.relay.internal/files/generated.png",
|
||||
auditContext: "fal-image-download",
|
||||
policy: undefined,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
429
openclaw/extensions/fal/image-generation-provider.ts
Normal file
429
openclaw/extensions/fal/image-generation-provider.ts
Normal file
|
|
@ -0,0 +1,429 @@
|
|||
import type {
|
||||
GeneratedImageAsset,
|
||||
ImageGenerationProvider,
|
||||
} from "openclaw/plugin-sdk/image-generation";
|
||||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
resolveProviderHttpRequestConfig,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
buildHostnameAllowlistPolicyFromSuffixAllowlist,
|
||||
fetchWithSsrFGuard,
|
||||
type SsrFPolicy,
|
||||
ssrfPolicyFromDangerouslyAllowPrivateNetwork,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalLowercaseString,
|
||||
} from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
const DEFAULT_FAL_BASE_URL = "https://fal.run";
|
||||
const DEFAULT_FAL_IMAGE_MODEL = "fal-ai/flux/dev";
|
||||
const DEFAULT_FAL_EDIT_SUBPATH = "image-to-image";
|
||||
const DEFAULT_OUTPUT_FORMAT = "png";
|
||||
const FAL_SUPPORTED_SIZES = [
|
||||
"1024x1024",
|
||||
"1024x1536",
|
||||
"1536x1024",
|
||||
"1024x1792",
|
||||
"1792x1024",
|
||||
] as const;
|
||||
const FAL_SUPPORTED_ASPECT_RATIOS = ["1:1", "4:3", "3:4", "16:9", "9:16"] as const;
|
||||
|
||||
type FalGeneratedImage = {
|
||||
url?: string;
|
||||
content_type?: string;
|
||||
};
|
||||
|
||||
type FalImageGenerationResponse = {
|
||||
images?: FalGeneratedImage[];
|
||||
prompt?: string;
|
||||
};
|
||||
|
||||
type FalImageSize = string | { width: number; height: number };
|
||||
type FalNetworkPolicy = {
|
||||
apiPolicy?: SsrFPolicy;
|
||||
trustedDownloadHostSuffix?: string;
|
||||
trustedDownloadPolicy?: SsrFPolicy;
|
||||
};
|
||||
|
||||
let falFetchGuard = fetchWithSsrFGuard;
|
||||
|
||||
export function _setFalFetchGuardForTesting(impl: typeof fetchWithSsrFGuard | null): void {
|
||||
falFetchGuard = impl ?? fetchWithSsrFGuard;
|
||||
}
|
||||
|
||||
function mergeSsrFPolicies(...policies: Array<SsrFPolicy | undefined>): SsrFPolicy | undefined {
|
||||
const merged: SsrFPolicy = {};
|
||||
for (const policy of policies) {
|
||||
if (!policy) {
|
||||
continue;
|
||||
}
|
||||
if (policy.allowPrivateNetwork) {
|
||||
merged.allowPrivateNetwork = true;
|
||||
}
|
||||
if (policy.dangerouslyAllowPrivateNetwork) {
|
||||
merged.dangerouslyAllowPrivateNetwork = true;
|
||||
}
|
||||
if (policy.allowRfc2544BenchmarkRange) {
|
||||
merged.allowRfc2544BenchmarkRange = true;
|
||||
}
|
||||
if (policy.allowedHostnames?.length) {
|
||||
merged.allowedHostnames = Array.from(
|
||||
new Set([...(merged.allowedHostnames ?? []), ...policy.allowedHostnames]),
|
||||
);
|
||||
}
|
||||
if (policy.hostnameAllowlist?.length) {
|
||||
merged.hostnameAllowlist = Array.from(
|
||||
new Set([...(merged.hostnameAllowlist ?? []), ...policy.hostnameAllowlist]),
|
||||
);
|
||||
}
|
||||
}
|
||||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||||
}
|
||||
|
||||
function matchesTrustedHostSuffix(hostname: string, trustedSuffix: string): boolean {
|
||||
const normalizedHost = normalizeLowercaseStringOrEmpty(hostname);
|
||||
const normalizedSuffix = normalizeLowercaseStringOrEmpty(trustedSuffix);
|
||||
return normalizedHost === normalizedSuffix || normalizedHost.endsWith(`.${normalizedSuffix}`);
|
||||
}
|
||||
|
||||
function resolveFalNetworkPolicy(params: {
|
||||
baseUrl: string;
|
||||
allowPrivateNetwork: boolean;
|
||||
}): FalNetworkPolicy {
|
||||
let parsedBaseUrl: URL;
|
||||
try {
|
||||
parsedBaseUrl = new URL(params.baseUrl);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
|
||||
const hostSuffix = normalizeLowercaseStringOrEmpty(parsedBaseUrl.hostname);
|
||||
if (!hostSuffix || !params.allowPrivateNetwork) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const hostPolicy = buildHostnameAllowlistPolicyFromSuffixAllowlist([hostSuffix]);
|
||||
const privateNetworkPolicy = ssrfPolicyFromDangerouslyAllowPrivateNetwork(true);
|
||||
const trustedHostPolicy = mergeSsrFPolicies(hostPolicy, privateNetworkPolicy);
|
||||
return {
|
||||
apiPolicy: trustedHostPolicy,
|
||||
trustedDownloadHostSuffix: hostSuffix,
|
||||
trustedDownloadPolicy: trustedHostPolicy,
|
||||
};
|
||||
}
|
||||
|
||||
function ensureFalModelPath(model: string | undefined, hasInputImages: boolean): string {
|
||||
const trimmed = model?.trim() || DEFAULT_FAL_IMAGE_MODEL;
|
||||
if (!hasInputImages) {
|
||||
return trimmed;
|
||||
}
|
||||
if (
|
||||
trimmed.endsWith(`/${DEFAULT_FAL_EDIT_SUBPATH}`) ||
|
||||
trimmed.endsWith("/edit") ||
|
||||
trimmed.includes("/image-to-image/")
|
||||
) {
|
||||
return trimmed;
|
||||
}
|
||||
return `${trimmed}/${DEFAULT_FAL_EDIT_SUBPATH}`;
|
||||
}
|
||||
|
||||
function parseSize(raw: string | undefined): { width: number; height: number } | null {
|
||||
const trimmed = raw?.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
const match = /^(\d{2,5})x(\d{2,5})$/iu.exec(trimmed);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const width = Number.parseInt(match[1] ?? "", 10);
|
||||
const height = Number.parseInt(match[2] ?? "", 10);
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
|
||||
return null;
|
||||
}
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function mapResolutionToEdge(resolution: "1K" | "2K" | "4K" | undefined): number | undefined {
|
||||
if (!resolution) {
|
||||
return undefined;
|
||||
}
|
||||
return resolution === "4K" ? 4096 : resolution === "2K" ? 2048 : 1024;
|
||||
}
|
||||
|
||||
function aspectRatioToEnum(aspectRatio: string | undefined): string | undefined {
|
||||
const normalized = aspectRatio?.trim();
|
||||
if (!normalized) {
|
||||
return undefined;
|
||||
}
|
||||
if (normalized === "1:1") {
|
||||
return "square_hd";
|
||||
}
|
||||
if (normalized === "4:3") {
|
||||
return "landscape_4_3";
|
||||
}
|
||||
if (normalized === "3:4") {
|
||||
return "portrait_4_3";
|
||||
}
|
||||
if (normalized === "16:9") {
|
||||
return "landscape_16_9";
|
||||
}
|
||||
if (normalized === "9:16") {
|
||||
return "portrait_16_9";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function aspectRatioToDimensions(
|
||||
aspectRatio: string,
|
||||
edge: number,
|
||||
): { width: number; height: number } {
|
||||
const match = /^(\d+):(\d+)$/u.exec(aspectRatio.trim());
|
||||
if (!match) {
|
||||
throw new Error(`Invalid fal aspect ratio: ${aspectRatio}`);
|
||||
}
|
||||
const widthRatio = Number.parseInt(match[1] ?? "", 10);
|
||||
const heightRatio = Number.parseInt(match[2] ?? "", 10);
|
||||
if (
|
||||
!Number.isFinite(widthRatio) ||
|
||||
!Number.isFinite(heightRatio) ||
|
||||
widthRatio <= 0 ||
|
||||
heightRatio <= 0
|
||||
) {
|
||||
throw new Error(`Invalid fal aspect ratio: ${aspectRatio}`);
|
||||
}
|
||||
if (widthRatio >= heightRatio) {
|
||||
return {
|
||||
width: edge,
|
||||
height: Math.max(1, Math.round((edge * heightRatio) / widthRatio)),
|
||||
};
|
||||
}
|
||||
return {
|
||||
width: Math.max(1, Math.round((edge * widthRatio) / heightRatio)),
|
||||
height: edge,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveFalImageSize(params: {
|
||||
size?: string;
|
||||
resolution?: "1K" | "2K" | "4K";
|
||||
aspectRatio?: string;
|
||||
hasInputImages: boolean;
|
||||
}): FalImageSize | undefined {
|
||||
const parsed = parseSize(params.size);
|
||||
if (parsed) {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const normalizedAspectRatio = params.aspectRatio?.trim();
|
||||
if (normalizedAspectRatio && params.hasInputImages) {
|
||||
throw new Error("fal image edit endpoint does not support aspectRatio overrides");
|
||||
}
|
||||
|
||||
const edge = mapResolutionToEdge(params.resolution);
|
||||
if (normalizedAspectRatio && edge) {
|
||||
return aspectRatioToDimensions(normalizedAspectRatio, edge);
|
||||
}
|
||||
if (edge) {
|
||||
return { width: edge, height: edge };
|
||||
}
|
||||
if (normalizedAspectRatio) {
|
||||
return (
|
||||
aspectRatioToEnum(normalizedAspectRatio) ??
|
||||
aspectRatioToDimensions(normalizedAspectRatio, 1024)
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function toDataUri(buffer: Buffer, mimeType: string): string {
|
||||
return `data:${mimeType};base64,${buffer.toString("base64")}`;
|
||||
}
|
||||
|
||||
function fileExtensionForMimeType(mimeType: string | undefined): string {
|
||||
const normalized = normalizeOptionalLowercaseString(mimeType);
|
||||
if (!normalized) {
|
||||
return "png";
|
||||
}
|
||||
if (normalized.includes("jpeg")) {
|
||||
return "jpg";
|
||||
}
|
||||
const slashIndex = normalized.indexOf("/");
|
||||
return slashIndex >= 0 ? normalized.slice(slashIndex + 1) || "png" : "png";
|
||||
}
|
||||
|
||||
async function fetchImageBuffer(
|
||||
url: string,
|
||||
networkPolicy?: FalNetworkPolicy,
|
||||
): Promise<{ buffer: Buffer; mimeType: string }> {
|
||||
const downloadPolicy = (() => {
|
||||
const trustedSuffix = networkPolicy?.trustedDownloadHostSuffix;
|
||||
const trustedPolicy = networkPolicy?.trustedDownloadPolicy;
|
||||
if (!trustedSuffix || !trustedPolicy) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return matchesTrustedHostSuffix(parsed.hostname, trustedSuffix) ? trustedPolicy : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
})();
|
||||
const { response, release } = await falFetchGuard({
|
||||
url,
|
||||
policy: downloadPolicy,
|
||||
auditContext: "fal-image-download",
|
||||
});
|
||||
try {
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => "");
|
||||
throw new Error(
|
||||
`fal image download failed (${response.status}): ${text || response.statusText}`,
|
||||
);
|
||||
}
|
||||
const mimeType = response.headers.get("content-type")?.trim() || "image/png";
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return { buffer: Buffer.from(arrayBuffer), mimeType };
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFalImageGenerationProvider(): ImageGenerationProvider {
|
||||
return {
|
||||
id: "fal",
|
||||
label: "fal",
|
||||
defaultModel: DEFAULT_FAL_IMAGE_MODEL,
|
||||
models: [DEFAULT_FAL_IMAGE_MODEL, `${DEFAULT_FAL_IMAGE_MODEL}/${DEFAULT_FAL_EDIT_SUBPATH}`],
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "fal",
|
||||
agentDir,
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 4,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 4,
|
||||
maxInputImages: 1,
|
||||
supportsSize: true,
|
||||
supportsAspectRatio: false,
|
||||
supportsResolution: true,
|
||||
},
|
||||
geometry: {
|
||||
sizes: [...FAL_SUPPORTED_SIZES],
|
||||
aspectRatios: [...FAL_SUPPORTED_ASPECT_RATIOS],
|
||||
resolutions: ["1K", "2K", "4K"],
|
||||
},
|
||||
},
|
||||
async generateImage(req) {
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "fal",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("fal API key missing");
|
||||
}
|
||||
if ((req.inputImages?.length ?? 0) > 1) {
|
||||
throw new Error("fal image generation currently supports at most one reference image");
|
||||
}
|
||||
|
||||
const hasInputImages = (req.inputImages?.length ?? 0) > 0;
|
||||
const imageSize = resolveFalImageSize({
|
||||
size: req.size,
|
||||
resolution: req.resolution,
|
||||
aspectRatio: req.aspectRatio,
|
||||
hasInputImages,
|
||||
});
|
||||
const model = ensureFalModelPath(req.model, hasInputImages);
|
||||
const explicitBaseUrl = req.cfg?.models?.providers?.fal?.baseUrl?.trim();
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: explicitBaseUrl,
|
||||
defaultBaseUrl: DEFAULT_FAL_BASE_URL,
|
||||
allowPrivateNetwork: false,
|
||||
defaultHeaders: {
|
||||
Authorization: `Key ${auth.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
provider: "fal",
|
||||
capability: "image",
|
||||
transport: "http",
|
||||
});
|
||||
const networkPolicy = resolveFalNetworkPolicy({ baseUrl, allowPrivateNetwork });
|
||||
const requestBody: Record<string, unknown> = {
|
||||
prompt: req.prompt,
|
||||
num_images: req.count ?? 1,
|
||||
output_format: DEFAULT_OUTPUT_FORMAT,
|
||||
};
|
||||
if (imageSize !== undefined) {
|
||||
requestBody.image_size = imageSize;
|
||||
}
|
||||
|
||||
if (hasInputImages) {
|
||||
const [input] = req.inputImages ?? [];
|
||||
if (!input) {
|
||||
throw new Error("fal image edit request missing reference image");
|
||||
}
|
||||
requestBody.image_url = toDataUri(input.buffer, input.mimeType);
|
||||
}
|
||||
const { response, release } = await falFetchGuard({
|
||||
url: `${baseUrl}/${model}`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
},
|
||||
timeoutMs: req.timeoutMs,
|
||||
policy: networkPolicy.apiPolicy,
|
||||
dispatcherPolicy,
|
||||
auditContext: "fal-image-generate",
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "fal image generation failed");
|
||||
|
||||
const payload = (await response.json()) as FalImageGenerationResponse;
|
||||
const images: GeneratedImageAsset[] = [];
|
||||
let imageIndex = 0;
|
||||
for (const entry of payload.images ?? []) {
|
||||
const url = entry.url?.trim();
|
||||
if (!url) {
|
||||
continue;
|
||||
}
|
||||
const downloaded = await fetchImageBuffer(url, networkPolicy);
|
||||
imageIndex += 1;
|
||||
images.push({
|
||||
buffer: downloaded.buffer,
|
||||
mimeType: downloaded.mimeType,
|
||||
fileName: `image-${imageIndex}.${fileExtensionForMimeType(
|
||||
downloaded.mimeType || entry.content_type,
|
||||
)}`,
|
||||
});
|
||||
}
|
||||
|
||||
if (images.length === 0) {
|
||||
throw new Error("fal image generation response missing image data");
|
||||
}
|
||||
|
||||
return {
|
||||
images,
|
||||
model,
|
||||
metadata: payload.prompt ? { prompt: payload.prompt } : undefined,
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
17
openclaw/extensions/fal/index.ts
Normal file
17
openclaw/extensions/fal/index.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { buildFalImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { createFalProvider } from "./provider-registration.js";
|
||||
import { buildFalVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
|
||||
const PROVIDER_ID = "fal";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: PROVIDER_ID,
|
||||
name: "fal Provider",
|
||||
description: "Bundled fal image and video generation provider",
|
||||
register(api) {
|
||||
api.registerProvider(createFalProvider());
|
||||
api.registerImageGenerationProvider(buildFalImageGenerationProvider());
|
||||
api.registerVideoGenerationProvider(buildFalVideoGenerationProvider());
|
||||
},
|
||||
});
|
||||
21
openclaw/extensions/fal/onboard.ts
Normal file
21
openclaw/extensions/fal/onboard.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
import type { OpenClawConfig } from "openclaw/plugin-sdk/provider-onboard";
|
||||
|
||||
export const FAL_DEFAULT_IMAGE_MODEL_REF = "fal/fal-ai/flux/dev";
|
||||
|
||||
export function applyFalConfig(cfg: OpenClawConfig): OpenClawConfig {
|
||||
if (cfg.agents?.defaults?.imageGenerationModel) {
|
||||
return cfg;
|
||||
}
|
||||
return {
|
||||
...cfg,
|
||||
agents: {
|
||||
...cfg.agents,
|
||||
defaults: {
|
||||
...cfg.agents?.defaults,
|
||||
imageGenerationModel: {
|
||||
primary: FAL_DEFAULT_IMAGE_MODEL_REF,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
33
openclaw/extensions/fal/openclaw.plugin.json
Normal file
33
openclaw/extensions/fal/openclaw.plugin.json
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
{
|
||||
"id": "fal",
|
||||
"enabledByDefault": true,
|
||||
"providers": ["fal"],
|
||||
"providerAuthEnvVars": {
|
||||
"fal": ["FAL_KEY", "FAL_API_KEY"]
|
||||
},
|
||||
"providerAuthChoices": [
|
||||
{
|
||||
"provider": "fal",
|
||||
"method": "api-key",
|
||||
"choiceId": "fal-api-key",
|
||||
"choiceLabel": "fal API key",
|
||||
"groupId": "fal",
|
||||
"groupLabel": "fal",
|
||||
"groupHint": "Image and video generation",
|
||||
"onboardingScopes": ["image-generation"],
|
||||
"optionKey": "falApiKey",
|
||||
"cliFlag": "--fal-api-key",
|
||||
"cliOption": "--fal-api-key <key>",
|
||||
"cliDescription": "fal API key"
|
||||
}
|
||||
],
|
||||
"contracts": {
|
||||
"imageGenerationProviders": ["fal"],
|
||||
"videoGenerationProviders": ["fal"]
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {}
|
||||
}
|
||||
}
|
||||
15
openclaw/extensions/fal/package.json
Normal file
15
openclaw/extensions/fal/package.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"name": "@openclaw/fal-provider",
|
||||
"version": "2026.4.20",
|
||||
"private": true,
|
||||
"description": "OpenClaw fal provider plugin",
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
10
openclaw/extensions/fal/plugin-registration.contract.test.ts
Normal file
10
openclaw/extensions/fal/plugin-registration.contract.test.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js";
|
||||
|
||||
describePluginRegistrationContract({
|
||||
pluginId: "fal",
|
||||
providerIds: ["fal"],
|
||||
imageGenerationProviderIds: ["fal"],
|
||||
videoGenerationProviderIds: ["fal"],
|
||||
requireGenerateImage: true,
|
||||
requireGenerateVideo: true,
|
||||
});
|
||||
31
openclaw/extensions/fal/provider-contract-api.ts
Normal file
31
openclaw/extensions/fal/provider-contract-api.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
|
||||
const PROVIDER_ID = "fal";
|
||||
const FAL_DEFAULT_IMAGE_MODEL_REF = "fal/fal-ai/flux/dev";
|
||||
|
||||
export function createFalProvider(): ProviderPlugin {
|
||||
return {
|
||||
id: PROVIDER_ID,
|
||||
label: "fal",
|
||||
docsPath: "/providers/models",
|
||||
envVars: ["FAL_KEY"],
|
||||
auth: [
|
||||
{
|
||||
id: "api-key",
|
||||
kind: "api_key",
|
||||
label: "fal API key",
|
||||
hint: "Image and video generation API key",
|
||||
run: async () => ({ profiles: [], defaultModel: FAL_DEFAULT_IMAGE_MODEL_REF }),
|
||||
wizard: {
|
||||
choiceId: "fal-api-key",
|
||||
choiceLabel: "fal API key",
|
||||
choiceHint: "Image and video generation API key",
|
||||
groupId: "fal",
|
||||
groupLabel: "fal",
|
||||
groupHint: "Image and video generation",
|
||||
onboardingScopes: ["image-generation"],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
38
openclaw/extensions/fal/provider-registration.ts
Normal file
38
openclaw/extensions/fal/provider-registration.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { createProviderApiKeyAuthMethod } from "openclaw/plugin-sdk/provider-auth-api-key";
|
||||
import type { ProviderPlugin } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { applyFalConfig, FAL_DEFAULT_IMAGE_MODEL_REF } from "./onboard.js";
|
||||
|
||||
const PROVIDER_ID = "fal";
|
||||
|
||||
export function createFalProvider(): ProviderPlugin {
|
||||
return {
|
||||
id: PROVIDER_ID,
|
||||
label: "fal",
|
||||
docsPath: "/providers/models",
|
||||
envVars: ["FAL_KEY"],
|
||||
auth: [
|
||||
createProviderApiKeyAuthMethod({
|
||||
providerId: PROVIDER_ID,
|
||||
methodId: "api-key",
|
||||
label: "fal API key",
|
||||
hint: "Image and video generation API key",
|
||||
optionKey: "falApiKey",
|
||||
flagName: "--fal-api-key",
|
||||
envVar: "FAL_KEY",
|
||||
promptMessage: "Enter fal API key",
|
||||
defaultModel: FAL_DEFAULT_IMAGE_MODEL_REF,
|
||||
expectedProviders: ["fal"],
|
||||
applyConfig: (cfg) => applyFalConfig(cfg),
|
||||
wizard: {
|
||||
choiceId: "fal-api-key",
|
||||
choiceLabel: "fal API key",
|
||||
choiceHint: "Image and video generation API key",
|
||||
groupId: "fal",
|
||||
groupLabel: "fal",
|
||||
groupHint: "Image and video generation",
|
||||
onboardingScopes: ["image-generation"],
|
||||
},
|
||||
}),
|
||||
],
|
||||
};
|
||||
}
|
||||
2
openclaw/extensions/fal/test-api.ts
Normal file
2
openclaw/extensions/fal/test-api.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
export { buildFalImageGenerationProvider } from "./image-generation-provider.js";
|
||||
export { buildFalVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
16
openclaw/extensions/fal/tsconfig.json
Normal file
16
openclaw/extensions/fal/tsconfig.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"extends": "../tsconfig.package-boundary.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["./*.ts", "./src/**/*.ts"],
|
||||
"exclude": [
|
||||
"./**/*.test.ts",
|
||||
"./dist/**",
|
||||
"./node_modules/**",
|
||||
"./src/test-support/**",
|
||||
"./src/**/*test-helpers.ts",
|
||||
"./src/**/*test-harness.ts",
|
||||
"./src/**/*test-support.ts"
|
||||
]
|
||||
}
|
||||
317
openclaw/extensions/fal/video-generation-provider.test.ts
Normal file
317
openclaw/extensions/fal/video-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import * as providerHttp from "openclaw/plugin-sdk/provider-http";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { expectExplicitVideoGenerationCapabilities } from "../../test/helpers/media-generation/provider-capability-assertions.js";
|
||||
import {
|
||||
_setFalVideoFetchGuardForTesting,
|
||||
buildFalVideoGenerationProvider,
|
||||
} from "./video-generation-provider.js";
|
||||
|
||||
function createMockRequestConfig() {
|
||||
return {} as ReturnType<typeof providerHttp.resolveProviderHttpRequestConfig>["requestConfig"];
|
||||
}
|
||||
describe("fal video generation provider", () => {
|
||||
const fetchGuardMock = vi.fn();
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
fetchGuardMock.mockReset();
|
||||
_setFalVideoFetchGuardForTesting(null);
|
||||
});
|
||||
|
||||
it("declares explicit mode capabilities", () => {
|
||||
expectExplicitVideoGenerationCapabilities(buildFalVideoGenerationProvider());
|
||||
});
|
||||
|
||||
it("submits fal video jobs through the queue API and downloads the completed result", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
vi.spyOn(providerHttp, "resolveProviderHttpRequestConfig").mockReturnValue({
|
||||
baseUrl: "https://fal.run",
|
||||
allowPrivateNetwork: false,
|
||||
headers: new Headers({
|
||||
Authorization: "Key fal-key",
|
||||
"Content-Type": "application/json",
|
||||
}),
|
||||
dispatcherPolicy: undefined,
|
||||
requestConfig: createMockRequestConfig(),
|
||||
});
|
||||
vi.spyOn(providerHttp, "assertOkOrThrowHttpError").mockResolvedValue(undefined);
|
||||
_setFalVideoFetchGuardForTesting(fetchGuardMock as never);
|
||||
fetchGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
request_id: "req-123",
|
||||
status_url: "https://queue.fal.run/fal-ai/minimax/requests/req-123/status",
|
||||
response_url: "https://queue.fal.run/fal-ai/minimax/requests/req-123",
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
status: "COMPLETED",
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
status: "COMPLETED",
|
||||
response: {
|
||||
video: { url: "https://fal.run/files/video.mp4" },
|
||||
},
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("mp4-bytes"),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildFalVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "fal",
|
||||
model: "fal-ai/minimax/video-01-live",
|
||||
prompt: "A spaceship emerges from the clouds",
|
||||
durationSeconds: 5,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "720P",
|
||||
cfg: {},
|
||||
});
|
||||
|
||||
expect(fetchGuardMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "https://queue.fal.run/fal-ai/minimax/video-01-live",
|
||||
}),
|
||||
);
|
||||
const submitBody = JSON.parse(
|
||||
String(fetchGuardMock.mock.calls[0]?.[0]?.init?.body ?? "{}"),
|
||||
) as Record<string, unknown>;
|
||||
expect(submitBody).toEqual({
|
||||
prompt: "A spaceship emerges from the clouds",
|
||||
});
|
||||
expect(fetchGuardMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
url: "https://queue.fal.run/fal-ai/minimax/requests/req-123/status",
|
||||
}),
|
||||
);
|
||||
expect(fetchGuardMock).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
url: "https://queue.fal.run/fal-ai/minimax/requests/req-123",
|
||||
}),
|
||||
);
|
||||
expect(result.videos).toHaveLength(1);
|
||||
expect(result.videos[0]?.mimeType).toBe("video/mp4");
|
||||
expect(result.metadata).toEqual({
|
||||
requestId: "req-123",
|
||||
});
|
||||
});
|
||||
|
||||
it("exposes Seedance 2 models", () => {
|
||||
const provider = buildFalVideoGenerationProvider();
|
||||
|
||||
expect(provider.models).toEqual(
|
||||
expect.arrayContaining([
|
||||
"fal-ai/heygen/v2/video-agent",
|
||||
"bytedance/seedance-2.0/fast/text-to-video",
|
||||
"bytedance/seedance-2.0/fast/image-to-video",
|
||||
"bytedance/seedance-2.0/text-to-video",
|
||||
"bytedance/seedance-2.0/image-to-video",
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("submits HeyGen video-agent requests without unsupported fal controls", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
vi.spyOn(providerHttp, "resolveProviderHttpRequestConfig").mockReturnValue({
|
||||
baseUrl: "https://fal.run",
|
||||
allowPrivateNetwork: false,
|
||||
headers: new Headers({
|
||||
Authorization: "Key fal-key",
|
||||
"Content-Type": "application/json",
|
||||
}),
|
||||
dispatcherPolicy: undefined,
|
||||
requestConfig: createMockRequestConfig(),
|
||||
});
|
||||
vi.spyOn(providerHttp, "assertOkOrThrowHttpError").mockResolvedValue(undefined);
|
||||
_setFalVideoFetchGuardForTesting(fetchGuardMock as never);
|
||||
fetchGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
request_id: "heygen-req-123",
|
||||
status_url:
|
||||
"https://queue.fal.run/fal-ai/heygen/v2/video-agent/requests/heygen-req-123/status",
|
||||
response_url:
|
||||
"https://queue.fal.run/fal-ai/heygen/v2/video-agent/requests/heygen-req-123",
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
status: "COMPLETED",
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
status: "COMPLETED",
|
||||
response: {
|
||||
video: { url: "https://fal.run/files/heygen.mp4" },
|
||||
},
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("heygen-mp4-bytes"),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildFalVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "fal",
|
||||
model: "fal-ai/heygen/v2/video-agent",
|
||||
prompt: "A founder explains OpenClaw in a concise studio video",
|
||||
durationSeconds: 8,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "720P",
|
||||
audio: true,
|
||||
cfg: {},
|
||||
});
|
||||
|
||||
expect(fetchGuardMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "https://queue.fal.run/fal-ai/heygen/v2/video-agent",
|
||||
}),
|
||||
);
|
||||
const submitBody = JSON.parse(
|
||||
String(fetchGuardMock.mock.calls[0]?.[0]?.init?.body ?? "{}"),
|
||||
) as Record<string, unknown>;
|
||||
expect(submitBody).toEqual({
|
||||
prompt: "A founder explains OpenClaw in a concise studio video",
|
||||
});
|
||||
expect(result.metadata).toEqual({
|
||||
requestId: "heygen-req-123",
|
||||
});
|
||||
});
|
||||
|
||||
it("submits Seedance 2 requests with fal schema fields", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "fal-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
vi.spyOn(providerHttp, "resolveProviderHttpRequestConfig").mockReturnValue({
|
||||
baseUrl: "https://fal.run",
|
||||
allowPrivateNetwork: false,
|
||||
headers: new Headers({
|
||||
Authorization: "Key fal-key",
|
||||
"Content-Type": "application/json",
|
||||
}),
|
||||
dispatcherPolicy: undefined,
|
||||
requestConfig: createMockRequestConfig(),
|
||||
});
|
||||
vi.spyOn(providerHttp, "assertOkOrThrowHttpError").mockResolvedValue(undefined);
|
||||
_setFalVideoFetchGuardForTesting(fetchGuardMock as never);
|
||||
fetchGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
request_id: "seedance-req-123",
|
||||
status_url:
|
||||
"https://queue.fal.run/bytedance/seedance-2.0/fast/text-to-video/requests/seedance-req-123/status",
|
||||
response_url:
|
||||
"https://queue.fal.run/bytedance/seedance-2.0/fast/text-to-video/requests/seedance-req-123",
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
status: "COMPLETED",
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
json: async () => ({
|
||||
status: "COMPLETED",
|
||||
response: {
|
||||
video: { url: "https://fal.run/files/seedance.mp4" },
|
||||
seed: 42,
|
||||
},
|
||||
}),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: {
|
||||
headers: new Headers({ "content-type": "video/mp4" }),
|
||||
arrayBuffer: async () => Buffer.from("seedance-mp4-bytes"),
|
||||
},
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildFalVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "fal",
|
||||
model: "bytedance/seedance-2.0/fast/text-to-video",
|
||||
prompt: "A chrome lobster drives a tiny kart across a neon pier",
|
||||
durationSeconds: 7,
|
||||
aspectRatio: "16:9",
|
||||
resolution: "720P",
|
||||
audio: false,
|
||||
cfg: {},
|
||||
});
|
||||
|
||||
expect(fetchGuardMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "https://queue.fal.run/bytedance/seedance-2.0/fast/text-to-video",
|
||||
}),
|
||||
);
|
||||
const submitBody = JSON.parse(
|
||||
String(fetchGuardMock.mock.calls[0]?.[0]?.init?.body ?? "{}"),
|
||||
) as Record<string, unknown>;
|
||||
expect(submitBody).toEqual({
|
||||
prompt: "A chrome lobster drives a tiny kart across a neon pier",
|
||||
aspect_ratio: "16:9",
|
||||
resolution: "720p",
|
||||
duration: "7",
|
||||
generate_audio: false,
|
||||
});
|
||||
expect(result.metadata).toEqual({
|
||||
requestId: "seedance-req-123",
|
||||
seed: 42,
|
||||
});
|
||||
});
|
||||
});
|
||||
410
openclaw/extensions/fal/video-generation-provider.ts
Normal file
410
openclaw/extensions/fal/video-generation-provider.ts
Normal file
|
|
@ -0,0 +1,410 @@
|
|||
import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
resolveProviderHttpRequestConfig,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
fetchWithSsrFGuard,
|
||||
type SsrFPolicy,
|
||||
ssrfPolicyFromDangerouslyAllowPrivateNetwork,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "openclaw/plugin-sdk/text-runtime";
|
||||
import type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationRequest,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
|
||||
const DEFAULT_FAL_BASE_URL = "https://fal.run";
|
||||
const DEFAULT_FAL_QUEUE_BASE_URL = "https://queue.fal.run";
|
||||
const DEFAULT_FAL_VIDEO_MODEL = "fal-ai/minimax/video-01-live";
|
||||
const HEYGEN_VIDEO_AGENT_MODEL = "fal-ai/heygen/v2/video-agent";
|
||||
const SEEDANCE_2_VIDEO_MODELS = [
|
||||
"bytedance/seedance-2.0/fast/text-to-video",
|
||||
"bytedance/seedance-2.0/fast/image-to-video",
|
||||
"bytedance/seedance-2.0/text-to-video",
|
||||
"bytedance/seedance-2.0/image-to-video",
|
||||
] as const;
|
||||
const SEEDANCE_2_DURATION_SECONDS = [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] as const;
|
||||
const DEFAULT_HTTP_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_OPERATION_TIMEOUT_MS = 600_000;
|
||||
const POLL_INTERVAL_MS = 5_000;
|
||||
|
||||
type FalVideoResponse = {
|
||||
video?: {
|
||||
url?: string;
|
||||
content_type?: string;
|
||||
};
|
||||
videos?: Array<{
|
||||
url?: string;
|
||||
content_type?: string;
|
||||
}>;
|
||||
prompt?: string;
|
||||
seed?: number;
|
||||
};
|
||||
|
||||
type FalQueueResponse = {
|
||||
status?: string;
|
||||
request_id?: string;
|
||||
response_url?: string;
|
||||
status_url?: string;
|
||||
cancel_url?: string;
|
||||
detail?: string;
|
||||
response?: FalVideoResponse;
|
||||
prompt?: string;
|
||||
error?: {
|
||||
message?: string;
|
||||
};
|
||||
};
|
||||
|
||||
let falFetchGuard = fetchWithSsrFGuard;
|
||||
|
||||
export function _setFalVideoFetchGuardForTesting(impl: typeof fetchWithSsrFGuard | null): void {
|
||||
falFetchGuard = impl ?? fetchWithSsrFGuard;
|
||||
}
|
||||
|
||||
function toDataUrl(buffer: Buffer, mimeType: string): string {
|
||||
return `data:${mimeType};base64,${buffer.toString("base64")}`;
|
||||
}
|
||||
|
||||
function buildPolicy(allowPrivateNetwork: boolean): SsrFPolicy | undefined {
|
||||
return allowPrivateNetwork ? ssrfPolicyFromDangerouslyAllowPrivateNetwork(true) : undefined;
|
||||
}
|
||||
|
||||
function extractFalVideoEntry(payload: FalVideoResponse) {
|
||||
if (normalizeOptionalString(payload.video?.url)) {
|
||||
return payload.video;
|
||||
}
|
||||
return payload.videos?.find((entry) => normalizeOptionalString(entry.url));
|
||||
}
|
||||
|
||||
async function downloadFalVideo(
|
||||
url: string,
|
||||
policy: SsrFPolicy | undefined,
|
||||
): Promise<GeneratedVideoAsset> {
|
||||
const { response, release } = await falFetchGuard({
|
||||
url,
|
||||
timeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
|
||||
policy,
|
||||
auditContext: "fal-video-download",
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, "fal generated video download failed");
|
||||
const mimeType = normalizeOptionalString(response.headers.get("content-type")) ?? "video/mp4";
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
return {
|
||||
buffer: Buffer.from(arrayBuffer),
|
||||
mimeType,
|
||||
fileName: `video-1.${mimeType.includes("webm") ? "webm" : "mp4"}`,
|
||||
};
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
function resolveFalQueueBaseUrl(baseUrl: string): string {
|
||||
try {
|
||||
const url = new URL(baseUrl);
|
||||
if (url.hostname === "fal.run") {
|
||||
url.hostname = "queue.fal.run";
|
||||
return url.toString().replace(/\/$/, "");
|
||||
}
|
||||
return baseUrl.replace(/\/$/, "");
|
||||
} catch {
|
||||
return DEFAULT_FAL_QUEUE_BASE_URL;
|
||||
}
|
||||
}
|
||||
|
||||
function isFalMiniMaxLiveModel(model: string): boolean {
|
||||
return normalizeLowercaseStringOrEmpty(model) === DEFAULT_FAL_VIDEO_MODEL;
|
||||
}
|
||||
|
||||
function isFalSeedance2Model(model: string): boolean {
|
||||
return SEEDANCE_2_VIDEO_MODELS.includes(model as (typeof SEEDANCE_2_VIDEO_MODELS)[number]);
|
||||
}
|
||||
|
||||
function isFalHeyGenVideoAgentModel(model: string): boolean {
|
||||
return normalizeLowercaseStringOrEmpty(model) === HEYGEN_VIDEO_AGENT_MODEL;
|
||||
}
|
||||
|
||||
function resolveFalResolution(resolution: VideoGenerationRequest["resolution"], model: string) {
|
||||
if (!resolution) {
|
||||
return undefined;
|
||||
}
|
||||
if (isFalSeedance2Model(model)) {
|
||||
return resolution.toLowerCase();
|
||||
}
|
||||
return resolution;
|
||||
}
|
||||
|
||||
function resolveFalDuration(
|
||||
durationSeconds: number | undefined,
|
||||
model: string,
|
||||
): number | string | undefined {
|
||||
if (typeof durationSeconds !== "number" || !Number.isFinite(durationSeconds)) {
|
||||
return undefined;
|
||||
}
|
||||
const duration = Math.max(1, Math.round(durationSeconds));
|
||||
if (isFalSeedance2Model(model)) {
|
||||
return String(duration);
|
||||
}
|
||||
return duration;
|
||||
}
|
||||
|
||||
function buildFalVideoRequestBody(params: {
|
||||
req: VideoGenerationRequest;
|
||||
model: string;
|
||||
}): Record<string, unknown> {
|
||||
const requestBody: Record<string, unknown> = {
|
||||
prompt: params.req.prompt,
|
||||
};
|
||||
const input = params.req.inputImages?.[0];
|
||||
if (input) {
|
||||
requestBody.image_url = normalizeOptionalString(input.url)
|
||||
? normalizeOptionalString(input.url)
|
||||
: input.buffer
|
||||
? toDataUrl(input.buffer, normalizeOptionalString(input.mimeType) ?? "image/png")
|
||||
: undefined;
|
||||
}
|
||||
// MiniMax Live on fal currently documents prompt + optional image_url only.
|
||||
// Keep the default model conservative so queue requests do not hang behind
|
||||
// unsupported knobs such as duration/resolution/aspect-ratio overrides.
|
||||
if (isFalMiniMaxLiveModel(params.model) || isFalHeyGenVideoAgentModel(params.model)) {
|
||||
return requestBody;
|
||||
}
|
||||
const aspectRatio = normalizeOptionalString(params.req.aspectRatio);
|
||||
if (aspectRatio) {
|
||||
requestBody.aspect_ratio = aspectRatio;
|
||||
}
|
||||
const size = normalizeOptionalString(params.req.size);
|
||||
if (size) {
|
||||
requestBody.size = size;
|
||||
}
|
||||
const resolution = resolveFalResolution(params.req.resolution, params.model);
|
||||
if (resolution) {
|
||||
requestBody.resolution = resolution;
|
||||
}
|
||||
const duration = resolveFalDuration(params.req.durationSeconds, params.model);
|
||||
if (duration) {
|
||||
requestBody.duration = duration;
|
||||
}
|
||||
if (isFalSeedance2Model(params.model) && typeof params.req.audio === "boolean") {
|
||||
requestBody.generate_audio = params.req.audio;
|
||||
}
|
||||
return requestBody;
|
||||
}
|
||||
|
||||
async function fetchFalJson(params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
timeoutMs: number;
|
||||
policy: SsrFPolicy | undefined;
|
||||
dispatcherPolicy: Parameters<typeof fetchWithSsrFGuard>[0]["dispatcherPolicy"];
|
||||
auditContext: string;
|
||||
errorContext: string;
|
||||
}): Promise<unknown> {
|
||||
const { response, release } = await falFetchGuard({
|
||||
url: params.url,
|
||||
init: params.init,
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: params.auditContext,
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, params.errorContext);
|
||||
return await response.json();
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForFalQueueResult(params: {
|
||||
statusUrl: string;
|
||||
responseUrl: string;
|
||||
headers: Headers;
|
||||
timeoutMs: number;
|
||||
policy: SsrFPolicy | undefined;
|
||||
dispatcherPolicy: Parameters<typeof fetchWithSsrFGuard>[0]["dispatcherPolicy"];
|
||||
}): Promise<FalQueueResponse> {
|
||||
const deadline = Date.now() + params.timeoutMs;
|
||||
let lastStatus = "unknown";
|
||||
while (Date.now() < deadline) {
|
||||
const payload = (await fetchFalJson({
|
||||
url: params.statusUrl,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
},
|
||||
timeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: "fal-video-status",
|
||||
errorContext: "fal video status request failed",
|
||||
})) as FalQueueResponse;
|
||||
const status = normalizeOptionalString(payload.status)?.toUpperCase();
|
||||
if (status) {
|
||||
lastStatus = status;
|
||||
}
|
||||
if (status === "COMPLETED") {
|
||||
return (await fetchFalJson({
|
||||
url: params.responseUrl,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
},
|
||||
timeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: "fal-video-result",
|
||||
errorContext: "fal video result request failed",
|
||||
})) as FalQueueResponse;
|
||||
}
|
||||
if (status === "FAILED" || status === "CANCELLED") {
|
||||
throw new Error(
|
||||
normalizeOptionalString(payload.detail) ||
|
||||
normalizeOptionalString(payload.error?.message) ||
|
||||
`fal video generation ${normalizeLowercaseStringOrEmpty(status)}`,
|
||||
);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
|
||||
}
|
||||
throw new Error(`fal video generation did not finish in time (last status: ${lastStatus})`);
|
||||
}
|
||||
|
||||
function extractFalVideoPayload(payload: FalQueueResponse): FalVideoResponse {
|
||||
if (payload.response && typeof payload.response === "object") {
|
||||
return payload.response;
|
||||
}
|
||||
return payload as FalVideoResponse;
|
||||
}
|
||||
|
||||
export function buildFalVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
id: "fal",
|
||||
label: "fal",
|
||||
defaultModel: DEFAULT_FAL_VIDEO_MODEL,
|
||||
models: [
|
||||
DEFAULT_FAL_VIDEO_MODEL,
|
||||
HEYGEN_VIDEO_AGENT_MODEL,
|
||||
...SEEDANCE_2_VIDEO_MODELS,
|
||||
"fal-ai/kling-video/v2.1/master/text-to-video",
|
||||
"fal-ai/wan/v2.2-a14b/text-to-video",
|
||||
"fal-ai/wan/v2.2-a14b/image-to-video",
|
||||
],
|
||||
isConfigured: ({ agentDir }) =>
|
||||
isProviderApiKeyConfigured({
|
||||
provider: "fal",
|
||||
agentDir,
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
supportedDurationSecondsByModel: Object.fromEntries(
|
||||
SEEDANCE_2_VIDEO_MODELS.map((model) => [model, SEEDANCE_2_DURATION_SECONDS]),
|
||||
),
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
supportedDurationSecondsByModel: Object.fromEntries(
|
||||
SEEDANCE_2_VIDEO_MODELS.map((model) => [model, SEEDANCE_2_DURATION_SECONDS]),
|
||||
),
|
||||
supportsAspectRatio: true,
|
||||
supportsResolution: true,
|
||||
supportsSize: true,
|
||||
supportsAudio: true,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
async generateVideo(req) {
|
||||
if ((req.inputVideos?.length ?? 0) > 0) {
|
||||
throw new Error("fal video generation does not support video reference inputs.");
|
||||
}
|
||||
if ((req.inputImages?.length ?? 0) > 1) {
|
||||
throw new Error("fal video generation supports at most one image reference.");
|
||||
}
|
||||
const auth = await resolveApiKeyForProvider({
|
||||
provider: "fal",
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
store: req.authStore,
|
||||
});
|
||||
if (!auth.apiKey) {
|
||||
throw new Error("fal API key missing");
|
||||
}
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: normalizeOptionalString(req.cfg?.models?.providers?.fal?.baseUrl),
|
||||
defaultBaseUrl: DEFAULT_FAL_BASE_URL,
|
||||
allowPrivateNetwork: false,
|
||||
defaultHeaders: {
|
||||
Authorization: `Key ${auth.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
provider: "fal",
|
||||
capability: "video",
|
||||
transport: "http",
|
||||
});
|
||||
const model = normalizeOptionalString(req.model) || DEFAULT_FAL_VIDEO_MODEL;
|
||||
const requestBody = buildFalVideoRequestBody({ req, model });
|
||||
const policy = buildPolicy(allowPrivateNetwork);
|
||||
const queueBaseUrl = resolveFalQueueBaseUrl(baseUrl);
|
||||
const submitted = (await fetchFalJson({
|
||||
url: `${queueBaseUrl}/${model}`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
},
|
||||
timeoutMs: DEFAULT_HTTP_TIMEOUT_MS,
|
||||
policy,
|
||||
dispatcherPolicy,
|
||||
auditContext: "fal-video-submit",
|
||||
errorContext: "fal video generation failed",
|
||||
})) as FalQueueResponse;
|
||||
const statusUrl = normalizeOptionalString(submitted.status_url);
|
||||
const responseUrl = normalizeOptionalString(submitted.response_url);
|
||||
if (!statusUrl || !responseUrl) {
|
||||
throw new Error("fal video generation response missing queue URLs");
|
||||
}
|
||||
const payload = await waitForFalQueueResult({
|
||||
statusUrl,
|
||||
responseUrl,
|
||||
headers,
|
||||
timeoutMs: req.timeoutMs ?? DEFAULT_OPERATION_TIMEOUT_MS,
|
||||
policy,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
const videoPayload = extractFalVideoPayload(payload);
|
||||
const entry = extractFalVideoEntry(videoPayload);
|
||||
const url = normalizeOptionalString(entry?.url);
|
||||
if (!url) {
|
||||
throw new Error("fal video generation response missing output URL");
|
||||
}
|
||||
const video = await downloadFalVideo(url, policy);
|
||||
return {
|
||||
videos: [video],
|
||||
model,
|
||||
metadata: {
|
||||
...(normalizeOptionalString(submitted.request_id)
|
||||
? { requestId: normalizeOptionalString(submitted.request_id) }
|
||||
: {}),
|
||||
...(videoPayload.prompt ? { prompt: videoPayload.prompt } : {}),
|
||||
...(typeof videoPayload.seed === "number" ? { seed: videoPayload.seed } : {}),
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue