mirror of
https://github.com/hansjone/oclaw.git
synced 2026-10-11 17:43:56 +08:00
重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。
同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。 Made-with: Cursor
This commit is contained in:
parent
4a23b715a2
commit
dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions
122
openclaw/extensions/comfy/comfy.live.test.ts
Normal file
122
openclaw/extensions/comfy/comfy.live.test.ts
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
import { resolveOpenClawAgentDir } from "../../src/agents/agent-paths.js";
|
||||
import { isLiveTestEnabled } from "../../src/agents/live-test-helpers.js";
|
||||
import { loadConfig } from "../../src/config/config.js";
|
||||
import { createTestPluginApi } from "../../test/helpers/plugins/plugin-api.js";
|
||||
import plugin from "./index.js";
|
||||
import { getComfyConfig, isComfyCapabilityConfigured } from "./workflow-runtime.js";
|
||||
|
||||
const LIVE =
|
||||
isLiveTestEnabled(["COMFY_LIVE_TEST"]) && (process.env.COMFY_LIVE_TEST ?? "").trim() === "1";
|
||||
const describeLive = LIVE ? describe : describe.skip;
|
||||
|
||||
type RegisteredMusicProvider = {
|
||||
id: string;
|
||||
generateMusic: Function;
|
||||
isConfigured?: Function;
|
||||
};
|
||||
|
||||
function withPluginsEnabled<T>(cfg: T): T {
|
||||
if (!cfg || typeof cfg !== "object") {
|
||||
return cfg;
|
||||
}
|
||||
const record = cfg as Record<string, unknown>;
|
||||
return {
|
||||
...record,
|
||||
plugins: {
|
||||
...(record.plugins && typeof record.plugins === "object" ? record.plugins : {}),
|
||||
enabled: true,
|
||||
},
|
||||
} as T;
|
||||
}
|
||||
|
||||
describeLive("comfy live", () => {
|
||||
let cfg = {} as ReturnType<typeof loadConfig>;
|
||||
let agentDir = "";
|
||||
const imageProviders: Array<{ id: string; generateImage: Function; isConfigured?: Function }> =
|
||||
[];
|
||||
const musicProviders: RegisteredMusicProvider[] = [];
|
||||
const videoProviders: Array<{ id: string; generateVideo: Function; isConfigured?: Function }> =
|
||||
[];
|
||||
|
||||
beforeAll(async () => {
|
||||
cfg = withPluginsEnabled(loadConfig());
|
||||
agentDir = resolveOpenClawAgentDir();
|
||||
plugin.register(
|
||||
createTestPluginApi({
|
||||
config: cfg as never,
|
||||
registerImageGenerationProvider(provider) {
|
||||
imageProviders.push(provider as never);
|
||||
},
|
||||
registerMusicGenerationProvider(provider) {
|
||||
musicProviders.push(provider as never);
|
||||
},
|
||||
registerVideoGenerationProvider(provider) {
|
||||
videoProviders.push(provider as never);
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.skipIf(!isComfyCapabilityConfigured({ cfg: cfg as never, agentDir, capability: "image" }))(
|
||||
"runs an image workflow",
|
||||
async () => {
|
||||
const provider = imageProviders.find((entry) => entry.id === "comfy");
|
||||
expect(provider).toBeDefined();
|
||||
const result = await provider!.generateImage({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "A tiny orange lobster icon on a clean background.",
|
||||
cfg: cfg as never,
|
||||
agentDir,
|
||||
});
|
||||
expect(result.images.length).toBeGreaterThan(0);
|
||||
expect(result.images[0]?.mimeType.startsWith("image/")).toBe(true);
|
||||
expect(result.images[0]?.buffer.byteLength).toBeGreaterThan(128);
|
||||
},
|
||||
120_000,
|
||||
);
|
||||
|
||||
it.skipIf(!isComfyCapabilityConfigured({ cfg: cfg as never, agentDir, capability: "video" }))(
|
||||
"runs a video workflow",
|
||||
async () => {
|
||||
const provider = videoProviders.find((entry) => entry.id === "comfy");
|
||||
expect(provider).toBeDefined();
|
||||
const result = await provider!.generateVideo({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "A tiny paper lobster gently waving, cinematic motion.",
|
||||
cfg: cfg as never,
|
||||
agentDir,
|
||||
});
|
||||
expect(result.videos.length).toBeGreaterThan(0);
|
||||
expect(result.videos[0]?.mimeType.startsWith("video/")).toBe(true);
|
||||
expect(result.videos[0]?.buffer.byteLength).toBeGreaterThan(512);
|
||||
},
|
||||
180_000,
|
||||
);
|
||||
|
||||
it.skipIf(!isComfyCapabilityConfigured({ cfg: cfg as never, agentDir, capability: "music" }))(
|
||||
"runs a music workflow",
|
||||
async () => {
|
||||
const provider = musicProviders.find((entry) => entry.id === "comfy");
|
||||
expect(provider).toBeDefined();
|
||||
const result = await provider!.generateMusic({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "A gentle ambient synth loop with warm analog pads.",
|
||||
cfg: cfg as never,
|
||||
agentDir,
|
||||
});
|
||||
expect(result.tracks.length).toBeGreaterThan(0);
|
||||
expect(result.tracks[0]?.mimeType.startsWith("audio/")).toBe(true);
|
||||
expect(result.tracks[0]?.buffer.byteLength).toBeGreaterThan(512);
|
||||
},
|
||||
180_000,
|
||||
);
|
||||
|
||||
it("documents the effective comfy config shape for live debugging", () => {
|
||||
const comfyConfig = getComfyConfig(cfg as never);
|
||||
expect(typeof comfyConfig).toBe("object");
|
||||
});
|
||||
});
|
||||
355
openclaw/extensions/comfy/image-generation-provider.test.ts
Normal file
355
openclaw/extensions/comfy/image-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
_setComfyFetchGuardForTesting,
|
||||
buildComfyImageGenerationProvider,
|
||||
} from "./image-generation-provider.js";
|
||||
|
||||
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuardMock: vi.fn(),
|
||||
}));
|
||||
|
||||
function parseJsonBody(call: number): Record<string, unknown> {
|
||||
const request = fetchWithSsrFGuardMock.mock.calls[call - 1]?.[0];
|
||||
expect(request?.init?.body).toBeTruthy();
|
||||
return JSON.parse(String(request.init.body)) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function buildComfyConfig(config: Record<string, unknown>): OpenClawConfig {
|
||||
return {
|
||||
models: {
|
||||
providers: {
|
||||
comfy: config,
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
}
|
||||
|
||||
describe("comfy image-generation provider", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
_setComfyFetchGuardForTesting(null);
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("treats local comfy workflows as configured without an API key", () => {
|
||||
const provider = buildComfyImageGenerationProvider();
|
||||
expect(
|
||||
provider.isConfigured?.({
|
||||
cfg: buildComfyConfig({
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
}),
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("submits a local workflow, waits for history, and downloads images", async () => {
|
||||
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ prompt_id: "local-prompt-1" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
"local-prompt-1": {
|
||||
outputs: {
|
||||
"9": {
|
||||
images: [{ filename: "generated.png", subfolder: "", type: "output" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("png-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildComfyImageGenerationProvider();
|
||||
const result = await provider.generateImage({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "draw a lobster",
|
||||
cfg: buildComfyConfig({
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
outputNodeId: "9",
|
||||
}),
|
||||
});
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8188/prompt",
|
||||
auditContext: "comfy-image-generate",
|
||||
}),
|
||||
);
|
||||
expect(parseJsonBody(1)).toEqual({
|
||||
prompt: {
|
||||
"6": { inputs: { text: "draw a lobster" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
});
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8188/history/local-prompt-1",
|
||||
auditContext: "comfy-history",
|
||||
}),
|
||||
);
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8188/view?filename=generated.png&subfolder=&type=output",
|
||||
auditContext: "comfy-image-download",
|
||||
}),
|
||||
);
|
||||
expect(result).toEqual({
|
||||
images: [
|
||||
{
|
||||
buffer: Buffer.from("png-data"),
|
||||
mimeType: "image/png",
|
||||
fileName: "generated.png",
|
||||
metadata: {
|
||||
nodeId: "9",
|
||||
promptId: "local-prompt-1",
|
||||
},
|
||||
},
|
||||
],
|
||||
model: "workflow",
|
||||
metadata: {
|
||||
promptId: "local-prompt-1",
|
||||
outputNodeIds: ["9"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("uploads reference images for local edit workflows", async () => {
|
||||
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ name: "upload.png" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ prompt_id: "local-edit-1" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
"local-edit-1": {
|
||||
outputs: {
|
||||
"9": {
|
||||
images: [{ filename: "edited.png", subfolder: "", type: "output" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("edited-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildComfyImageGenerationProvider();
|
||||
await provider.generateImage({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "turn this into a poster",
|
||||
cfg: buildComfyConfig({
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
"7": { inputs: { image: "" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
inputImageNodeId: "7",
|
||||
outputNodeId: "9",
|
||||
}),
|
||||
inputImages: [
|
||||
{
|
||||
buffer: Buffer.from("source"),
|
||||
mimeType: "image/png",
|
||||
fileName: "source.png",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const uploadRequest = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
|
||||
expect(uploadRequest?.url).toBe("http://127.0.0.1:8188/upload/image");
|
||||
expect(uploadRequest?.auditContext).toBe("comfy-image-upload");
|
||||
expect(uploadRequest?.init?.method).toBe("POST");
|
||||
const uploadForm = uploadRequest?.init?.body;
|
||||
expect(uploadForm).toBeInstanceOf(FormData);
|
||||
expect(uploadForm?.get("type")).toBe("input");
|
||||
expect(uploadForm?.get("overwrite")).toBe("true");
|
||||
|
||||
expect(parseJsonBody(2)).toEqual({
|
||||
prompt: {
|
||||
"6": { inputs: { text: "turn this into a poster" } },
|
||||
"7": { inputs: { image: "upload.png" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("uses cloud endpoints, auth headers, and partner-node extra_data", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "comfy-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ prompt_id: "cloud-job-1" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ status: "completed" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
"cloud-job-1": {
|
||||
outputs: {
|
||||
"9": {
|
||||
images: [{ filename: "cloud.png", subfolder: "", type: "output" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "https://cdn.example.com/cloud.png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("cloud-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "image/png" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildComfyImageGenerationProvider();
|
||||
const result = await provider.generateImage({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "cloud workflow prompt",
|
||||
cfg: buildComfyConfig({
|
||||
mode: "cloud",
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
outputNodeId: "9",
|
||||
}),
|
||||
});
|
||||
|
||||
const submitRequest = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
|
||||
expect(submitRequest?.url).toBe("https://cloud.comfy.org/api/prompt");
|
||||
expect(submitRequest?.auditContext).toBe("comfy-image-generate");
|
||||
const submitHeaders = new Headers(submitRequest?.init?.headers);
|
||||
expect(submitHeaders.get("x-api-key")).toBe("comfy-test-key");
|
||||
expect(parseJsonBody(1)).toEqual({
|
||||
prompt: {
|
||||
"6": { inputs: { text: "cloud workflow prompt" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
extra_data: {
|
||||
api_key_comfy_org: "comfy-test-key",
|
||||
},
|
||||
});
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
url: "https://cloud.comfy.org/api/job/cloud-job-1/status",
|
||||
auditContext: "comfy-status",
|
||||
}),
|
||||
);
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
url: "https://cloud.comfy.org/api/history_v2/cloud-job-1",
|
||||
auditContext: "comfy-history",
|
||||
}),
|
||||
);
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
4,
|
||||
expect.objectContaining({
|
||||
url: "https://cloud.comfy.org/api/view?filename=cloud.png&subfolder=&type=output",
|
||||
auditContext: "comfy-image-download",
|
||||
}),
|
||||
);
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
5,
|
||||
expect.objectContaining({
|
||||
url: "https://cdn.example.com/cloud.png",
|
||||
auditContext: "comfy-image-download",
|
||||
}),
|
||||
);
|
||||
expect(result.metadata).toEqual({
|
||||
promptId: "cloud-job-1",
|
||||
outputNodeIds: ["9"],
|
||||
});
|
||||
});
|
||||
});
|
||||
79
openclaw/extensions/comfy/image-generation-provider.ts
Normal file
79
openclaw/extensions/comfy/image-generation-provider.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import type {
|
||||
GeneratedImageAsset,
|
||||
ImageGenerationProvider,
|
||||
} from "openclaw/plugin-sdk/image-generation";
|
||||
import {
|
||||
DEFAULT_COMFY_MODEL,
|
||||
_setComfyFetchGuardForTesting,
|
||||
isComfyCapabilityConfigured,
|
||||
runComfyWorkflow,
|
||||
} from "./workflow-runtime.js";
|
||||
|
||||
export { _setComfyFetchGuardForTesting };
|
||||
|
||||
export function buildComfyImageGenerationProvider(): ImageGenerationProvider {
|
||||
return {
|
||||
id: "comfy",
|
||||
label: "ComfyUI",
|
||||
defaultModel: DEFAULT_COMFY_MODEL,
|
||||
models: [DEFAULT_COMFY_MODEL],
|
||||
isConfigured: ({ cfg, agentDir }) =>
|
||||
isComfyCapabilityConfigured({
|
||||
cfg,
|
||||
agentDir,
|
||||
capability: "image",
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxCount: 1,
|
||||
supportsSize: false,
|
||||
supportsAspectRatio: false,
|
||||
supportsResolution: false,
|
||||
},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxCount: 1,
|
||||
maxInputImages: 1,
|
||||
supportsSize: false,
|
||||
supportsAspectRatio: false,
|
||||
supportsResolution: false,
|
||||
},
|
||||
},
|
||||
async generateImage(req) {
|
||||
if ((req.inputImages?.length ?? 0) > 1) {
|
||||
throw new Error("Comfy image generation currently supports at most one reference image");
|
||||
}
|
||||
|
||||
const result = await runComfyWorkflow({
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
authStore: req.authStore,
|
||||
prompt: req.prompt,
|
||||
model: req.model,
|
||||
timeoutMs: req.timeoutMs,
|
||||
capability: "image",
|
||||
outputKinds: ["images"],
|
||||
inputImage: req.inputImages?.[0],
|
||||
});
|
||||
|
||||
const images: GeneratedImageAsset[] = result.assets.map((asset) => ({
|
||||
buffer: asset.buffer,
|
||||
mimeType: asset.mimeType,
|
||||
fileName: asset.fileName,
|
||||
metadata: {
|
||||
nodeId: asset.nodeId,
|
||||
promptId: result.promptId,
|
||||
},
|
||||
}));
|
||||
|
||||
return {
|
||||
images,
|
||||
model: result.model,
|
||||
metadata: {
|
||||
promptId: result.promptId,
|
||||
outputNodeIds: result.outputNodeIds,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
24
openclaw/extensions/comfy/index.ts
Normal file
24
openclaw/extensions/comfy/index.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
import { buildComfyImageGenerationProvider } from "./image-generation-provider.js";
|
||||
import { buildComfyMusicGenerationProvider } from "./music-generation-provider.js";
|
||||
import { buildComfyVideoGenerationProvider } from "./video-generation-provider.js";
|
||||
|
||||
const PROVIDER_ID = "comfy";
|
||||
|
||||
export default definePluginEntry({
|
||||
id: PROVIDER_ID,
|
||||
name: "ComfyUI Provider",
|
||||
description: "Bundled ComfyUI workflow media generation provider",
|
||||
register(api) {
|
||||
api.registerProvider({
|
||||
id: PROVIDER_ID,
|
||||
label: "ComfyUI",
|
||||
docsPath: "/providers/comfy",
|
||||
envVars: ["COMFY_API_KEY", "COMFY_CLOUD_API_KEY"],
|
||||
auth: [],
|
||||
});
|
||||
api.registerImageGenerationProvider(buildComfyImageGenerationProvider());
|
||||
api.registerMusicGenerationProvider(buildComfyMusicGenerationProvider());
|
||||
api.registerVideoGenerationProvider(buildComfyVideoGenerationProvider());
|
||||
},
|
||||
});
|
||||
94
openclaw/extensions/comfy/music-generation-provider.test.ts
Normal file
94
openclaw/extensions/comfy/music-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { expectExplicitMusicGenerationCapabilities } from "../../test/helpers/media-generation/provider-capability-assertions.js";
|
||||
import { buildComfyMusicGenerationProvider } from "./music-generation-provider.js";
|
||||
import { _setComfyFetchGuardForTesting } from "./workflow-runtime.js";
|
||||
|
||||
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuardMock: vi.fn(),
|
||||
}));
|
||||
|
||||
describe("comfy music-generation provider", () => {
|
||||
it("registers the workflow model", () => {
|
||||
const provider = buildComfyMusicGenerationProvider();
|
||||
|
||||
expect(provider.defaultModel).toBe("workflow");
|
||||
expect(provider.models).toEqual(["workflow"]);
|
||||
expectExplicitMusicGenerationCapabilities(provider);
|
||||
});
|
||||
|
||||
it("runs a music workflow and returns audio outputs", async () => {
|
||||
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ prompt_id: "music-job-1" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
"music-job-1": {
|
||||
outputs: {
|
||||
"9": {
|
||||
audio: [{ filename: "song.mp3", subfolder: "", type: "output" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("music-bytes"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "audio/mpeg" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildComfyMusicGenerationProvider();
|
||||
const result = await provider.generateMusic({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "gentle ambient synth loop",
|
||||
cfg: {
|
||||
models: {
|
||||
providers: {
|
||||
comfy: {
|
||||
music: {
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
outputNodeId: "9",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as never,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
model: "workflow",
|
||||
tracks: [
|
||||
{
|
||||
mimeType: "audio/mpeg",
|
||||
fileName: "song.mp3",
|
||||
},
|
||||
],
|
||||
metadata: {
|
||||
promptId: "music-job-1",
|
||||
outputNodeIds: ["9"],
|
||||
inputImageCount: 0,
|
||||
},
|
||||
});
|
||||
expect(result.tracks[0]?.buffer).toEqual(Buffer.from("music-bytes"));
|
||||
});
|
||||
});
|
||||
88
openclaw/extensions/comfy/music-generation-provider.ts
Normal file
88
openclaw/extensions/comfy/music-generation-provider.ts
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
import type {
|
||||
GeneratedMusicAsset,
|
||||
MusicGenerationProvider,
|
||||
MusicGenerationSourceImage,
|
||||
} from "openclaw/plugin-sdk/music-generation";
|
||||
import {
|
||||
DEFAULT_COMFY_MODEL,
|
||||
isComfyCapabilityConfigured,
|
||||
runComfyWorkflow,
|
||||
} from "./workflow-runtime.js";
|
||||
|
||||
const COMFY_MAX_INPUT_IMAGES = 1;
|
||||
|
||||
function toGeneratedTrack(asset: {
|
||||
buffer: Buffer;
|
||||
mimeType: string;
|
||||
fileName: string;
|
||||
}): GeneratedMusicAsset {
|
||||
return {
|
||||
buffer: asset.buffer,
|
||||
mimeType: asset.mimeType,
|
||||
fileName: asset.fileName,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveInputImage(inputImage: MusicGenerationSourceImage | undefined) {
|
||||
if (!inputImage) {
|
||||
return undefined;
|
||||
}
|
||||
if (!inputImage.buffer) {
|
||||
throw new Error("Comfy music generation requires loaded reference image bytes.");
|
||||
}
|
||||
return {
|
||||
buffer: inputImage.buffer,
|
||||
mimeType: inputImage.mimeType ?? "image/png",
|
||||
fileName: inputImage.fileName,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildComfyMusicGenerationProvider(): MusicGenerationProvider {
|
||||
return {
|
||||
id: "comfy",
|
||||
label: "ComfyUI",
|
||||
defaultModel: DEFAULT_COMFY_MODEL,
|
||||
models: [DEFAULT_COMFY_MODEL],
|
||||
isConfigured: ({ cfg, agentDir }) =>
|
||||
isComfyCapabilityConfigured({
|
||||
cfg,
|
||||
agentDir,
|
||||
capability: "music",
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {},
|
||||
edit: {
|
||||
enabled: true,
|
||||
maxInputImages: COMFY_MAX_INPUT_IMAGES,
|
||||
},
|
||||
},
|
||||
async generateMusic(req) {
|
||||
if ((req.inputImages?.length ?? 0) > COMFY_MAX_INPUT_IMAGES) {
|
||||
throw new Error(
|
||||
`Comfy music generation supports at most ${COMFY_MAX_INPUT_IMAGES} reference image.`,
|
||||
);
|
||||
}
|
||||
|
||||
const result = await runComfyWorkflow({
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
authStore: req.authStore,
|
||||
prompt: req.prompt,
|
||||
model: req.model,
|
||||
capability: "music",
|
||||
outputKinds: ["audio"],
|
||||
inputImage: resolveInputImage(req.inputImages?.[0]),
|
||||
});
|
||||
|
||||
return {
|
||||
tracks: result.assets.map(toGeneratedTrack),
|
||||
model: result.model,
|
||||
metadata: {
|
||||
promptId: result.promptId,
|
||||
outputNodeIds: result.outputNodeIds,
|
||||
inputImageCount: req.inputImages?.length ?? 0,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
104
openclaw/extensions/comfy/openclaw.plugin.json
Normal file
104
openclaw/extensions/comfy/openclaw.plugin.json
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
{
|
||||
"id": "comfy",
|
||||
"enabledByDefault": true,
|
||||
"providers": ["comfy"],
|
||||
"providerAuthEnvVars": {
|
||||
"comfy": ["COMFY_API_KEY", "COMFY_CLOUD_API_KEY"]
|
||||
},
|
||||
"contracts": {
|
||||
"imageGenerationProviders": ["comfy"],
|
||||
"musicGenerationProviders": ["comfy"],
|
||||
"videoGenerationProviders": ["comfy"]
|
||||
},
|
||||
"configSchema": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"enum": ["local", "cloud"]
|
||||
},
|
||||
"baseUrl": {
|
||||
"type": "string"
|
||||
},
|
||||
"apiKey": {
|
||||
"type": ["string", "object"]
|
||||
},
|
||||
"allowPrivateNetwork": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"workflowPath": {
|
||||
"type": "string"
|
||||
},
|
||||
"workflow": {
|
||||
"type": "object"
|
||||
},
|
||||
"promptNodeId": {
|
||||
"type": "string"
|
||||
},
|
||||
"promptInputName": {
|
||||
"type": "string"
|
||||
},
|
||||
"inputImageNodeId": {
|
||||
"type": "string"
|
||||
},
|
||||
"inputImageInputName": {
|
||||
"type": "string"
|
||||
},
|
||||
"outputNodeId": {
|
||||
"type": "string"
|
||||
},
|
||||
"pollIntervalMs": {
|
||||
"type": "integer",
|
||||
"minimum": 100
|
||||
},
|
||||
"timeoutMs": {
|
||||
"type": "integer",
|
||||
"minimum": 1000
|
||||
},
|
||||
"image": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"workflowPath": { "type": "string" },
|
||||
"workflow": { "type": "object" },
|
||||
"promptNodeId": { "type": "string" },
|
||||
"promptInputName": { "type": "string" },
|
||||
"inputImageNodeId": { "type": "string" },
|
||||
"inputImageInputName": { "type": "string" },
|
||||
"outputNodeId": { "type": "string" },
|
||||
"pollIntervalMs": { "type": "integer", "minimum": 100 },
|
||||
"timeoutMs": { "type": "integer", "minimum": 1000 }
|
||||
}
|
||||
},
|
||||
"video": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"workflowPath": { "type": "string" },
|
||||
"workflow": { "type": "object" },
|
||||
"promptNodeId": { "type": "string" },
|
||||
"promptInputName": { "type": "string" },
|
||||
"inputImageNodeId": { "type": "string" },
|
||||
"inputImageInputName": { "type": "string" },
|
||||
"outputNodeId": { "type": "string" },
|
||||
"pollIntervalMs": { "type": "integer", "minimum": 100 },
|
||||
"timeoutMs": { "type": "integer", "minimum": 1000 }
|
||||
}
|
||||
},
|
||||
"music": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"workflowPath": { "type": "string" },
|
||||
"workflow": { "type": "object" },
|
||||
"promptNodeId": { "type": "string" },
|
||||
"promptInputName": { "type": "string" },
|
||||
"outputNodeId": { "type": "string" },
|
||||
"pollIntervalMs": { "type": "integer", "minimum": 100 },
|
||||
"timeoutMs": { "type": "integer", "minimum": 1000 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
15
openclaw/extensions/comfy/package.json
Normal file
15
openclaw/extensions/comfy/package.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"name": "@openclaw/comfy-provider",
|
||||
"version": "2026.4.20",
|
||||
"private": true,
|
||||
"description": "OpenClaw ComfyUI provider plugin",
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@openclaw/plugin-sdk": "workspace:*"
|
||||
},
|
||||
"openclaw": {
|
||||
"extensions": [
|
||||
"./index.ts"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js";
|
||||
|
||||
describePluginRegistrationContract({
|
||||
pluginId: "comfy",
|
||||
providerIds: ["comfy"],
|
||||
imageGenerationProviderIds: ["comfy"],
|
||||
musicGenerationProviderIds: ["comfy"],
|
||||
videoGenerationProviderIds: ["comfy"],
|
||||
requireGenerateImage: true,
|
||||
requireGenerateVideo: true,
|
||||
});
|
||||
16
openclaw/extensions/comfy/tsconfig.json
Normal file
16
openclaw/extensions/comfy/tsconfig.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"extends": "../tsconfig.package-boundary.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "."
|
||||
},
|
||||
"include": ["./*.ts", "./src/**/*.ts"],
|
||||
"exclude": [
|
||||
"./**/*.test.ts",
|
||||
"./dist/**",
|
||||
"./node_modules/**",
|
||||
"./src/test-support/**",
|
||||
"./src/**/*test-helpers.ts",
|
||||
"./src/**/*test-harness.ts",
|
||||
"./src/**/*test-support.ts"
|
||||
]
|
||||
}
|
||||
245
openclaw/extensions/comfy/video-generation-provider.test.ts
Normal file
245
openclaw/extensions/comfy/video-generation-provider.test.ts
Normal file
|
|
@ -0,0 +1,245 @@
|
|||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { expectExplicitVideoGenerationCapabilities } from "../../test/helpers/media-generation/provider-capability-assertions.js";
|
||||
import {
|
||||
_setComfyFetchGuardForTesting,
|
||||
buildComfyVideoGenerationProvider,
|
||||
} from "./video-generation-provider.js";
|
||||
|
||||
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuardMock: vi.fn(),
|
||||
}));
|
||||
|
||||
function parseJsonBody(call: number): Record<string, unknown> {
|
||||
const request = fetchWithSsrFGuardMock.mock.calls[call - 1]?.[0];
|
||||
expect(request?.init?.body).toBeTruthy();
|
||||
return JSON.parse(String(request.init.body)) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function buildComfyConfig(config: Record<string, unknown>): OpenClawConfig {
|
||||
return {
|
||||
models: {
|
||||
providers: {
|
||||
comfy: config,
|
||||
},
|
||||
},
|
||||
} as unknown as OpenClawConfig;
|
||||
}
|
||||
|
||||
describe("comfy video-generation provider", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
_setComfyFetchGuardForTesting(null);
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("declares explicit mode capabilities", () => {
|
||||
expectExplicitVideoGenerationCapabilities(buildComfyVideoGenerationProvider());
|
||||
});
|
||||
|
||||
it("treats local comfy video workflows as configured without an API key", () => {
|
||||
const provider = buildComfyVideoGenerationProvider();
|
||||
expect(
|
||||
provider.isConfigured?.({
|
||||
cfg: buildComfyConfig({
|
||||
video: {
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("submits a local workflow, waits for history, and downloads videos", async () => {
|
||||
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ prompt_id: "local-video-1" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
"local-video-1": {
|
||||
outputs: {
|
||||
"9": {
|
||||
gifs: [{ filename: "generated.mp4", subfolder: "", type: "output" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("mp4-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "video/mp4" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildComfyVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "animate a lobster",
|
||||
cfg: buildComfyConfig({
|
||||
video: {
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
outputNodeId: "9",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8188/prompt",
|
||||
auditContext: "comfy-video-generate",
|
||||
}),
|
||||
);
|
||||
expect(parseJsonBody(1)).toEqual({
|
||||
prompt: {
|
||||
"6": { inputs: { text: "animate a lobster" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
});
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8188/history/local-video-1",
|
||||
auditContext: "comfy-history",
|
||||
}),
|
||||
);
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
expect.objectContaining({
|
||||
url: "http://127.0.0.1:8188/view?filename=generated.mp4&subfolder=&type=output",
|
||||
auditContext: "comfy-video-download",
|
||||
}),
|
||||
);
|
||||
expect(result).toEqual({
|
||||
videos: [
|
||||
{
|
||||
buffer: Buffer.from("mp4-data"),
|
||||
mimeType: "video/mp4",
|
||||
fileName: "generated.mp4",
|
||||
metadata: {
|
||||
nodeId: "9",
|
||||
promptId: "local-video-1",
|
||||
},
|
||||
},
|
||||
],
|
||||
model: "workflow",
|
||||
metadata: {
|
||||
promptId: "local-video-1",
|
||||
outputNodeIds: ["9"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("uses cloud endpoints for video workflows", async () => {
|
||||
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
|
||||
apiKey: "comfy-test-key",
|
||||
source: "env",
|
||||
mode: "api-key",
|
||||
});
|
||||
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
|
||||
fetchWithSsrFGuardMock
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ prompt_id: "cloud-video-1" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(JSON.stringify({ status: "completed" }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(
|
||||
JSON.stringify({
|
||||
"cloud-video-1": {
|
||||
outputs: {
|
||||
"9": {
|
||||
gifs: [{ filename: "cloud.mp4", subfolder: "", type: "output" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
},
|
||||
),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "https://cdn.example.com/cloud.mp4" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
response: new Response(Buffer.from("cloud-video-data"), {
|
||||
status: 200,
|
||||
headers: { "content-type": "video/mp4" },
|
||||
}),
|
||||
release: vi.fn(async () => {}),
|
||||
});
|
||||
|
||||
const provider = buildComfyVideoGenerationProvider();
|
||||
const result = await provider.generateVideo({
|
||||
provider: "comfy",
|
||||
model: "workflow",
|
||||
prompt: "cloud video workflow",
|
||||
cfg: buildComfyConfig({
|
||||
mode: "cloud",
|
||||
video: {
|
||||
workflow: {
|
||||
"6": { inputs: { text: "" } },
|
||||
"9": { inputs: {} },
|
||||
},
|
||||
promptNodeId: "6",
|
||||
outputNodeId: "9",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
url: "https://cloud.comfy.org/api/prompt",
|
||||
auditContext: "comfy-video-generate",
|
||||
}),
|
||||
);
|
||||
expect(result.metadata).toEqual({
|
||||
promptId: "cloud-video-1",
|
||||
outputNodeIds: ["9"],
|
||||
});
|
||||
});
|
||||
});
|
||||
104
openclaw/extensions/comfy/video-generation-provider.ts
Normal file
104
openclaw/extensions/comfy/video-generation-provider.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import type {
|
||||
GeneratedVideoAsset,
|
||||
VideoGenerationProvider,
|
||||
VideoGenerationSourceAsset,
|
||||
} from "openclaw/plugin-sdk/video-generation";
|
||||
import {
|
||||
DEFAULT_COMFY_MODEL,
|
||||
_setComfyFetchGuardForTesting,
|
||||
isComfyCapabilityConfigured,
|
||||
runComfyWorkflow,
|
||||
} from "./workflow-runtime.js";
|
||||
|
||||
export { _setComfyFetchGuardForTesting };
|
||||
|
||||
function toComfyInputImage(inputImage?: VideoGenerationSourceAsset) {
|
||||
if (!inputImage) {
|
||||
return undefined;
|
||||
}
|
||||
if (!inputImage.buffer || !inputImage.mimeType) {
|
||||
throw new Error("Comfy video generation requires a local reference image file");
|
||||
}
|
||||
return {
|
||||
buffer: inputImage.buffer,
|
||||
mimeType: inputImage.mimeType,
|
||||
fileName: inputImage.fileName,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildComfyVideoGenerationProvider(): VideoGenerationProvider {
|
||||
return {
|
||||
id: "comfy",
|
||||
label: "ComfyUI",
|
||||
defaultModel: DEFAULT_COMFY_MODEL,
|
||||
models: [DEFAULT_COMFY_MODEL],
|
||||
isConfigured: ({ cfg, agentDir }) =>
|
||||
isComfyCapabilityConfigured({
|
||||
cfg,
|
||||
agentDir,
|
||||
capability: "video",
|
||||
}),
|
||||
capabilities: {
|
||||
generate: {
|
||||
maxVideos: 1,
|
||||
supportsSize: false,
|
||||
supportsAspectRatio: false,
|
||||
supportsResolution: false,
|
||||
supportsAudio: false,
|
||||
supportsWatermark: false,
|
||||
},
|
||||
imageToVideo: {
|
||||
enabled: true,
|
||||
maxVideos: 1,
|
||||
maxInputImages: 1,
|
||||
supportsSize: false,
|
||||
supportsAspectRatio: false,
|
||||
supportsResolution: false,
|
||||
supportsAudio: false,
|
||||
supportsWatermark: false,
|
||||
},
|
||||
videoToVideo: {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
async generateVideo(req) {
|
||||
if ((req.inputImages?.length ?? 0) > 1) {
|
||||
throw new Error("Comfy video generation currently supports at most one reference image");
|
||||
}
|
||||
if ((req.inputVideos?.length ?? 0) > 0) {
|
||||
throw new Error("Comfy video generation does not support input videos");
|
||||
}
|
||||
|
||||
const result = await runComfyWorkflow({
|
||||
cfg: req.cfg,
|
||||
agentDir: req.agentDir,
|
||||
authStore: req.authStore,
|
||||
prompt: req.prompt,
|
||||
model: req.model,
|
||||
timeoutMs: req.timeoutMs,
|
||||
capability: "video",
|
||||
outputKinds: ["gifs", "videos"],
|
||||
inputImage: toComfyInputImage(req.inputImages?.[0]),
|
||||
});
|
||||
|
||||
const videos: GeneratedVideoAsset[] = result.assets.map((asset) => ({
|
||||
buffer: asset.buffer,
|
||||
mimeType: asset.mimeType,
|
||||
fileName: asset.fileName,
|
||||
metadata: {
|
||||
nodeId: asset.nodeId,
|
||||
promptId: result.promptId,
|
||||
},
|
||||
}));
|
||||
|
||||
return {
|
||||
videos,
|
||||
model: result.model,
|
||||
metadata: {
|
||||
promptId: result.promptId,
|
||||
outputNodeIds: result.outputNodeIds,
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
780
openclaw/extensions/comfy/workflow-runtime.ts
Normal file
780
openclaw/extensions/comfy/workflow-runtime.ts
Normal file
|
|
@ -0,0 +1,780 @@
|
|||
import fs from "node:fs/promises";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import {
|
||||
isProviderApiKeyConfigured,
|
||||
type AuthProfileStore,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
|
||||
import {
|
||||
assertOkOrThrowHttpError,
|
||||
normalizeBaseUrl,
|
||||
resolveProviderHttpRequestConfig,
|
||||
} from "openclaw/plugin-sdk/provider-http";
|
||||
import {
|
||||
buildHostnameAllowlistPolicyFromSuffixAllowlist,
|
||||
fetchWithSsrFGuard,
|
||||
isPrivateOrLoopbackHost,
|
||||
ssrfPolicyFromDangerouslyAllowPrivateNetwork,
|
||||
type SsrFPolicy,
|
||||
} from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
isRecord,
|
||||
normalizeOptionalLowercaseString,
|
||||
normalizeOptionalString,
|
||||
resolveUserPath,
|
||||
} from "openclaw/plugin-sdk/text-runtime";
|
||||
|
||||
const DEFAULT_COMFY_LOCAL_BASE_URL = "http://127.0.0.1:8188";
|
||||
const DEFAULT_COMFY_CLOUD_BASE_URL = "https://cloud.comfy.org";
|
||||
const DEFAULT_PROMPT_INPUT_NAME = "text";
|
||||
const DEFAULT_INPUT_IMAGE_INPUT_NAME = "image";
|
||||
const DEFAULT_POLL_INTERVAL_MS = 1_500;
|
||||
const DEFAULT_TIMEOUT_MS = 5 * 60_000;
|
||||
|
||||
export const DEFAULT_COMFY_MODEL = "workflow";
|
||||
|
||||
export type ComfyMode = "local" | "cloud";
|
||||
export type ComfyCapability = "image" | "music" | "video";
|
||||
export type ComfyOutputKind = "audio" | "gifs" | "images" | "videos";
|
||||
export type ComfyWorkflow = Record<string, unknown>;
|
||||
export type ComfyProviderConfig = Record<string, unknown>;
|
||||
type ComfyFetchGuardParams = Parameters<typeof fetchWithSsrFGuard>[0];
|
||||
type ComfyDispatcherPolicy = ComfyFetchGuardParams["dispatcherPolicy"];
|
||||
type ComfyPromptResponse = {
|
||||
prompt_id?: string;
|
||||
};
|
||||
type ComfyOutputFile = {
|
||||
filename?: string;
|
||||
name?: string;
|
||||
subfolder?: string;
|
||||
type?: string;
|
||||
};
|
||||
type ComfyHistoryOutputEntry = Partial<Record<ComfyOutputKind, ComfyOutputFile[]>>;
|
||||
type ComfyHistoryEntry = {
|
||||
outputs?: Record<string, ComfyHistoryOutputEntry>;
|
||||
};
|
||||
type ComfyUploadResponse = {
|
||||
name?: string;
|
||||
filename?: string;
|
||||
};
|
||||
type ComfyStatusResponse = {
|
||||
status?: string;
|
||||
message?: string;
|
||||
error?: string;
|
||||
};
|
||||
type ComfyNetworkPolicy = {
|
||||
apiPolicy?: SsrFPolicy;
|
||||
};
|
||||
|
||||
export type ComfySourceImage = {
|
||||
buffer: Buffer;
|
||||
mimeType: string;
|
||||
fileName?: string;
|
||||
};
|
||||
|
||||
export type ComfyGeneratedAsset = {
|
||||
buffer: Buffer;
|
||||
mimeType: string;
|
||||
fileName: string;
|
||||
nodeId: string;
|
||||
};
|
||||
|
||||
export type ComfyWorkflowResult = {
|
||||
assets: ComfyGeneratedAsset[];
|
||||
model: string;
|
||||
promptId: string;
|
||||
outputNodeIds: string[];
|
||||
};
|
||||
|
||||
let comfyFetchGuard = fetchWithSsrFGuard;
|
||||
|
||||
export function _setComfyFetchGuardForTesting(impl: typeof fetchWithSsrFGuard | null): void {
|
||||
comfyFetchGuard = impl ?? fetchWithSsrFGuard;
|
||||
}
|
||||
|
||||
function readConfigBoolean(config: ComfyProviderConfig, key: string): boolean | undefined {
|
||||
const value = config[key];
|
||||
return typeof value === "boolean" ? value : undefined;
|
||||
}
|
||||
|
||||
function readConfigInteger(config: ComfyProviderConfig, key: string): number | undefined {
|
||||
const value = config[key];
|
||||
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
function mergeSsrFPolicies(...policies: Array<SsrFPolicy | undefined>): SsrFPolicy | undefined {
|
||||
const merged: SsrFPolicy = {};
|
||||
for (const policy of policies) {
|
||||
if (!policy) {
|
||||
continue;
|
||||
}
|
||||
if (policy.allowPrivateNetwork) {
|
||||
merged.allowPrivateNetwork = true;
|
||||
}
|
||||
if (policy.dangerouslyAllowPrivateNetwork) {
|
||||
merged.dangerouslyAllowPrivateNetwork = true;
|
||||
}
|
||||
if (policy.allowRfc2544BenchmarkRange) {
|
||||
merged.allowRfc2544BenchmarkRange = true;
|
||||
}
|
||||
if (policy.allowedHostnames?.length) {
|
||||
merged.allowedHostnames = Array.from(
|
||||
new Set([...(merged.allowedHostnames ?? []), ...policy.allowedHostnames]),
|
||||
);
|
||||
}
|
||||
if (policy.hostnameAllowlist?.length) {
|
||||
merged.hostnameAllowlist = Array.from(
|
||||
new Set([...(merged.hostnameAllowlist ?? []), ...policy.hostnameAllowlist]),
|
||||
);
|
||||
}
|
||||
}
|
||||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||||
}
|
||||
|
||||
export function getComfyConfig(cfg?: OpenClawConfig): ComfyProviderConfig {
|
||||
const raw = cfg?.models?.providers?.comfy;
|
||||
return isRecord(raw) ? raw : {};
|
||||
}
|
||||
|
||||
function stripNestedCapabilityConfig(config: ComfyProviderConfig): ComfyProviderConfig {
|
||||
const next = { ...config };
|
||||
delete next.image;
|
||||
delete next.video;
|
||||
delete next.music;
|
||||
return next;
|
||||
}
|
||||
|
||||
export function getComfyCapabilityConfig(
|
||||
config: ComfyProviderConfig,
|
||||
capability: ComfyCapability,
|
||||
): ComfyProviderConfig {
|
||||
const shared = stripNestedCapabilityConfig(config);
|
||||
const nested = config[capability];
|
||||
if (!isRecord(nested)) {
|
||||
return shared;
|
||||
}
|
||||
return { ...shared, ...nested };
|
||||
}
|
||||
|
||||
export function resolveComfyMode(config: ComfyProviderConfig): ComfyMode {
|
||||
return normalizeOptionalString(config.mode) === "cloud" ? "cloud" : "local";
|
||||
}
|
||||
|
||||
function getRequiredConfigString(config: ComfyProviderConfig, key: string): string {
|
||||
const value = normalizeOptionalString(config[key]);
|
||||
if (!value) {
|
||||
throw new Error(`models.providers.comfy.${key} is required`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function resolveComfyWorkflowSource(config: ComfyProviderConfig): {
|
||||
workflow?: ComfyWorkflow;
|
||||
workflowPath?: string;
|
||||
} {
|
||||
const workflow = config.workflow;
|
||||
if (isRecord(workflow)) {
|
||||
return { workflow: structuredClone(workflow) };
|
||||
}
|
||||
const workflowPath = normalizeOptionalString(config.workflowPath);
|
||||
return { workflowPath };
|
||||
}
|
||||
|
||||
async function loadComfyWorkflow(config: ComfyProviderConfig): Promise<ComfyWorkflow> {
|
||||
const source = resolveComfyWorkflowSource(config);
|
||||
if (source.workflow) {
|
||||
return source.workflow;
|
||||
}
|
||||
if (!source.workflowPath) {
|
||||
throw new Error("models.providers.comfy.<capability>.workflow or workflowPath is required");
|
||||
}
|
||||
|
||||
const resolvedPath = resolveUserPath(source.workflowPath);
|
||||
const raw = await fs.readFile(resolvedPath, "utf8");
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!isRecord(parsed)) {
|
||||
throw new Error(`Comfy workflow at ${resolvedPath} must be a JSON object`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function setWorkflowInput(params: {
|
||||
workflow: ComfyWorkflow;
|
||||
nodeId: string;
|
||||
inputName: string;
|
||||
value: unknown;
|
||||
}): void {
|
||||
const node = params.workflow[params.nodeId];
|
||||
if (!isRecord(node)) {
|
||||
throw new Error(`Comfy workflow missing node "${params.nodeId}"`);
|
||||
}
|
||||
const inputs = node.inputs;
|
||||
if (!isRecord(inputs)) {
|
||||
throw new Error(`Comfy workflow node "${params.nodeId}" is missing an inputs object`);
|
||||
}
|
||||
inputs[params.inputName] = params.value;
|
||||
}
|
||||
|
||||
function resolveComfyNetworkPolicy(params: {
|
||||
baseUrl: string;
|
||||
allowPrivateNetwork: boolean;
|
||||
}): ComfyNetworkPolicy {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(params.baseUrl);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
|
||||
const hostname = normalizeOptionalLowercaseString(parsed.hostname) ?? "";
|
||||
if (!hostname || !params.allowPrivateNetwork || !isPrivateOrLoopbackHost(hostname)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const hostnamePolicy = buildHostnameAllowlistPolicyFromSuffixAllowlist([hostname]);
|
||||
const privateNetworkPolicy = ssrfPolicyFromDangerouslyAllowPrivateNetwork(true);
|
||||
return {
|
||||
apiPolicy: mergeSsrFPolicies(hostnamePolicy, privateNetworkPolicy),
|
||||
};
|
||||
}
|
||||
|
||||
async function readJsonResponse<T>(params: {
|
||||
url: string;
|
||||
init?: RequestInit;
|
||||
timeoutMs?: number;
|
||||
policy?: SsrFPolicy;
|
||||
dispatcherPolicy?: ComfyDispatcherPolicy;
|
||||
auditContext: string;
|
||||
errorPrefix: string;
|
||||
}): Promise<T> {
|
||||
const { response, release } = await comfyFetchGuard({
|
||||
url: params.url,
|
||||
init: params.init,
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: params.auditContext,
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(response, params.errorPrefix);
|
||||
return (await response.json()) as T;
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
function inferFileExtension(params: { fileName?: string; mimeType?: string }): string {
|
||||
const normalizedMime = normalizeOptionalLowercaseString(params.mimeType);
|
||||
if (normalizedMime?.includes("jpeg")) {
|
||||
return "jpg";
|
||||
}
|
||||
if (normalizedMime?.includes("png")) {
|
||||
return "png";
|
||||
}
|
||||
if (normalizedMime?.includes("webm")) {
|
||||
return "webm";
|
||||
}
|
||||
if (normalizedMime?.includes("mp4")) {
|
||||
return "mp4";
|
||||
}
|
||||
if (normalizedMime?.includes("mpeg")) {
|
||||
return "mp3";
|
||||
}
|
||||
if (normalizedMime?.includes("wav")) {
|
||||
return "wav";
|
||||
}
|
||||
const fileName = params.fileName?.trim();
|
||||
if (!fileName) {
|
||||
return "bin";
|
||||
}
|
||||
const dotIndex = fileName.lastIndexOf(".");
|
||||
if (dotIndex < 0 || dotIndex === fileName.length - 1) {
|
||||
return "bin";
|
||||
}
|
||||
return fileName.slice(dotIndex + 1);
|
||||
}
|
||||
|
||||
function toBlobBytes(buffer: Buffer): ArrayBuffer {
|
||||
const arrayBuffer = new ArrayBuffer(buffer.byteLength);
|
||||
new Uint8Array(arrayBuffer).set(buffer);
|
||||
return arrayBuffer;
|
||||
}
|
||||
|
||||
async function uploadInputImage(params: {
|
||||
baseUrl: string;
|
||||
headers: Headers;
|
||||
timeoutMs: number;
|
||||
policy?: SsrFPolicy;
|
||||
dispatcherPolicy?: ComfyDispatcherPolicy;
|
||||
image: ComfySourceImage;
|
||||
mode: ComfyMode;
|
||||
capability: ComfyCapability;
|
||||
}): Promise<string> {
|
||||
const form = new FormData();
|
||||
form.set(
|
||||
"image",
|
||||
new Blob([toBlobBytes(params.image.buffer)], { type: params.image.mimeType }),
|
||||
normalizeOptionalString(params.image.fileName) ||
|
||||
`input.${inferFileExtension({ mimeType: params.image.mimeType })}`,
|
||||
);
|
||||
form.set("type", "input");
|
||||
form.set("overwrite", "true");
|
||||
|
||||
const headers = new Headers(params.headers);
|
||||
headers.delete("Content-Type");
|
||||
|
||||
const payload = await readJsonResponse<ComfyUploadResponse>({
|
||||
url: `${params.baseUrl}${params.mode === "cloud" ? "/api/upload/image" : "/upload/image"}`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: form,
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: `comfy-${params.capability}-upload`,
|
||||
errorPrefix: "Comfy image upload failed",
|
||||
});
|
||||
|
||||
const uploadedName =
|
||||
normalizeOptionalString(payload.filename) || normalizeOptionalString(payload.name);
|
||||
if (!uploadedName) {
|
||||
throw new Error("Comfy image upload response missing filename");
|
||||
}
|
||||
return uploadedName;
|
||||
}
|
||||
|
||||
function extractHistoryEntry(history: unknown, promptId: string): ComfyHistoryEntry | null {
|
||||
if (!isRecord(history)) {
|
||||
return null;
|
||||
}
|
||||
const directOutputs = history.outputs;
|
||||
if (isRecord(directOutputs)) {
|
||||
return history as ComfyHistoryEntry;
|
||||
}
|
||||
const nested = history[promptId];
|
||||
if (isRecord(nested)) {
|
||||
return nested as ComfyHistoryEntry;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function waitForLocalHistory(params: {
|
||||
baseUrl: string;
|
||||
promptId: string;
|
||||
headers: Headers;
|
||||
timeoutMs: number;
|
||||
pollIntervalMs: number;
|
||||
policy?: SsrFPolicy;
|
||||
dispatcherPolicy?: ComfyDispatcherPolicy;
|
||||
}): Promise<ComfyHistoryEntry> {
|
||||
const deadline = Date.now() + params.timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
const history = await readJsonResponse<unknown>({
|
||||
url: `${params.baseUrl}/history/${params.promptId}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: "comfy-history",
|
||||
errorPrefix: "Comfy history lookup failed",
|
||||
});
|
||||
|
||||
const entry = extractHistoryEntry(history, params.promptId);
|
||||
if (entry?.outputs && Object.keys(entry.outputs).length > 0) {
|
||||
return entry;
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
||||
}
|
||||
|
||||
throw new Error(`Comfy workflow did not finish within ${Math.ceil(params.timeoutMs / 1000)}s`);
|
||||
}
|
||||
|
||||
async function waitForCloudCompletion(params: {
|
||||
baseUrl: string;
|
||||
promptId: string;
|
||||
headers: Headers;
|
||||
timeoutMs: number;
|
||||
pollIntervalMs: number;
|
||||
policy?: SsrFPolicy;
|
||||
dispatcherPolicy?: ComfyDispatcherPolicy;
|
||||
}): Promise<void> {
|
||||
const deadline = Date.now() + params.timeoutMs;
|
||||
while (Date.now() <= deadline) {
|
||||
const status = await readJsonResponse<ComfyStatusResponse>({
|
||||
url: `${params.baseUrl}/api/job/${params.promptId}/status`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext: "comfy-status",
|
||||
errorPrefix: "Comfy status lookup failed",
|
||||
});
|
||||
|
||||
if (status.status === "completed") {
|
||||
return;
|
||||
}
|
||||
if (status.status === "failed" || status.status === "cancelled") {
|
||||
throw new Error(
|
||||
`Comfy workflow ${status.status}: ${status.error ?? status.message ?? params.promptId}`,
|
||||
);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
|
||||
}
|
||||
|
||||
throw new Error(`Comfy workflow did not finish within ${Math.ceil(params.timeoutMs / 1000)}s`);
|
||||
}
|
||||
|
||||
function collectOutputFiles(params: {
|
||||
history: ComfyHistoryEntry;
|
||||
outputNodeId?: string;
|
||||
outputKinds: readonly ComfyOutputKind[];
|
||||
}): Array<{ nodeId: string; file: ComfyOutputFile }> {
|
||||
const outputs = params.history.outputs;
|
||||
if (!outputs) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const nodeIds = params.outputNodeId ? [params.outputNodeId] : Object.keys(outputs);
|
||||
const files: Array<{ nodeId: string; file: ComfyOutputFile }> = [];
|
||||
for (const nodeId of nodeIds) {
|
||||
const entry = outputs[nodeId];
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
for (const kind of params.outputKinds) {
|
||||
const bucket = entry[kind];
|
||||
if (!Array.isArray(bucket)) {
|
||||
continue;
|
||||
}
|
||||
for (const file of bucket) {
|
||||
files.push({ nodeId, file });
|
||||
}
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function downloadOutputFile(params: {
|
||||
baseUrl: string;
|
||||
headers: Headers;
|
||||
timeoutMs: number;
|
||||
policy?: SsrFPolicy;
|
||||
dispatcherPolicy?: ComfyDispatcherPolicy;
|
||||
file: ComfyOutputFile;
|
||||
mode: ComfyMode;
|
||||
capability: ComfyCapability;
|
||||
}): Promise<{ buffer: Buffer; mimeType: string }> {
|
||||
const fileName =
|
||||
normalizeOptionalString(params.file.filename) || normalizeOptionalString(params.file.name);
|
||||
if (!fileName) {
|
||||
throw new Error("Comfy output entry missing filename");
|
||||
}
|
||||
|
||||
const query = new URLSearchParams({
|
||||
filename: fileName,
|
||||
subfolder: normalizeOptionalString(params.file.subfolder) ?? "",
|
||||
type: normalizeOptionalString(params.file.type) ?? "output",
|
||||
});
|
||||
const viewPath = params.mode === "cloud" ? "/api/view" : "/view";
|
||||
const auditContext = `comfy-${params.capability}-download`;
|
||||
|
||||
const firstResponse = await comfyFetchGuard({
|
||||
url: `${params.baseUrl}${viewPath}?${query.toString()}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: params.headers,
|
||||
...(params.mode === "cloud" ? { redirect: "manual" } : {}),
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
policy: params.policy,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext,
|
||||
});
|
||||
|
||||
try {
|
||||
if (
|
||||
params.mode === "cloud" &&
|
||||
[301, 302, 303, 307, 308].includes(firstResponse.response.status)
|
||||
) {
|
||||
const redirectUrl = normalizeOptionalString(firstResponse.response.headers.get("location"));
|
||||
if (!redirectUrl) {
|
||||
throw new Error("Comfy cloud output redirect missing location header");
|
||||
}
|
||||
const redirected = await comfyFetchGuard({
|
||||
url: redirectUrl,
|
||||
init: {
|
||||
method: "GET",
|
||||
},
|
||||
timeoutMs: params.timeoutMs,
|
||||
dispatcherPolicy: params.dispatcherPolicy,
|
||||
auditContext,
|
||||
});
|
||||
try {
|
||||
await assertOkOrThrowHttpError(redirected.response, "Comfy output download failed");
|
||||
const mimeType =
|
||||
normalizeOptionalString(redirected.response.headers.get("content-type")) ||
|
||||
"application/octet-stream";
|
||||
return {
|
||||
buffer: Buffer.from(await redirected.response.arrayBuffer()),
|
||||
mimeType,
|
||||
};
|
||||
} finally {
|
||||
await redirected.release();
|
||||
}
|
||||
}
|
||||
|
||||
await assertOkOrThrowHttpError(firstResponse.response, "Comfy output download failed");
|
||||
const mimeType =
|
||||
normalizeOptionalString(firstResponse.response.headers.get("content-type")) ||
|
||||
"application/octet-stream";
|
||||
return {
|
||||
buffer: Buffer.from(await firstResponse.response.arrayBuffer()),
|
||||
mimeType,
|
||||
};
|
||||
} finally {
|
||||
await firstResponse.release();
|
||||
}
|
||||
}
|
||||
|
||||
export function isComfyCapabilityConfigured(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
capability: ComfyCapability;
|
||||
}): boolean {
|
||||
const config = getComfyConfig(params.cfg);
|
||||
const capabilityConfig = getComfyCapabilityConfig(config, params.capability);
|
||||
const hasWorkflow = Boolean(
|
||||
resolveComfyWorkflowSource(capabilityConfig).workflow ||
|
||||
normalizeOptionalString(capabilityConfig.workflowPath),
|
||||
);
|
||||
const hasPromptNode = Boolean(normalizeOptionalString(capabilityConfig.promptNodeId));
|
||||
if (!hasWorkflow || !hasPromptNode) {
|
||||
return false;
|
||||
}
|
||||
if (resolveComfyMode(capabilityConfig) === "local") {
|
||||
return true;
|
||||
}
|
||||
return isProviderApiKeyConfigured({
|
||||
provider: "comfy",
|
||||
agentDir: params.agentDir,
|
||||
});
|
||||
}
|
||||
|
||||
export async function runComfyWorkflow(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
authStore?: AuthProfileStore;
|
||||
prompt: string;
|
||||
model?: string;
|
||||
timeoutMs?: number;
|
||||
capability: ComfyCapability;
|
||||
outputKinds: readonly ComfyOutputKind[];
|
||||
inputImage?: ComfySourceImage;
|
||||
}): Promise<ComfyWorkflowResult> {
|
||||
const config = getComfyConfig(params.cfg);
|
||||
const capabilityConfig = getComfyCapabilityConfig(config, params.capability);
|
||||
const mode = resolveComfyMode(capabilityConfig);
|
||||
const workflow = await loadComfyWorkflow(capabilityConfig);
|
||||
const promptNodeId = getRequiredConfigString(capabilityConfig, "promptNodeId");
|
||||
const promptInputName =
|
||||
normalizeOptionalString(capabilityConfig.promptInputName) ?? DEFAULT_PROMPT_INPUT_NAME;
|
||||
const inputImageNodeId = normalizeOptionalString(capabilityConfig.inputImageNodeId);
|
||||
const inputImageInputName =
|
||||
normalizeOptionalString(capabilityConfig.inputImageInputName) ?? DEFAULT_INPUT_IMAGE_INPUT_NAME;
|
||||
const outputNodeId = normalizeOptionalString(capabilityConfig.outputNodeId);
|
||||
const pollIntervalMs =
|
||||
readConfigInteger(capabilityConfig, "pollIntervalMs") ?? DEFAULT_POLL_INTERVAL_MS;
|
||||
const timeoutMs =
|
||||
readConfigInteger(capabilityConfig, "timeoutMs") ?? params.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const providerModel = normalizeOptionalString(params.model) || DEFAULT_COMFY_MODEL;
|
||||
|
||||
setWorkflowInput({
|
||||
workflow,
|
||||
nodeId: promptNodeId,
|
||||
inputName: promptInputName,
|
||||
value: params.prompt,
|
||||
});
|
||||
|
||||
const resolvedAuth =
|
||||
mode === "cloud"
|
||||
? await resolveApiKeyForProvider({
|
||||
provider: "comfy",
|
||||
cfg: params.cfg,
|
||||
agentDir: params.agentDir,
|
||||
store: params.authStore,
|
||||
})
|
||||
: null;
|
||||
if (mode === "cloud" && !resolvedAuth?.apiKey) {
|
||||
throw new Error("Comfy Cloud API key missing");
|
||||
}
|
||||
|
||||
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
|
||||
resolveProviderHttpRequestConfig({
|
||||
baseUrl: normalizeOptionalString(capabilityConfig.baseUrl),
|
||||
defaultBaseUrl:
|
||||
mode === "cloud" ? DEFAULT_COMFY_CLOUD_BASE_URL : DEFAULT_COMFY_LOCAL_BASE_URL,
|
||||
allowPrivateNetwork:
|
||||
mode === "local" || readConfigBoolean(capabilityConfig, "allowPrivateNetwork") === true,
|
||||
defaultHeaders:
|
||||
mode === "cloud"
|
||||
? {
|
||||
"X-API-Key": resolvedAuth?.apiKey ?? "",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
provider: "comfy",
|
||||
capability: params.capability === "music" ? "audio" : params.capability,
|
||||
transport: "http",
|
||||
});
|
||||
const normalizedBaseUrl =
|
||||
normalizeBaseUrl(baseUrl) ||
|
||||
(mode === "cloud" ? DEFAULT_COMFY_CLOUD_BASE_URL : DEFAULT_COMFY_LOCAL_BASE_URL);
|
||||
const networkPolicy = resolveComfyNetworkPolicy({
|
||||
baseUrl: normalizedBaseUrl,
|
||||
allowPrivateNetwork,
|
||||
});
|
||||
|
||||
if (params.inputImage) {
|
||||
if (!inputImageNodeId) {
|
||||
throw new Error(
|
||||
"Comfy edit requests require models.providers.comfy.<capability>.inputImageNodeId to be configured",
|
||||
);
|
||||
}
|
||||
const uploadedName = await uploadInputImage({
|
||||
baseUrl: normalizedBaseUrl,
|
||||
headers: new Headers(headers),
|
||||
timeoutMs,
|
||||
policy: networkPolicy.apiPolicy,
|
||||
dispatcherPolicy,
|
||||
image: params.inputImage,
|
||||
mode,
|
||||
capability: params.capability,
|
||||
});
|
||||
setWorkflowInput({
|
||||
workflow,
|
||||
nodeId: inputImageNodeId,
|
||||
inputName: inputImageInputName,
|
||||
value: uploadedName,
|
||||
});
|
||||
}
|
||||
|
||||
const submitPayload = {
|
||||
prompt: workflow,
|
||||
...(mode === "cloud" && resolvedAuth?.apiKey
|
||||
? { extra_data: { api_key_comfy_org: resolvedAuth.apiKey } }
|
||||
: {}),
|
||||
};
|
||||
|
||||
const promptResponse = await readJsonResponse<ComfyPromptResponse>({
|
||||
url: `${normalizedBaseUrl}${mode === "cloud" ? "/api/prompt" : "/prompt"}`,
|
||||
init: {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(submitPayload),
|
||||
},
|
||||
timeoutMs,
|
||||
policy: networkPolicy.apiPolicy,
|
||||
dispatcherPolicy,
|
||||
auditContext: `comfy-${params.capability}-generate`,
|
||||
errorPrefix: "Comfy workflow submit failed",
|
||||
});
|
||||
|
||||
const promptId = normalizeOptionalString(promptResponse.prompt_id);
|
||||
if (!promptId) {
|
||||
throw new Error("Comfy workflow submit response missing prompt_id");
|
||||
}
|
||||
|
||||
const history =
|
||||
mode === "cloud"
|
||||
? await (async () => {
|
||||
await waitForCloudCompletion({
|
||||
baseUrl: normalizedBaseUrl,
|
||||
promptId,
|
||||
headers: new Headers(headers),
|
||||
timeoutMs,
|
||||
pollIntervalMs,
|
||||
policy: networkPolicy.apiPolicy,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
return await readJsonResponse<unknown>({
|
||||
url: `${normalizedBaseUrl}/api/history_v2/${promptId}`,
|
||||
init: {
|
||||
method: "GET",
|
||||
headers: new Headers(headers),
|
||||
},
|
||||
timeoutMs,
|
||||
policy: networkPolicy.apiPolicy,
|
||||
dispatcherPolicy,
|
||||
auditContext: "comfy-history",
|
||||
errorPrefix: "Comfy history lookup failed",
|
||||
});
|
||||
})()
|
||||
: await waitForLocalHistory({
|
||||
baseUrl: normalizedBaseUrl,
|
||||
promptId,
|
||||
headers: new Headers(headers),
|
||||
timeoutMs,
|
||||
pollIntervalMs,
|
||||
policy: networkPolicy.apiPolicy,
|
||||
dispatcherPolicy,
|
||||
});
|
||||
|
||||
const historyEntry = extractHistoryEntry(history, promptId);
|
||||
if (!historyEntry) {
|
||||
throw new Error(`Comfy history response missing outputs for prompt ${promptId}`);
|
||||
}
|
||||
|
||||
const outputFiles = collectOutputFiles({
|
||||
history: historyEntry,
|
||||
outputNodeId,
|
||||
outputKinds: params.outputKinds,
|
||||
});
|
||||
if (outputFiles.length === 0) {
|
||||
throw new Error(`Comfy workflow ${promptId} completed without ${params.capability} outputs`);
|
||||
}
|
||||
|
||||
const assets: ComfyGeneratedAsset[] = [];
|
||||
let assetIndex = 0;
|
||||
for (const output of outputFiles) {
|
||||
const downloaded = await downloadOutputFile({
|
||||
baseUrl: normalizedBaseUrl,
|
||||
headers: new Headers(headers),
|
||||
timeoutMs,
|
||||
policy: networkPolicy.apiPolicy,
|
||||
dispatcherPolicy,
|
||||
file: output.file,
|
||||
mode,
|
||||
capability: params.capability,
|
||||
});
|
||||
assetIndex += 1;
|
||||
const originalName =
|
||||
normalizeOptionalString(output.file.filename) || normalizeOptionalString(output.file.name);
|
||||
assets.push({
|
||||
buffer: downloaded.buffer,
|
||||
mimeType: downloaded.mimeType,
|
||||
fileName:
|
||||
originalName ||
|
||||
`${params.capability}-${assetIndex}.${inferFileExtension({ mimeType: downloaded.mimeType })}`,
|
||||
nodeId: output.nodeId,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
assets,
|
||||
model: providerModel,
|
||||
promptId,
|
||||
outputNodeIds: Array.from(new Set(outputFiles.map((entry) => entry.nodeId))),
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue