重构主控编排与运行时预热链路,统一工作区提示词/专家调度协议并补齐 wiki 记忆注入与写回闭环。

同时收敛启动与运维脚本默认行为(含 wiki worker)、更新 Admin 可观测性与相关测试,降低首轮时延并提高运行稳定性。

Made-with: Cursor
This commit is contained in:
oliver 2026-04-26 08:34:33 +08:00
parent 4a23b715a2
commit dbbe3add6a
14438 changed files with 2693620 additions and 2546 deletions

View file

@ -0,0 +1,122 @@
import { beforeAll, describe, expect, it } from "vitest";
import { resolveOpenClawAgentDir } from "../../src/agents/agent-paths.js";
import { isLiveTestEnabled } from "../../src/agents/live-test-helpers.js";
import { loadConfig } from "../../src/config/config.js";
import { createTestPluginApi } from "../../test/helpers/plugins/plugin-api.js";
import plugin from "./index.js";
import { getComfyConfig, isComfyCapabilityConfigured } from "./workflow-runtime.js";
const LIVE =
isLiveTestEnabled(["COMFY_LIVE_TEST"]) && (process.env.COMFY_LIVE_TEST ?? "").trim() === "1";
const describeLive = LIVE ? describe : describe.skip;
type RegisteredMusicProvider = {
id: string;
generateMusic: Function;
isConfigured?: Function;
};
function withPluginsEnabled<T>(cfg: T): T {
if (!cfg || typeof cfg !== "object") {
return cfg;
}
const record = cfg as Record<string, unknown>;
return {
...record,
plugins: {
...(record.plugins && typeof record.plugins === "object" ? record.plugins : {}),
enabled: true,
},
} as T;
}
describeLive("comfy live", () => {
let cfg = {} as ReturnType<typeof loadConfig>;
let agentDir = "";
const imageProviders: Array<{ id: string; generateImage: Function; isConfigured?: Function }> =
[];
const musicProviders: RegisteredMusicProvider[] = [];
const videoProviders: Array<{ id: string; generateVideo: Function; isConfigured?: Function }> =
[];
beforeAll(async () => {
cfg = withPluginsEnabled(loadConfig());
agentDir = resolveOpenClawAgentDir();
plugin.register(
createTestPluginApi({
config: cfg as never,
registerImageGenerationProvider(provider) {
imageProviders.push(provider as never);
},
registerMusicGenerationProvider(provider) {
musicProviders.push(provider as never);
},
registerVideoGenerationProvider(provider) {
videoProviders.push(provider as never);
},
}),
);
});
it.skipIf(!isComfyCapabilityConfigured({ cfg: cfg as never, agentDir, capability: "image" }))(
"runs an image workflow",
async () => {
const provider = imageProviders.find((entry) => entry.id === "comfy");
expect(provider).toBeDefined();
const result = await provider!.generateImage({
provider: "comfy",
model: "workflow",
prompt: "A tiny orange lobster icon on a clean background.",
cfg: cfg as never,
agentDir,
});
expect(result.images.length).toBeGreaterThan(0);
expect(result.images[0]?.mimeType.startsWith("image/")).toBe(true);
expect(result.images[0]?.buffer.byteLength).toBeGreaterThan(128);
},
120_000,
);
it.skipIf(!isComfyCapabilityConfigured({ cfg: cfg as never, agentDir, capability: "video" }))(
"runs a video workflow",
async () => {
const provider = videoProviders.find((entry) => entry.id === "comfy");
expect(provider).toBeDefined();
const result = await provider!.generateVideo({
provider: "comfy",
model: "workflow",
prompt: "A tiny paper lobster gently waving, cinematic motion.",
cfg: cfg as never,
agentDir,
});
expect(result.videos.length).toBeGreaterThan(0);
expect(result.videos[0]?.mimeType.startsWith("video/")).toBe(true);
expect(result.videos[0]?.buffer.byteLength).toBeGreaterThan(512);
},
180_000,
);
it.skipIf(!isComfyCapabilityConfigured({ cfg: cfg as never, agentDir, capability: "music" }))(
"runs a music workflow",
async () => {
const provider = musicProviders.find((entry) => entry.id === "comfy");
expect(provider).toBeDefined();
const result = await provider!.generateMusic({
provider: "comfy",
model: "workflow",
prompt: "A gentle ambient synth loop with warm analog pads.",
cfg: cfg as never,
agentDir,
});
expect(result.tracks.length).toBeGreaterThan(0);
expect(result.tracks[0]?.mimeType.startsWith("audio/")).toBe(true);
expect(result.tracks[0]?.buffer.byteLength).toBeGreaterThan(512);
},
180_000,
);
it("documents the effective comfy config shape for live debugging", () => {
const comfyConfig = getComfyConfig(cfg as never);
expect(typeof comfyConfig).toBe("object");
});
});

View file

@ -0,0 +1,355 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
_setComfyFetchGuardForTesting,
buildComfyImageGenerationProvider,
} from "./image-generation-provider.js";
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
fetchWithSsrFGuardMock: vi.fn(),
}));
function parseJsonBody(call: number): Record<string, unknown> {
const request = fetchWithSsrFGuardMock.mock.calls[call - 1]?.[0];
expect(request?.init?.body).toBeTruthy();
return JSON.parse(String(request.init.body)) as Record<string, unknown>;
}
function buildComfyConfig(config: Record<string, unknown>): OpenClawConfig {
return {
models: {
providers: {
comfy: config,
},
},
} as unknown as OpenClawConfig;
}
describe("comfy image-generation provider", () => {
beforeEach(() => {
vi.clearAllMocks();
});
afterEach(() => {
_setComfyFetchGuardForTesting(null);
vi.restoreAllMocks();
});
it("treats local comfy workflows as configured without an API key", () => {
const provider = buildComfyImageGenerationProvider();
expect(
provider.isConfigured?.({
cfg: buildComfyConfig({
workflow: {
"6": { inputs: { text: "" } },
},
promptNodeId: "6",
}),
}),
).toBe(true);
});
it("submits a local workflow, waits for history, and downloads images", async () => {
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
fetchWithSsrFGuardMock
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ prompt_id: "local-prompt-1" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(
JSON.stringify({
"local-prompt-1": {
outputs: {
"9": {
images: [{ filename: "generated.png", subfolder: "", type: "output" }],
},
},
},
}),
{
status: 200,
headers: { "content-type": "application/json" },
},
),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(Buffer.from("png-data"), {
status: 200,
headers: { "content-type": "image/png" },
}),
release: vi.fn(async () => {}),
});
const provider = buildComfyImageGenerationProvider();
const result = await provider.generateImage({
provider: "comfy",
model: "workflow",
prompt: "draw a lobster",
cfg: buildComfyConfig({
workflow: {
"6": { inputs: { text: "" } },
"9": { inputs: {} },
},
promptNodeId: "6",
outputNodeId: "9",
}),
});
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
url: "http://127.0.0.1:8188/prompt",
auditContext: "comfy-image-generate",
}),
);
expect(parseJsonBody(1)).toEqual({
prompt: {
"6": { inputs: { text: "draw a lobster" } },
"9": { inputs: {} },
},
});
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
url: "http://127.0.0.1:8188/history/local-prompt-1",
auditContext: "comfy-history",
}),
);
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
3,
expect.objectContaining({
url: "http://127.0.0.1:8188/view?filename=generated.png&subfolder=&type=output",
auditContext: "comfy-image-download",
}),
);
expect(result).toEqual({
images: [
{
buffer: Buffer.from("png-data"),
mimeType: "image/png",
fileName: "generated.png",
metadata: {
nodeId: "9",
promptId: "local-prompt-1",
},
},
],
model: "workflow",
metadata: {
promptId: "local-prompt-1",
outputNodeIds: ["9"],
},
});
});
it("uploads reference images for local edit workflows", async () => {
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
fetchWithSsrFGuardMock
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ name: "upload.png" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ prompt_id: "local-edit-1" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(
JSON.stringify({
"local-edit-1": {
outputs: {
"9": {
images: [{ filename: "edited.png", subfolder: "", type: "output" }],
},
},
},
}),
{
status: 200,
headers: { "content-type": "application/json" },
},
),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(Buffer.from("edited-data"), {
status: 200,
headers: { "content-type": "image/png" },
}),
release: vi.fn(async () => {}),
});
const provider = buildComfyImageGenerationProvider();
await provider.generateImage({
provider: "comfy",
model: "workflow",
prompt: "turn this into a poster",
cfg: buildComfyConfig({
workflow: {
"6": { inputs: { text: "" } },
"7": { inputs: { image: "" } },
"9": { inputs: {} },
},
promptNodeId: "6",
inputImageNodeId: "7",
outputNodeId: "9",
}),
inputImages: [
{
buffer: Buffer.from("source"),
mimeType: "image/png",
fileName: "source.png",
},
],
});
const uploadRequest = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
expect(uploadRequest?.url).toBe("http://127.0.0.1:8188/upload/image");
expect(uploadRequest?.auditContext).toBe("comfy-image-upload");
expect(uploadRequest?.init?.method).toBe("POST");
const uploadForm = uploadRequest?.init?.body;
expect(uploadForm).toBeInstanceOf(FormData);
expect(uploadForm?.get("type")).toBe("input");
expect(uploadForm?.get("overwrite")).toBe("true");
expect(parseJsonBody(2)).toEqual({
prompt: {
"6": { inputs: { text: "turn this into a poster" } },
"7": { inputs: { image: "upload.png" } },
"9": { inputs: {} },
},
});
});
it("uses cloud endpoints, auth headers, and partner-node extra_data", async () => {
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
apiKey: "comfy-test-key",
source: "env",
mode: "api-key",
});
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
fetchWithSsrFGuardMock
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ prompt_id: "cloud-job-1" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ status: "completed" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(
JSON.stringify({
"cloud-job-1": {
outputs: {
"9": {
images: [{ filename: "cloud.png", subfolder: "", type: "output" }],
},
},
},
}),
{
status: 200,
headers: { "content-type": "application/json" },
},
),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(null, {
status: 302,
headers: { location: "https://cdn.example.com/cloud.png" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(Buffer.from("cloud-data"), {
status: 200,
headers: { "content-type": "image/png" },
}),
release: vi.fn(async () => {}),
});
const provider = buildComfyImageGenerationProvider();
const result = await provider.generateImage({
provider: "comfy",
model: "workflow",
prompt: "cloud workflow prompt",
cfg: buildComfyConfig({
mode: "cloud",
workflow: {
"6": { inputs: { text: "" } },
"9": { inputs: {} },
},
promptNodeId: "6",
outputNodeId: "9",
}),
});
const submitRequest = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
expect(submitRequest?.url).toBe("https://cloud.comfy.org/api/prompt");
expect(submitRequest?.auditContext).toBe("comfy-image-generate");
const submitHeaders = new Headers(submitRequest?.init?.headers);
expect(submitHeaders.get("x-api-key")).toBe("comfy-test-key");
expect(parseJsonBody(1)).toEqual({
prompt: {
"6": { inputs: { text: "cloud workflow prompt" } },
"9": { inputs: {} },
},
extra_data: {
api_key_comfy_org: "comfy-test-key",
},
});
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
url: "https://cloud.comfy.org/api/job/cloud-job-1/status",
auditContext: "comfy-status",
}),
);
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
3,
expect.objectContaining({
url: "https://cloud.comfy.org/api/history_v2/cloud-job-1",
auditContext: "comfy-history",
}),
);
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
4,
expect.objectContaining({
url: "https://cloud.comfy.org/api/view?filename=cloud.png&subfolder=&type=output",
auditContext: "comfy-image-download",
}),
);
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
5,
expect.objectContaining({
url: "https://cdn.example.com/cloud.png",
auditContext: "comfy-image-download",
}),
);
expect(result.metadata).toEqual({
promptId: "cloud-job-1",
outputNodeIds: ["9"],
});
});
});

View file

@ -0,0 +1,79 @@
import type {
GeneratedImageAsset,
ImageGenerationProvider,
} from "openclaw/plugin-sdk/image-generation";
import {
DEFAULT_COMFY_MODEL,
_setComfyFetchGuardForTesting,
isComfyCapabilityConfigured,
runComfyWorkflow,
} from "./workflow-runtime.js";
export { _setComfyFetchGuardForTesting };
export function buildComfyImageGenerationProvider(): ImageGenerationProvider {
return {
id: "comfy",
label: "ComfyUI",
defaultModel: DEFAULT_COMFY_MODEL,
models: [DEFAULT_COMFY_MODEL],
isConfigured: ({ cfg, agentDir }) =>
isComfyCapabilityConfigured({
cfg,
agentDir,
capability: "image",
}),
capabilities: {
generate: {
maxCount: 1,
supportsSize: false,
supportsAspectRatio: false,
supportsResolution: false,
},
edit: {
enabled: true,
maxCount: 1,
maxInputImages: 1,
supportsSize: false,
supportsAspectRatio: false,
supportsResolution: false,
},
},
async generateImage(req) {
if ((req.inputImages?.length ?? 0) > 1) {
throw new Error("Comfy image generation currently supports at most one reference image");
}
const result = await runComfyWorkflow({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
prompt: req.prompt,
model: req.model,
timeoutMs: req.timeoutMs,
capability: "image",
outputKinds: ["images"],
inputImage: req.inputImages?.[0],
});
const images: GeneratedImageAsset[] = result.assets.map((asset) => ({
buffer: asset.buffer,
mimeType: asset.mimeType,
fileName: asset.fileName,
metadata: {
nodeId: asset.nodeId,
promptId: result.promptId,
},
}));
return {
images,
model: result.model,
metadata: {
promptId: result.promptId,
outputNodeIds: result.outputNodeIds,
},
};
},
};
}

View file

@ -0,0 +1,24 @@
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
import { buildComfyImageGenerationProvider } from "./image-generation-provider.js";
import { buildComfyMusicGenerationProvider } from "./music-generation-provider.js";
import { buildComfyVideoGenerationProvider } from "./video-generation-provider.js";
const PROVIDER_ID = "comfy";
export default definePluginEntry({
id: PROVIDER_ID,
name: "ComfyUI Provider",
description: "Bundled ComfyUI workflow media generation provider",
register(api) {
api.registerProvider({
id: PROVIDER_ID,
label: "ComfyUI",
docsPath: "/providers/comfy",
envVars: ["COMFY_API_KEY", "COMFY_CLOUD_API_KEY"],
auth: [],
});
api.registerImageGenerationProvider(buildComfyImageGenerationProvider());
api.registerMusicGenerationProvider(buildComfyMusicGenerationProvider());
api.registerVideoGenerationProvider(buildComfyVideoGenerationProvider());
},
});

View file

@ -0,0 +1,94 @@
import { describe, expect, it, vi } from "vitest";
import { expectExplicitMusicGenerationCapabilities } from "../../test/helpers/media-generation/provider-capability-assertions.js";
import { buildComfyMusicGenerationProvider } from "./music-generation-provider.js";
import { _setComfyFetchGuardForTesting } from "./workflow-runtime.js";
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
fetchWithSsrFGuardMock: vi.fn(),
}));
describe("comfy music-generation provider", () => {
it("registers the workflow model", () => {
const provider = buildComfyMusicGenerationProvider();
expect(provider.defaultModel).toBe("workflow");
expect(provider.models).toEqual(["workflow"]);
expectExplicitMusicGenerationCapabilities(provider);
});
it("runs a music workflow and returns audio outputs", async () => {
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
fetchWithSsrFGuardMock
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ prompt_id: "music-job-1" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(
JSON.stringify({
"music-job-1": {
outputs: {
"9": {
audio: [{ filename: "song.mp3", subfolder: "", type: "output" }],
},
},
},
}),
{
status: 200,
headers: { "content-type": "application/json" },
},
),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(Buffer.from("music-bytes"), {
status: 200,
headers: { "content-type": "audio/mpeg" },
}),
release: vi.fn(async () => {}),
});
const provider = buildComfyMusicGenerationProvider();
const result = await provider.generateMusic({
provider: "comfy",
model: "workflow",
prompt: "gentle ambient synth loop",
cfg: {
models: {
providers: {
comfy: {
music: {
workflow: {
"6": { inputs: { text: "" } },
"9": { inputs: {} },
},
promptNodeId: "6",
outputNodeId: "9",
},
},
},
},
} as never,
});
expect(result).toMatchObject({
model: "workflow",
tracks: [
{
mimeType: "audio/mpeg",
fileName: "song.mp3",
},
],
metadata: {
promptId: "music-job-1",
outputNodeIds: ["9"],
inputImageCount: 0,
},
});
expect(result.tracks[0]?.buffer).toEqual(Buffer.from("music-bytes"));
});
});

View file

@ -0,0 +1,88 @@
import type {
GeneratedMusicAsset,
MusicGenerationProvider,
MusicGenerationSourceImage,
} from "openclaw/plugin-sdk/music-generation";
import {
DEFAULT_COMFY_MODEL,
isComfyCapabilityConfigured,
runComfyWorkflow,
} from "./workflow-runtime.js";
const COMFY_MAX_INPUT_IMAGES = 1;
function toGeneratedTrack(asset: {
buffer: Buffer;
mimeType: string;
fileName: string;
}): GeneratedMusicAsset {
return {
buffer: asset.buffer,
mimeType: asset.mimeType,
fileName: asset.fileName,
};
}
function resolveInputImage(inputImage: MusicGenerationSourceImage | undefined) {
if (!inputImage) {
return undefined;
}
if (!inputImage.buffer) {
throw new Error("Comfy music generation requires loaded reference image bytes.");
}
return {
buffer: inputImage.buffer,
mimeType: inputImage.mimeType ?? "image/png",
fileName: inputImage.fileName,
};
}
export function buildComfyMusicGenerationProvider(): MusicGenerationProvider {
return {
id: "comfy",
label: "ComfyUI",
defaultModel: DEFAULT_COMFY_MODEL,
models: [DEFAULT_COMFY_MODEL],
isConfigured: ({ cfg, agentDir }) =>
isComfyCapabilityConfigured({
cfg,
agentDir,
capability: "music",
}),
capabilities: {
generate: {},
edit: {
enabled: true,
maxInputImages: COMFY_MAX_INPUT_IMAGES,
},
},
async generateMusic(req) {
if ((req.inputImages?.length ?? 0) > COMFY_MAX_INPUT_IMAGES) {
throw new Error(
`Comfy music generation supports at most ${COMFY_MAX_INPUT_IMAGES} reference image.`,
);
}
const result = await runComfyWorkflow({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
prompt: req.prompt,
model: req.model,
capability: "music",
outputKinds: ["audio"],
inputImage: resolveInputImage(req.inputImages?.[0]),
});
return {
tracks: result.assets.map(toGeneratedTrack),
model: result.model,
metadata: {
promptId: result.promptId,
outputNodeIds: result.outputNodeIds,
inputImageCount: req.inputImages?.length ?? 0,
},
};
},
};
}

View file

@ -0,0 +1,104 @@
{
"id": "comfy",
"enabledByDefault": true,
"providers": ["comfy"],
"providerAuthEnvVars": {
"comfy": ["COMFY_API_KEY", "COMFY_CLOUD_API_KEY"]
},
"contracts": {
"imageGenerationProviders": ["comfy"],
"musicGenerationProviders": ["comfy"],
"videoGenerationProviders": ["comfy"]
},
"configSchema": {
"type": "object",
"additionalProperties": false,
"properties": {
"mode": {
"type": "string",
"enum": ["local", "cloud"]
},
"baseUrl": {
"type": "string"
},
"apiKey": {
"type": ["string", "object"]
},
"allowPrivateNetwork": {
"type": "boolean"
},
"workflowPath": {
"type": "string"
},
"workflow": {
"type": "object"
},
"promptNodeId": {
"type": "string"
},
"promptInputName": {
"type": "string"
},
"inputImageNodeId": {
"type": "string"
},
"inputImageInputName": {
"type": "string"
},
"outputNodeId": {
"type": "string"
},
"pollIntervalMs": {
"type": "integer",
"minimum": 100
},
"timeoutMs": {
"type": "integer",
"minimum": 1000
},
"image": {
"type": "object",
"additionalProperties": false,
"properties": {
"workflowPath": { "type": "string" },
"workflow": { "type": "object" },
"promptNodeId": { "type": "string" },
"promptInputName": { "type": "string" },
"inputImageNodeId": { "type": "string" },
"inputImageInputName": { "type": "string" },
"outputNodeId": { "type": "string" },
"pollIntervalMs": { "type": "integer", "minimum": 100 },
"timeoutMs": { "type": "integer", "minimum": 1000 }
}
},
"video": {
"type": "object",
"additionalProperties": false,
"properties": {
"workflowPath": { "type": "string" },
"workflow": { "type": "object" },
"promptNodeId": { "type": "string" },
"promptInputName": { "type": "string" },
"inputImageNodeId": { "type": "string" },
"inputImageInputName": { "type": "string" },
"outputNodeId": { "type": "string" },
"pollIntervalMs": { "type": "integer", "minimum": 100 },
"timeoutMs": { "type": "integer", "minimum": 1000 }
}
},
"music": {
"type": "object",
"additionalProperties": false,
"properties": {
"workflowPath": { "type": "string" },
"workflow": { "type": "object" },
"promptNodeId": { "type": "string" },
"promptInputName": { "type": "string" },
"outputNodeId": { "type": "string" },
"pollIntervalMs": { "type": "integer", "minimum": 100 },
"timeoutMs": { "type": "integer", "minimum": 1000 }
}
}
}
}
}

View file

@ -0,0 +1,15 @@
{
"name": "@openclaw/comfy-provider",
"version": "2026.4.20",
"private": true,
"description": "OpenClaw ComfyUI provider plugin",
"type": "module",
"devDependencies": {
"@openclaw/plugin-sdk": "workspace:*"
},
"openclaw": {
"extensions": [
"./index.ts"
]
}
}

View file

@ -0,0 +1,11 @@
import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js";
describePluginRegistrationContract({
pluginId: "comfy",
providerIds: ["comfy"],
imageGenerationProviderIds: ["comfy"],
musicGenerationProviderIds: ["comfy"],
videoGenerationProviderIds: ["comfy"],
requireGenerateImage: true,
requireGenerateVideo: true,
});

View file

@ -0,0 +1,16 @@
{
"extends": "../tsconfig.package-boundary.base.json",
"compilerOptions": {
"rootDir": "."
},
"include": ["./*.ts", "./src/**/*.ts"],
"exclude": [
"./**/*.test.ts",
"./dist/**",
"./node_modules/**",
"./src/test-support/**",
"./src/**/*test-helpers.ts",
"./src/**/*test-harness.ts",
"./src/**/*test-support.ts"
]
}

View file

@ -0,0 +1,245 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import * as providerAuth from "openclaw/plugin-sdk/provider-auth-runtime";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { expectExplicitVideoGenerationCapabilities } from "../../test/helpers/media-generation/provider-capability-assertions.js";
import {
_setComfyFetchGuardForTesting,
buildComfyVideoGenerationProvider,
} from "./video-generation-provider.js";
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
fetchWithSsrFGuardMock: vi.fn(),
}));
function parseJsonBody(call: number): Record<string, unknown> {
const request = fetchWithSsrFGuardMock.mock.calls[call - 1]?.[0];
expect(request?.init?.body).toBeTruthy();
return JSON.parse(String(request.init.body)) as Record<string, unknown>;
}
function buildComfyConfig(config: Record<string, unknown>): OpenClawConfig {
return {
models: {
providers: {
comfy: config,
},
},
} as unknown as OpenClawConfig;
}
describe("comfy video-generation provider", () => {
beforeEach(() => {
vi.clearAllMocks();
});
afterEach(() => {
_setComfyFetchGuardForTesting(null);
vi.restoreAllMocks();
});
it("declares explicit mode capabilities", () => {
expectExplicitVideoGenerationCapabilities(buildComfyVideoGenerationProvider());
});
it("treats local comfy video workflows as configured without an API key", () => {
const provider = buildComfyVideoGenerationProvider();
expect(
provider.isConfigured?.({
cfg: buildComfyConfig({
video: {
workflow: {
"6": { inputs: { text: "" } },
},
promptNodeId: "6",
},
}),
}),
).toBe(true);
});
it("submits a local workflow, waits for history, and downloads videos", async () => {
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
fetchWithSsrFGuardMock
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ prompt_id: "local-video-1" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(
JSON.stringify({
"local-video-1": {
outputs: {
"9": {
gifs: [{ filename: "generated.mp4", subfolder: "", type: "output" }],
},
},
},
}),
{
status: 200,
headers: { "content-type": "application/json" },
},
),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(Buffer.from("mp4-data"), {
status: 200,
headers: { "content-type": "video/mp4" },
}),
release: vi.fn(async () => {}),
});
const provider = buildComfyVideoGenerationProvider();
const result = await provider.generateVideo({
provider: "comfy",
model: "workflow",
prompt: "animate a lobster",
cfg: buildComfyConfig({
video: {
workflow: {
"6": { inputs: { text: "" } },
"9": { inputs: {} },
},
promptNodeId: "6",
outputNodeId: "9",
},
}),
});
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
url: "http://127.0.0.1:8188/prompt",
auditContext: "comfy-video-generate",
}),
);
expect(parseJsonBody(1)).toEqual({
prompt: {
"6": { inputs: { text: "animate a lobster" } },
"9": { inputs: {} },
},
});
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
url: "http://127.0.0.1:8188/history/local-video-1",
auditContext: "comfy-history",
}),
);
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
3,
expect.objectContaining({
url: "http://127.0.0.1:8188/view?filename=generated.mp4&subfolder=&type=output",
auditContext: "comfy-video-download",
}),
);
expect(result).toEqual({
videos: [
{
buffer: Buffer.from("mp4-data"),
mimeType: "video/mp4",
fileName: "generated.mp4",
metadata: {
nodeId: "9",
promptId: "local-video-1",
},
},
],
model: "workflow",
metadata: {
promptId: "local-video-1",
outputNodeIds: ["9"],
},
});
});
it("uses cloud endpoints for video workflows", async () => {
vi.spyOn(providerAuth, "resolveApiKeyForProvider").mockResolvedValue({
apiKey: "comfy-test-key",
source: "env",
mode: "api-key",
});
_setComfyFetchGuardForTesting(fetchWithSsrFGuardMock);
fetchWithSsrFGuardMock
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ prompt_id: "cloud-video-1" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(JSON.stringify({ status: "completed" }), {
status: 200,
headers: { "content-type": "application/json" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(
JSON.stringify({
"cloud-video-1": {
outputs: {
"9": {
gifs: [{ filename: "cloud.mp4", subfolder: "", type: "output" }],
},
},
},
}),
{
status: 200,
headers: { "content-type": "application/json" },
},
),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(null, {
status: 302,
headers: { location: "https://cdn.example.com/cloud.mp4" },
}),
release: vi.fn(async () => {}),
})
.mockResolvedValueOnce({
response: new Response(Buffer.from("cloud-video-data"), {
status: 200,
headers: { "content-type": "video/mp4" },
}),
release: vi.fn(async () => {}),
});
const provider = buildComfyVideoGenerationProvider();
const result = await provider.generateVideo({
provider: "comfy",
model: "workflow",
prompt: "cloud video workflow",
cfg: buildComfyConfig({
mode: "cloud",
video: {
workflow: {
"6": { inputs: { text: "" } },
"9": { inputs: {} },
},
promptNodeId: "6",
outputNodeId: "9",
},
}),
});
expect(fetchWithSsrFGuardMock).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
url: "https://cloud.comfy.org/api/prompt",
auditContext: "comfy-video-generate",
}),
);
expect(result.metadata).toEqual({
promptId: "cloud-video-1",
outputNodeIds: ["9"],
});
});
});

View file

@ -0,0 +1,104 @@
import type {
GeneratedVideoAsset,
VideoGenerationProvider,
VideoGenerationSourceAsset,
} from "openclaw/plugin-sdk/video-generation";
import {
DEFAULT_COMFY_MODEL,
_setComfyFetchGuardForTesting,
isComfyCapabilityConfigured,
runComfyWorkflow,
} from "./workflow-runtime.js";
export { _setComfyFetchGuardForTesting };
function toComfyInputImage(inputImage?: VideoGenerationSourceAsset) {
if (!inputImage) {
return undefined;
}
if (!inputImage.buffer || !inputImage.mimeType) {
throw new Error("Comfy video generation requires a local reference image file");
}
return {
buffer: inputImage.buffer,
mimeType: inputImage.mimeType,
fileName: inputImage.fileName,
};
}
export function buildComfyVideoGenerationProvider(): VideoGenerationProvider {
return {
id: "comfy",
label: "ComfyUI",
defaultModel: DEFAULT_COMFY_MODEL,
models: [DEFAULT_COMFY_MODEL],
isConfigured: ({ cfg, agentDir }) =>
isComfyCapabilityConfigured({
cfg,
agentDir,
capability: "video",
}),
capabilities: {
generate: {
maxVideos: 1,
supportsSize: false,
supportsAspectRatio: false,
supportsResolution: false,
supportsAudio: false,
supportsWatermark: false,
},
imageToVideo: {
enabled: true,
maxVideos: 1,
maxInputImages: 1,
supportsSize: false,
supportsAspectRatio: false,
supportsResolution: false,
supportsAudio: false,
supportsWatermark: false,
},
videoToVideo: {
enabled: false,
},
},
async generateVideo(req) {
if ((req.inputImages?.length ?? 0) > 1) {
throw new Error("Comfy video generation currently supports at most one reference image");
}
if ((req.inputVideos?.length ?? 0) > 0) {
throw new Error("Comfy video generation does not support input videos");
}
const result = await runComfyWorkflow({
cfg: req.cfg,
agentDir: req.agentDir,
authStore: req.authStore,
prompt: req.prompt,
model: req.model,
timeoutMs: req.timeoutMs,
capability: "video",
outputKinds: ["gifs", "videos"],
inputImage: toComfyInputImage(req.inputImages?.[0]),
});
const videos: GeneratedVideoAsset[] = result.assets.map((asset) => ({
buffer: asset.buffer,
mimeType: asset.mimeType,
fileName: asset.fileName,
metadata: {
nodeId: asset.nodeId,
promptId: result.promptId,
},
}));
return {
videos,
model: result.model,
metadata: {
promptId: result.promptId,
outputNodeIds: result.outputNodeIds,
},
};
},
};
}

View file

@ -0,0 +1,780 @@
import fs from "node:fs/promises";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import {
isProviderApiKeyConfigured,
type AuthProfileStore,
} from "openclaw/plugin-sdk/provider-auth";
import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
import {
assertOkOrThrowHttpError,
normalizeBaseUrl,
resolveProviderHttpRequestConfig,
} from "openclaw/plugin-sdk/provider-http";
import {
buildHostnameAllowlistPolicyFromSuffixAllowlist,
fetchWithSsrFGuard,
isPrivateOrLoopbackHost,
ssrfPolicyFromDangerouslyAllowPrivateNetwork,
type SsrFPolicy,
} from "openclaw/plugin-sdk/ssrf-runtime";
import {
isRecord,
normalizeOptionalLowercaseString,
normalizeOptionalString,
resolveUserPath,
} from "openclaw/plugin-sdk/text-runtime";
const DEFAULT_COMFY_LOCAL_BASE_URL = "http://127.0.0.1:8188";
const DEFAULT_COMFY_CLOUD_BASE_URL = "https://cloud.comfy.org";
const DEFAULT_PROMPT_INPUT_NAME = "text";
const DEFAULT_INPUT_IMAGE_INPUT_NAME = "image";
const DEFAULT_POLL_INTERVAL_MS = 1_500;
const DEFAULT_TIMEOUT_MS = 5 * 60_000;
export const DEFAULT_COMFY_MODEL = "workflow";
export type ComfyMode = "local" | "cloud";
export type ComfyCapability = "image" | "music" | "video";
export type ComfyOutputKind = "audio" | "gifs" | "images" | "videos";
export type ComfyWorkflow = Record<string, unknown>;
export type ComfyProviderConfig = Record<string, unknown>;
type ComfyFetchGuardParams = Parameters<typeof fetchWithSsrFGuard>[0];
type ComfyDispatcherPolicy = ComfyFetchGuardParams["dispatcherPolicy"];
type ComfyPromptResponse = {
prompt_id?: string;
};
type ComfyOutputFile = {
filename?: string;
name?: string;
subfolder?: string;
type?: string;
};
type ComfyHistoryOutputEntry = Partial<Record<ComfyOutputKind, ComfyOutputFile[]>>;
type ComfyHistoryEntry = {
outputs?: Record<string, ComfyHistoryOutputEntry>;
};
type ComfyUploadResponse = {
name?: string;
filename?: string;
};
type ComfyStatusResponse = {
status?: string;
message?: string;
error?: string;
};
type ComfyNetworkPolicy = {
apiPolicy?: SsrFPolicy;
};
export type ComfySourceImage = {
buffer: Buffer;
mimeType: string;
fileName?: string;
};
export type ComfyGeneratedAsset = {
buffer: Buffer;
mimeType: string;
fileName: string;
nodeId: string;
};
export type ComfyWorkflowResult = {
assets: ComfyGeneratedAsset[];
model: string;
promptId: string;
outputNodeIds: string[];
};
let comfyFetchGuard = fetchWithSsrFGuard;
export function _setComfyFetchGuardForTesting(impl: typeof fetchWithSsrFGuard | null): void {
comfyFetchGuard = impl ?? fetchWithSsrFGuard;
}
function readConfigBoolean(config: ComfyProviderConfig, key: string): boolean | undefined {
const value = config[key];
return typeof value === "boolean" ? value : undefined;
}
function readConfigInteger(config: ComfyProviderConfig, key: string): number | undefined {
const value = config[key];
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined;
}
function mergeSsrFPolicies(...policies: Array<SsrFPolicy | undefined>): SsrFPolicy | undefined {
const merged: SsrFPolicy = {};
for (const policy of policies) {
if (!policy) {
continue;
}
if (policy.allowPrivateNetwork) {
merged.allowPrivateNetwork = true;
}
if (policy.dangerouslyAllowPrivateNetwork) {
merged.dangerouslyAllowPrivateNetwork = true;
}
if (policy.allowRfc2544BenchmarkRange) {
merged.allowRfc2544BenchmarkRange = true;
}
if (policy.allowedHostnames?.length) {
merged.allowedHostnames = Array.from(
new Set([...(merged.allowedHostnames ?? []), ...policy.allowedHostnames]),
);
}
if (policy.hostnameAllowlist?.length) {
merged.hostnameAllowlist = Array.from(
new Set([...(merged.hostnameAllowlist ?? []), ...policy.hostnameAllowlist]),
);
}
}
return Object.keys(merged).length > 0 ? merged : undefined;
}
export function getComfyConfig(cfg?: OpenClawConfig): ComfyProviderConfig {
const raw = cfg?.models?.providers?.comfy;
return isRecord(raw) ? raw : {};
}
function stripNestedCapabilityConfig(config: ComfyProviderConfig): ComfyProviderConfig {
const next = { ...config };
delete next.image;
delete next.video;
delete next.music;
return next;
}
export function getComfyCapabilityConfig(
config: ComfyProviderConfig,
capability: ComfyCapability,
): ComfyProviderConfig {
const shared = stripNestedCapabilityConfig(config);
const nested = config[capability];
if (!isRecord(nested)) {
return shared;
}
return { ...shared, ...nested };
}
export function resolveComfyMode(config: ComfyProviderConfig): ComfyMode {
return normalizeOptionalString(config.mode) === "cloud" ? "cloud" : "local";
}
function getRequiredConfigString(config: ComfyProviderConfig, key: string): string {
const value = normalizeOptionalString(config[key]);
if (!value) {
throw new Error(`models.providers.comfy.${key} is required`);
}
return value;
}
function resolveComfyWorkflowSource(config: ComfyProviderConfig): {
workflow?: ComfyWorkflow;
workflowPath?: string;
} {
const workflow = config.workflow;
if (isRecord(workflow)) {
return { workflow: structuredClone(workflow) };
}
const workflowPath = normalizeOptionalString(config.workflowPath);
return { workflowPath };
}
async function loadComfyWorkflow(config: ComfyProviderConfig): Promise<ComfyWorkflow> {
const source = resolveComfyWorkflowSource(config);
if (source.workflow) {
return source.workflow;
}
if (!source.workflowPath) {
throw new Error("models.providers.comfy.<capability>.workflow or workflowPath is required");
}
const resolvedPath = resolveUserPath(source.workflowPath);
const raw = await fs.readFile(resolvedPath, "utf8");
const parsed = JSON.parse(raw) as unknown;
if (!isRecord(parsed)) {
throw new Error(`Comfy workflow at ${resolvedPath} must be a JSON object`);
}
return parsed;
}
function setWorkflowInput(params: {
workflow: ComfyWorkflow;
nodeId: string;
inputName: string;
value: unknown;
}): void {
const node = params.workflow[params.nodeId];
if (!isRecord(node)) {
throw new Error(`Comfy workflow missing node "${params.nodeId}"`);
}
const inputs = node.inputs;
if (!isRecord(inputs)) {
throw new Error(`Comfy workflow node "${params.nodeId}" is missing an inputs object`);
}
inputs[params.inputName] = params.value;
}
function resolveComfyNetworkPolicy(params: {
baseUrl: string;
allowPrivateNetwork: boolean;
}): ComfyNetworkPolicy {
let parsed: URL;
try {
parsed = new URL(params.baseUrl);
} catch {
return {};
}
const hostname = normalizeOptionalLowercaseString(parsed.hostname) ?? "";
if (!hostname || !params.allowPrivateNetwork || !isPrivateOrLoopbackHost(hostname)) {
return {};
}
const hostnamePolicy = buildHostnameAllowlistPolicyFromSuffixAllowlist([hostname]);
const privateNetworkPolicy = ssrfPolicyFromDangerouslyAllowPrivateNetwork(true);
return {
apiPolicy: mergeSsrFPolicies(hostnamePolicy, privateNetworkPolicy),
};
}
async function readJsonResponse<T>(params: {
url: string;
init?: RequestInit;
timeoutMs?: number;
policy?: SsrFPolicy;
dispatcherPolicy?: ComfyDispatcherPolicy;
auditContext: string;
errorPrefix: string;
}): Promise<T> {
const { response, release } = await comfyFetchGuard({
url: params.url,
init: params.init,
timeoutMs: params.timeoutMs,
policy: params.policy,
dispatcherPolicy: params.dispatcherPolicy,
auditContext: params.auditContext,
});
try {
await assertOkOrThrowHttpError(response, params.errorPrefix);
return (await response.json()) as T;
} finally {
await release();
}
}
function inferFileExtension(params: { fileName?: string; mimeType?: string }): string {
const normalizedMime = normalizeOptionalLowercaseString(params.mimeType);
if (normalizedMime?.includes("jpeg")) {
return "jpg";
}
if (normalizedMime?.includes("png")) {
return "png";
}
if (normalizedMime?.includes("webm")) {
return "webm";
}
if (normalizedMime?.includes("mp4")) {
return "mp4";
}
if (normalizedMime?.includes("mpeg")) {
return "mp3";
}
if (normalizedMime?.includes("wav")) {
return "wav";
}
const fileName = params.fileName?.trim();
if (!fileName) {
return "bin";
}
const dotIndex = fileName.lastIndexOf(".");
if (dotIndex < 0 || dotIndex === fileName.length - 1) {
return "bin";
}
return fileName.slice(dotIndex + 1);
}
function toBlobBytes(buffer: Buffer): ArrayBuffer {
const arrayBuffer = new ArrayBuffer(buffer.byteLength);
new Uint8Array(arrayBuffer).set(buffer);
return arrayBuffer;
}
async function uploadInputImage(params: {
baseUrl: string;
headers: Headers;
timeoutMs: number;
policy?: SsrFPolicy;
dispatcherPolicy?: ComfyDispatcherPolicy;
image: ComfySourceImage;
mode: ComfyMode;
capability: ComfyCapability;
}): Promise<string> {
const form = new FormData();
form.set(
"image",
new Blob([toBlobBytes(params.image.buffer)], { type: params.image.mimeType }),
normalizeOptionalString(params.image.fileName) ||
`input.${inferFileExtension({ mimeType: params.image.mimeType })}`,
);
form.set("type", "input");
form.set("overwrite", "true");
const headers = new Headers(params.headers);
headers.delete("Content-Type");
const payload = await readJsonResponse<ComfyUploadResponse>({
url: `${params.baseUrl}${params.mode === "cloud" ? "/api/upload/image" : "/upload/image"}`,
init: {
method: "POST",
headers,
body: form,
},
timeoutMs: params.timeoutMs,
policy: params.policy,
dispatcherPolicy: params.dispatcherPolicy,
auditContext: `comfy-${params.capability}-upload`,
errorPrefix: "Comfy image upload failed",
});
const uploadedName =
normalizeOptionalString(payload.filename) || normalizeOptionalString(payload.name);
if (!uploadedName) {
throw new Error("Comfy image upload response missing filename");
}
return uploadedName;
}
function extractHistoryEntry(history: unknown, promptId: string): ComfyHistoryEntry | null {
if (!isRecord(history)) {
return null;
}
const directOutputs = history.outputs;
if (isRecord(directOutputs)) {
return history as ComfyHistoryEntry;
}
const nested = history[promptId];
if (isRecord(nested)) {
return nested as ComfyHistoryEntry;
}
return null;
}
async function waitForLocalHistory(params: {
baseUrl: string;
promptId: string;
headers: Headers;
timeoutMs: number;
pollIntervalMs: number;
policy?: SsrFPolicy;
dispatcherPolicy?: ComfyDispatcherPolicy;
}): Promise<ComfyHistoryEntry> {
const deadline = Date.now() + params.timeoutMs;
while (Date.now() <= deadline) {
const history = await readJsonResponse<unknown>({
url: `${params.baseUrl}/history/${params.promptId}`,
init: {
method: "GET",
headers: params.headers,
},
timeoutMs: params.timeoutMs,
policy: params.policy,
dispatcherPolicy: params.dispatcherPolicy,
auditContext: "comfy-history",
errorPrefix: "Comfy history lookup failed",
});
const entry = extractHistoryEntry(history, params.promptId);
if (entry?.outputs && Object.keys(entry.outputs).length > 0) {
return entry;
}
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
}
throw new Error(`Comfy workflow did not finish within ${Math.ceil(params.timeoutMs / 1000)}s`);
}
async function waitForCloudCompletion(params: {
baseUrl: string;
promptId: string;
headers: Headers;
timeoutMs: number;
pollIntervalMs: number;
policy?: SsrFPolicy;
dispatcherPolicy?: ComfyDispatcherPolicy;
}): Promise<void> {
const deadline = Date.now() + params.timeoutMs;
while (Date.now() <= deadline) {
const status = await readJsonResponse<ComfyStatusResponse>({
url: `${params.baseUrl}/api/job/${params.promptId}/status`,
init: {
method: "GET",
headers: params.headers,
},
timeoutMs: params.timeoutMs,
policy: params.policy,
dispatcherPolicy: params.dispatcherPolicy,
auditContext: "comfy-status",
errorPrefix: "Comfy status lookup failed",
});
if (status.status === "completed") {
return;
}
if (status.status === "failed" || status.status === "cancelled") {
throw new Error(
`Comfy workflow ${status.status}: ${status.error ?? status.message ?? params.promptId}`,
);
}
await new Promise((resolve) => setTimeout(resolve, params.pollIntervalMs));
}
throw new Error(`Comfy workflow did not finish within ${Math.ceil(params.timeoutMs / 1000)}s`);
}
function collectOutputFiles(params: {
history: ComfyHistoryEntry;
outputNodeId?: string;
outputKinds: readonly ComfyOutputKind[];
}): Array<{ nodeId: string; file: ComfyOutputFile }> {
const outputs = params.history.outputs;
if (!outputs) {
return [];
}
const nodeIds = params.outputNodeId ? [params.outputNodeId] : Object.keys(outputs);
const files: Array<{ nodeId: string; file: ComfyOutputFile }> = [];
for (const nodeId of nodeIds) {
const entry = outputs[nodeId];
if (!entry) {
continue;
}
for (const kind of params.outputKinds) {
const bucket = entry[kind];
if (!Array.isArray(bucket)) {
continue;
}
for (const file of bucket) {
files.push({ nodeId, file });
}
}
}
return files;
}
async function downloadOutputFile(params: {
baseUrl: string;
headers: Headers;
timeoutMs: number;
policy?: SsrFPolicy;
dispatcherPolicy?: ComfyDispatcherPolicy;
file: ComfyOutputFile;
mode: ComfyMode;
capability: ComfyCapability;
}): Promise<{ buffer: Buffer; mimeType: string }> {
const fileName =
normalizeOptionalString(params.file.filename) || normalizeOptionalString(params.file.name);
if (!fileName) {
throw new Error("Comfy output entry missing filename");
}
const query = new URLSearchParams({
filename: fileName,
subfolder: normalizeOptionalString(params.file.subfolder) ?? "",
type: normalizeOptionalString(params.file.type) ?? "output",
});
const viewPath = params.mode === "cloud" ? "/api/view" : "/view";
const auditContext = `comfy-${params.capability}-download`;
const firstResponse = await comfyFetchGuard({
url: `${params.baseUrl}${viewPath}?${query.toString()}`,
init: {
method: "GET",
headers: params.headers,
...(params.mode === "cloud" ? { redirect: "manual" } : {}),
},
timeoutMs: params.timeoutMs,
policy: params.policy,
dispatcherPolicy: params.dispatcherPolicy,
auditContext,
});
try {
if (
params.mode === "cloud" &&
[301, 302, 303, 307, 308].includes(firstResponse.response.status)
) {
const redirectUrl = normalizeOptionalString(firstResponse.response.headers.get("location"));
if (!redirectUrl) {
throw new Error("Comfy cloud output redirect missing location header");
}
const redirected = await comfyFetchGuard({
url: redirectUrl,
init: {
method: "GET",
},
timeoutMs: params.timeoutMs,
dispatcherPolicy: params.dispatcherPolicy,
auditContext,
});
try {
await assertOkOrThrowHttpError(redirected.response, "Comfy output download failed");
const mimeType =
normalizeOptionalString(redirected.response.headers.get("content-type")) ||
"application/octet-stream";
return {
buffer: Buffer.from(await redirected.response.arrayBuffer()),
mimeType,
};
} finally {
await redirected.release();
}
}
await assertOkOrThrowHttpError(firstResponse.response, "Comfy output download failed");
const mimeType =
normalizeOptionalString(firstResponse.response.headers.get("content-type")) ||
"application/octet-stream";
return {
buffer: Buffer.from(await firstResponse.response.arrayBuffer()),
mimeType,
};
} finally {
await firstResponse.release();
}
}
export function isComfyCapabilityConfigured(params: {
cfg?: OpenClawConfig;
agentDir?: string;
capability: ComfyCapability;
}): boolean {
const config = getComfyConfig(params.cfg);
const capabilityConfig = getComfyCapabilityConfig(config, params.capability);
const hasWorkflow = Boolean(
resolveComfyWorkflowSource(capabilityConfig).workflow ||
normalizeOptionalString(capabilityConfig.workflowPath),
);
const hasPromptNode = Boolean(normalizeOptionalString(capabilityConfig.promptNodeId));
if (!hasWorkflow || !hasPromptNode) {
return false;
}
if (resolveComfyMode(capabilityConfig) === "local") {
return true;
}
return isProviderApiKeyConfigured({
provider: "comfy",
agentDir: params.agentDir,
});
}
export async function runComfyWorkflow(params: {
cfg: OpenClawConfig;
agentDir?: string;
authStore?: AuthProfileStore;
prompt: string;
model?: string;
timeoutMs?: number;
capability: ComfyCapability;
outputKinds: readonly ComfyOutputKind[];
inputImage?: ComfySourceImage;
}): Promise<ComfyWorkflowResult> {
const config = getComfyConfig(params.cfg);
const capabilityConfig = getComfyCapabilityConfig(config, params.capability);
const mode = resolveComfyMode(capabilityConfig);
const workflow = await loadComfyWorkflow(capabilityConfig);
const promptNodeId = getRequiredConfigString(capabilityConfig, "promptNodeId");
const promptInputName =
normalizeOptionalString(capabilityConfig.promptInputName) ?? DEFAULT_PROMPT_INPUT_NAME;
const inputImageNodeId = normalizeOptionalString(capabilityConfig.inputImageNodeId);
const inputImageInputName =
normalizeOptionalString(capabilityConfig.inputImageInputName) ?? DEFAULT_INPUT_IMAGE_INPUT_NAME;
const outputNodeId = normalizeOptionalString(capabilityConfig.outputNodeId);
const pollIntervalMs =
readConfigInteger(capabilityConfig, "pollIntervalMs") ?? DEFAULT_POLL_INTERVAL_MS;
const timeoutMs =
readConfigInteger(capabilityConfig, "timeoutMs") ?? params.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const providerModel = normalizeOptionalString(params.model) || DEFAULT_COMFY_MODEL;
setWorkflowInput({
workflow,
nodeId: promptNodeId,
inputName: promptInputName,
value: params.prompt,
});
const resolvedAuth =
mode === "cloud"
? await resolveApiKeyForProvider({
provider: "comfy",
cfg: params.cfg,
agentDir: params.agentDir,
store: params.authStore,
})
: null;
if (mode === "cloud" && !resolvedAuth?.apiKey) {
throw new Error("Comfy Cloud API key missing");
}
const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
resolveProviderHttpRequestConfig({
baseUrl: normalizeOptionalString(capabilityConfig.baseUrl),
defaultBaseUrl:
mode === "cloud" ? DEFAULT_COMFY_CLOUD_BASE_URL : DEFAULT_COMFY_LOCAL_BASE_URL,
allowPrivateNetwork:
mode === "local" || readConfigBoolean(capabilityConfig, "allowPrivateNetwork") === true,
defaultHeaders:
mode === "cloud"
? {
"X-API-Key": resolvedAuth?.apiKey ?? "",
"Content-Type": "application/json",
}
: {
"Content-Type": "application/json",
},
provider: "comfy",
capability: params.capability === "music" ? "audio" : params.capability,
transport: "http",
});
const normalizedBaseUrl =
normalizeBaseUrl(baseUrl) ||
(mode === "cloud" ? DEFAULT_COMFY_CLOUD_BASE_URL : DEFAULT_COMFY_LOCAL_BASE_URL);
const networkPolicy = resolveComfyNetworkPolicy({
baseUrl: normalizedBaseUrl,
allowPrivateNetwork,
});
if (params.inputImage) {
if (!inputImageNodeId) {
throw new Error(
"Comfy edit requests require models.providers.comfy.<capability>.inputImageNodeId to be configured",
);
}
const uploadedName = await uploadInputImage({
baseUrl: normalizedBaseUrl,
headers: new Headers(headers),
timeoutMs,
policy: networkPolicy.apiPolicy,
dispatcherPolicy,
image: params.inputImage,
mode,
capability: params.capability,
});
setWorkflowInput({
workflow,
nodeId: inputImageNodeId,
inputName: inputImageInputName,
value: uploadedName,
});
}
const submitPayload = {
prompt: workflow,
...(mode === "cloud" && resolvedAuth?.apiKey
? { extra_data: { api_key_comfy_org: resolvedAuth.apiKey } }
: {}),
};
const promptResponse = await readJsonResponse<ComfyPromptResponse>({
url: `${normalizedBaseUrl}${mode === "cloud" ? "/api/prompt" : "/prompt"}`,
init: {
method: "POST",
headers,
body: JSON.stringify(submitPayload),
},
timeoutMs,
policy: networkPolicy.apiPolicy,
dispatcherPolicy,
auditContext: `comfy-${params.capability}-generate`,
errorPrefix: "Comfy workflow submit failed",
});
const promptId = normalizeOptionalString(promptResponse.prompt_id);
if (!promptId) {
throw new Error("Comfy workflow submit response missing prompt_id");
}
const history =
mode === "cloud"
? await (async () => {
await waitForCloudCompletion({
baseUrl: normalizedBaseUrl,
promptId,
headers: new Headers(headers),
timeoutMs,
pollIntervalMs,
policy: networkPolicy.apiPolicy,
dispatcherPolicy,
});
return await readJsonResponse<unknown>({
url: `${normalizedBaseUrl}/api/history_v2/${promptId}`,
init: {
method: "GET",
headers: new Headers(headers),
},
timeoutMs,
policy: networkPolicy.apiPolicy,
dispatcherPolicy,
auditContext: "comfy-history",
errorPrefix: "Comfy history lookup failed",
});
})()
: await waitForLocalHistory({
baseUrl: normalizedBaseUrl,
promptId,
headers: new Headers(headers),
timeoutMs,
pollIntervalMs,
policy: networkPolicy.apiPolicy,
dispatcherPolicy,
});
const historyEntry = extractHistoryEntry(history, promptId);
if (!historyEntry) {
throw new Error(`Comfy history response missing outputs for prompt ${promptId}`);
}
const outputFiles = collectOutputFiles({
history: historyEntry,
outputNodeId,
outputKinds: params.outputKinds,
});
if (outputFiles.length === 0) {
throw new Error(`Comfy workflow ${promptId} completed without ${params.capability} outputs`);
}
const assets: ComfyGeneratedAsset[] = [];
let assetIndex = 0;
for (const output of outputFiles) {
const downloaded = await downloadOutputFile({
baseUrl: normalizedBaseUrl,
headers: new Headers(headers),
timeoutMs,
policy: networkPolicy.apiPolicy,
dispatcherPolicy,
file: output.file,
mode,
capability: params.capability,
});
assetIndex += 1;
const originalName =
normalizeOptionalString(output.file.filename) || normalizeOptionalString(output.file.name);
assets.push({
buffer: downloaded.buffer,
mimeType: downloaded.mimeType,
fileName:
originalName ||
`${params.capability}-${assetIndex}.${inferFileExtension({ mimeType: downloaded.mimeType })}`,
nodeId: output.nodeId,
});
}
return {
assets,
model: providerModel,
promptId,
outputNodeIds: Array.from(new Set(outputFiles.map((entry) => entry.nodeId))),
};
}