Skip to content

Commit d8708a2

Browse files
author
Eero
committed
feat(media): add ComfyUI local image generation provider
This PR adds ComfyUI as a media provider for local image generation. Following maintainer guidance, this keeps ComfyUI in the media-provider flow and does NOT touch the BYOK/SettingsDialog logic (deferred to follow-up). Changes: - Daemon: renderComfyUIImage() function with OpenAI-compatible API contract - Daemon + web: MEDIA_PROVIDERS entry with integrated=true, credentialsRequired=false - Daemon + web: comfyui-sdxl model in IMAGE_MODELS (default: true) - Web: NewProjectPanel whitelist for 'comfyui' - Web: provider-readiness.ts already supports credentialsRequired=false (no changes needed) ComfyUI is now available as a local, keyless image generation provider accessible via Open Design's standard media generation flow.
1 parent fe1231e commit d8708a2

4 files changed

Lines changed: 58 additions & 4 deletions

File tree

apps/daemon/src/media/index.ts

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -659,6 +659,11 @@ export async function generateMedia(args: {
659659
bytes = result.bytes;
660660
providerNote = result.providerNote;
661661
suggestedExt = result.suggestedExt;
662+
} else if (def.provider === 'comfyui' && surface === 'image') {
663+
const result = await renderComfyUIImage(ctx, credentials);
664+
bytes = result.bytes;
665+
providerNote = result.providerNote;
666+
suggestedExt = result.suggestedExt;
662667
} else if (def.provider === 'openrouter' && surface === 'image') {
663668
const result = await renderOpenRouterImage(ctx, credentials);
664669
bytes = result.bytes;
@@ -1278,6 +1283,50 @@ async function renderCustomOpenAIImage(ctx: MediaContext, credentials: ProviderC
12781283
};
12791284
}
12801285

1286+
async function renderComfyUIImage(ctx: MediaContext, credentials: ProviderConfig): Promise<RenderResult> {
1287+
const baseUrl = (credentials.baseUrl || '').trim();
1288+
if (!baseUrl) {
1289+
throw new Error('ComfyUI base URL required — configure in Settings (default: http://127.0.0.1:8188)');
1290+
}
1291+
const wireModel = (
1292+
credentials.model
1293+
|| (ctx.wireModel !== 'comfyui-sdxl' ? ctx.wireModel : '')
1294+
).trim();
1295+
const customModel = wireModel || 'sd_xl_base_1.0.safetensors';
1296+
1297+
const headers: Record<string, string> = {
1298+
'content-type': 'application/json',
1299+
};
1300+
if (credentials.apiKey) {
1301+
headers.authorization = `Bearer ${credentials.apiKey}`;
1302+
}
1303+
1304+
const body: Record<string, unknown> = {
1305+
prompt: ctx.prompt || 'A high-quality image.',
1306+
model: customModel,
1307+
width: openaiSizeFor('gpt-image-1', ctx.aspect).split('x')[0],
1308+
height: openaiSizeFor('gpt-image-1', ctx.aspect).split('x')[1],
1309+
n: 1,
1310+
};
1311+
1312+
const resp = await fetchImageGenerationWithResponseRetry(
1313+
() => fetch(`${baseUrl}/v1/images/generations`, withMediaRequestInit(ctx, {
1314+
method: 'POST',
1315+
headers,
1316+
body: JSON.stringify(body),
1317+
})),
1318+
'comfyui',
1319+
);
1320+
1321+
const data = await parseOpenAICompatibleJson(resp, 'comfyui');
1322+
const bytes = await bytesFromOpenAICompatibleData(data, 'comfyui', ctx.requestInit);
1323+
return {
1324+
bytes,
1325+
providerNote: `comfyui/${customModel} · ${body.width}x${body.height} · ${bytes.length} bytes`,
1326+
suggestedExt: sniffImageExt(bytes),
1327+
};
1328+
}
1329+
12811330
function customImageOverridesOpenAIModel(
12821331
ctx: MediaContext,
12831332
credentials: ProviderConfig | null,

apps/daemon/src/media/models.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ export const MEDIA_PROVIDERS: MediaProvider[] = [
4040
{ id: 'imagerouter', label: 'ImageRouter', hint: 'OpenAI-compatible image + video routing', integrated: true, defaultBaseUrl: 'https://api.imagerouter.io/v1/openai', docsUrl: 'https://docs.imagerouter.io/api-reference/image-generation/', supportsCustomModel: true, customModelPlaceholder: 'openai/gpt-image-2 or xAI/grok-imagine-video' },
4141
{ id: 'openrouter', label: 'OpenRouter', hint: 'Unified gateway for image + video models', integrated: true, credentialsRequired: true, settingsVisible: true, defaultBaseUrl: 'https://openrouter.ai/api/v1', docsUrl: 'https://openrouter.ai/settings/keys' },
4242
{ id: 'custom-image', label: 'Custom Image API', hint: 'OpenAI-compatible images/generations + images/edits (local or cloud)', integrated: true, docsUrl: 'https://platform.openai.com/docs/api-reference/images', supportsCustomModel: true, customModelPlaceholder: 'my-image-model' },
43-
{ id: 'comfyui', label: 'ComfyUI', hint: 'Local JSON workflow server (planned adapter)', integrated: false, defaultBaseUrl: 'http://127.0.0.1:8188', docsUrl: 'https://docs.comfy.org/development/core-concepts/workflow' },
43+
{ id: 'comfyui', label: 'ComfyUI', hint: 'Local JSON workflow server', integrated: true, defaultBaseUrl: 'http://127.0.0.1:8188', docsUrl: 'https://docs.comfy.org/development/core-concepts/workflow', credentialsRequired: false, supportsCustomModel: true, customModelPlaceholder: 'sd_xl_base_1.0.safetensors' },
4444
{ id: 'bfl', label: 'Black Forest Labs', hint: 'FLUX 1.1 Pro / FLUX Pro / Dev', integrated: false, defaultBaseUrl: 'https://api.bfl.ai' },
4545
{ id: 'fal', label: 'Fal.ai', hint: 'FLUX / Sora / Veo / Wan / Ideogram / Recraft and any fal-ai/* model', integrated: true, defaultBaseUrl: 'https://fal.run', supportsCustomModel: true },
4646
{ id: 'leonardo', label: 'Leonardo.ai', hint: 'Phoenix / Kino XL / FLUX', integrated: true, credentialsRequired: true, settingsVisible: true, defaultBaseUrl: 'https://cloud.leonardo.ai/api/rest/v1' },
@@ -149,6 +149,7 @@ export const IMAGE_MODELS: MediaModel[] = [
149149
{ id: 'leonardo-anime-pastel', label: 'Anime Pastel Dream', hint: 'Leonardo · anime', provider: 'leonardo', caps: ['t2i'] },
150150

151151
{ id: 'midjourney-v7', label: 'midjourney-v7', hint: 'Midjourney · via proxy', provider: 'midjourney', caps: ['t2i'] },
152+
{ id: 'comfyui-sdxl', label: 'comfyui-sdxl', hint: 'Local ComfyUI · SDXL', provider: 'comfyui', caps: ['t2i'], default: true },
152153
];
153154

154155
export const VIDEO_MODELS: MediaModel[] = [

apps/web/src/components/NewProjectPanel.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2759,7 +2759,7 @@ function MediaProjectOptions(props:
27592759

27602760
export function supportedModels(surface: 'image' | 'video' | 'audio', models: MediaModel[]): MediaModel[] {
27612761
const supportedProviders: Record<'image' | 'video' | 'audio', Set<string>> = {
2762-
image: new Set(['openai', 'codex', 'volcengine', 'grok', 'nanobanana', 'openrouter', 'imagerouter', 'leonardo', 'custom-image', 'aihubmix', 'minimax']),
2762+
image: new Set(['openai', 'codex', 'volcengine', 'grok', 'nanobanana', 'openrouter', 'imagerouter', 'leonardo', 'custom-image', 'comfyui', 'aihubmix', 'minimax']),
27632763
video: new Set(['volcengine', 'hyperframes', 'grok', 'openrouter', 'imagerouter', 'aihubmix']),
27642764
audio: new Set(['minimax', 'fishaudio', 'senseaudio', 'elevenlabs', 'openai', 'volcengine', 'aihubmix']),
27652765
};

apps/web/src/media/models.ts

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,10 +166,13 @@ export const MEDIA_PROVIDERS: MediaProvider[] = [
166166
{
167167
id: 'comfyui',
168168
label: 'ComfyUI',
169-
hint: 'Local JSON workflow server (planned adapter)',
170-
integrated: false,
169+
hint: 'Local JSON workflow server',
170+
integrated: true,
171171
defaultBaseUrl: 'http://127.0.0.1:8188',
172172
docsUrl: 'https://docs.comfy.org/development/core-concepts/workflow',
173+
credentialsRequired: false,
174+
supportsCustomModel: true,
175+
customModelPlaceholder: 'sd_xl_base_1.0.safetensors',
173176
},
174177
{
175178
id: 'bfl',
@@ -534,6 +537,7 @@ export const IMAGE_MODELS: MediaModel[] = [
534537

535538
// Midjourney via community proxies.
536539
{ id: 'midjourney-v7', label: 'midjourney-v7', hint: 'Midjourney · via proxy', provider: 'midjourney', caps: ['t2i'] },
540+
{ id: 'comfyui-sdxl', label: 'comfyui-sdxl', hint: 'Local ComfyUI · SDXL', provider: 'comfyui', caps: ['t2i'], default: true },
537541
];
538542

539543
/**

0 commit comments

Comments
 (0)