From 41638bf5087679e79e792946c534e9fe753a76e9 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sat, 10 Oct 2026 21:53:41 -0600 Subject: [PATCH 01/10] feat(generation): restore hosted-provider options, canvas modes, and record editing Add OpenAI/Gemini/Seedream options and provider named sizes to the Generate panel, compile them into the provider node and metadata, and recall them from images. Let external models generate from the Canvas in the modes they advertise (txt2img, img2img, inpaint) and reject outpaint with a named reason. Add an External Provider editor to the model detail for provider ids, modes, support flags, limits, and defaults; extend the mock backend with external fixtures and model PATCH. --- .../Users Guide/03.Models/01.introduction.mdx | 8 + .../03.Models/External Models/gemini.mdx | 9 + .../03.Models/External Models/openai.mdx | 11 +- .../03.Models/External Models/seedream.mdx | 8 +- .../frontend/webv2/public/locales/en.json | 71 ++++- .../webv2/scripts/mock-backend-fixtures.mjs | 118 +++++++- .../frontend/webv2/scripts/mock-backend.mjs | 13 +- .../core/baseGenerationPolicies.test.ts | 21 ++ .../generation/core/baseGenerationPolicies.ts | 45 ++- .../core/canvas/compileCanvasGraph.test.ts | 97 +++++- .../core/canvas/compileCanvasGraph.ts | 72 +++-- .../generation/core/externalProvider.test.ts | 91 ++++++ .../generation/core/externalProvider.ts | 188 ++++++++++++ .../features/generation/core/graph.test.ts | 114 +++++++ .../src/features/generation/core/graph.ts | 40 ++- .../features/generation/core/settings.test.ts | 50 ++++ .../src/features/generation/core/settings.ts | 22 ++ .../src/features/generation/core/types.ts | 55 +++- .../webv2/src/features/generation/settings.ts | 10 + .../generation/ui/GenerateDimensionFields.tsx | 13 + ...ateProviderOptionsSection.browser.test.tsx | 182 ++++++++++++ .../ui/GenerateProviderOptionsSection.tsx | 260 ++++++++++++++++ .../generation/ui/GenerateSettingsForm.tsx | 3 + .../ExternalSizeSelect.browser.test.tsx | 148 +++++++++ .../ui/shared/ExternalSizeSelect.tsx | 101 +++++++ .../webv2/src/features/models/core/schemas.ts | 46 +++ .../webv2/src/features/models/core/types.ts | 43 ++- .../ExternalModelSection.browser.test.tsx | 187 ++++++++++++ .../models/ui/detail/ExternalModelSection.tsx | 280 ++++++++++++++++++ .../features/models/ui/detail/ModelDetail.tsx | 43 +++ .../ui/detail/externalModelFields.test.ts | 120 ++++++++ .../models/ui/detail/externalModelFields.ts | 83 ++++++ .../models/ui/modelDraftsStore.test.ts | 15 + .../features/models/ui/modelDraftsStore.ts | 12 +- .../importGalleryImages.test.ts | 8 + .../image-actions/imageRecall.test.ts | 148 ++++++++- .../workbench/image-actions/imageRecall.ts | 111 ++++++- .../src/workbench/workbenchState.test.ts | 8 + 38 files changed, 2787 insertions(+), 67 deletions(-) create mode 100644 invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts create mode 100644 invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts create mode 100644 invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx create mode 100644 invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.tsx create mode 100644 invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx create mode 100644 invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx create mode 100644 invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx create mode 100644 invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx create mode 100644 invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.test.ts create mode 100644 invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.ts diff --git a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx index 71708b62470..a61e2869594 100644 --- a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx @@ -98,6 +98,14 @@ information on how to generate these keys. Once a valid API key has been provided, the list of models hosted by that service will appear as selections in the Image Generation panel. +Opening an external model in the Model Manager shows an **External +Provider** section where you can change the provider and provider model +ID it routes to, the modes (text to image, image to image, inpaint) and +inputs (seed, reference images, negative prompt) it advertises, its +request limits, and the size and image count the provider is asked for +by default. The Generate panel and the Canvas only offer what the +record advertises, so keep it in step with the provider's documentation. + :::note[Gated Downloads] The API Keys section is also used to provide keys for services that require authentication to download certain models, such as those diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx index 53067376488..1573a60a242 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx @@ -39,6 +39,15 @@ Reference-image input is used to condition generation but counts as txt2img — All Gemini models are single-image-per-request — batch size is fixed at 1. To generate multiple variations, queue multiple invocations. +## Provider-Specific Options + +Selecting a Gemini model adds a **Provider options** section to the Generate panel: + +- **Temperature** — Override the sampling temperature (0–2; the provider default is 1). +- **Thinking level** — `minimal` or `high`, or the provider default. + +Gemini 3 models list their resolution presets under **Provider size** in the Size section. Picking one sets the width and height to the preset and sends its named size (`1K`, `2K`, `4K`) to the API. The named size and the sampling options are recorded in the image metadata and restored by **Recall All** and **Remix**. + ## Tips diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx index 2ee4628ebd3..adbb53c771c 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx @@ -44,12 +44,19 @@ Inpainting (mask-based editing) is not currently supported for any OpenAI model ## Provider-Specific Options -For **GPT Image** models, Invoke surfaces two provider-specific options in the parameters panel: +Selecting an OpenAI model adds a **Provider options** section to the Generate panel: - **Quality** — `low`, `medium`, `high`, or `auto`. Higher quality costs more and takes longer. - **Background** — `auto`, `transparent`, or `opaque`. Use `transparent` for PNG output with an alpha channel. +- **Input fidelity** — `low` or `high`, offered only for models that take image inputs (img2img or reference images). Leave it on the provider default to let OpenAI choose. -DALL·E 2 and DALL·E 3 do not expose these options. +These are recorded in the image metadata, so **Recall All** and **Remix** restore them when the image's model is selected. + +The **Size** section lists the provider's fixed sizes per aspect ratio (**Provider size**). Any other size is mapped to the closest one the provider accepts. + +## Canvas + +GPT Image models that advertise `img2img` can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. Inpainting and outpainting are not available for OpenAI models. ## How Requests Are Routed diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx index de25273d289..efd57170120 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx @@ -48,11 +48,17 @@ All Seedream models share the same aspect ratio set: `1:1`, `2:3`, `3:2`, `3:4`, ## Provider-Specific Options -Seedream exposes two provider-specific toggles in the parameters panel: +Selecting a Seedream model adds a **Provider options** section to the Generate panel with two toggles: - **Watermark** — When enabled, BytePlus adds a small watermark to the output. Off by default. - **Optimize Prompt** — When enabled, BytePlus rewrites your prompt server-side for better generation quality. Useful for short prompts; disable if you want the exact wording preserved. +Both are recorded in the image metadata and restored by **Recall All** and **Remix**. The Size section lists the provider's 2K sizes per aspect ratio under **Provider size**. + +## Canvas + +Seedream models advertise `img2img`, so they can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. Inpainting and outpainting are not available. + **Seed** and **guidance scale** are not accepted by the 4.x / 5.x family. ## Reference Images diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 728a9fad1f5..45c2750e3a5 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -1335,7 +1335,36 @@ "emptyCacheConfirmBody": "Unloads all models from RAM and VRAM. The next generation reloads them from disk.", "emptyCacheConfirmTitle": "Empty model cache?", "providerId": "Provider", + "providerIdHelp": "The provider whose API key and endpoint handle this model (e.g. openai, gemini, seedream).", "providerModelId": "Provider Model ID", + "providerModelIdHelp": "The model name the provider expects in requests.", + "externalProvider": "External Provider", + "externalProviderHelp": "Where requests for this model go, what the provider accepts, and the size and count it is asked for by default.", + "saveExternalSettings": "Save Provider Settings", + "externalSettingsSaved": "Provider settings saved", + "failedToSaveExternalSettings": "Failed to save provider settings.", + "invalidExternalSettings": "Invalid provider settings.", + "externalModes": { + "txt2img": "Text to image", + "img2img": "Image to image", + "inpaint": "Inpaint" + }, + "externalFields": { + "supportedModes": "Supported modes", + "supportedModesHelp": "Image to image and inpaint let the canvas send its composite to the provider.", + "supportsSeed": "Accepts a seed", + "supportsReferenceImages": "Accepts reference images", + "supportsNegativePrompt": "Accepts a negative prompt", + "maxImagesPerRequest": "Images per request", + "maxReferenceImages": "Max reference images", + "maxImageWidth": "Max width", + "maxImageHeight": "Max height", + "defaultWidth": "Default width", + "defaultHeight": "Default height", + "defaultNumImages": "Default image count", + "unlimited": "No limit", + "providerDefault": "Provider default" + }, "queueAttentionCount_one": "{{count}} needs attention", "queueAttentionCount_other": "{{count}} need attention", "queueActionFailed": "Queue action failed", @@ -2655,7 +2684,47 @@ "pidSteps": "PiD steps", "pidHelp": "Replaces the VAE decode with a 4x super-resolution decode.", "pidFitHelp": "Generates at the requested size, then decodes 4x and scales back down.", - "pidNativeHelp": "The requested size is the 4x target; generation runs at a quarter of it." + "pidNativeHelp": "The requested size is the 4x target; generation runs at a quarter of it.", + "providerSize": "Provider size", + "providerSizeHelp": "Sizes this provider renders natively. Other sizes are mapped to the closest one.", + "providerSizeCustom": "Custom", + "providerOptions": { + "title": "Provider options", + "providerDefault": "Provider default", + "openaiQuality": "Quality", + "openaiQualityHelp": "Higher quality costs more and takes longer.", + "openaiQualityOptions": { + "auto": "Auto", + "high": "High", + "medium": "Medium", + "low": "Low" + }, + "openaiBackground": "Background", + "openaiBackgroundHelp": "Transparent keeps an alpha channel in the PNG output.", + "openaiBackgroundOptions": { + "auto": "Auto", + "transparent": "Transparent", + "opaque": "Opaque" + }, + "openaiInputFidelity": "Input fidelity", + "openaiInputFidelityHelp": "How closely edits and reference images are preserved.", + "openaiInputFidelityOptions": { + "low": "Low", + "high": "High" + }, + "geminiTemperature": "Temperature", + "geminiTemperatureHelp": "Sampling temperature from 0 to 2; the provider default is 1.", + "geminiThinkingLevel": "Thinking level", + "geminiThinkingLevelHelp": "How much the model reasons about the prompt before rendering.", + "geminiThinkingLevelOptions": { + "minimal": "Minimal", + "high": "High" + }, + "seedreamWatermark": "Watermark", + "seedreamWatermarkHelp": "Adds the provider's small watermark to the output.", + "seedreamOptimizePrompt": "Optimize prompt", + "seedreamOptimizePromptHelp": "Lets the provider rewrite the prompt before generating. Turn off to keep the exact wording." + } }, "upscale": { "concepts": "Concepts", diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs index b29a99707e6..800e354d78d 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs @@ -396,7 +396,123 @@ const createQueueItems = (count) => const MODEL_BASES = ['sd-1', 'sdxl', 'flux', 'flux2', 'qwen-image']; const MODEL_TYPES = ['main', 'lora', 'controlnet', 'vae', 'embedding']; -const createModels = (count) => +/** + * Hosted-provider records with the capability shapes the Generate panel, Canvas, and Model Manager read: a named + * resolution table (Gemini), fixed sizes per ratio with an edit mode (OpenAI), and provider toggles (Seedream). + */ +const EXTERNAL_MODEL_FIXTURES = [ + { + base: 'external', + capabilities: { + allowed_aspect_ratios: ['1:1', '16:9', '9:16'], + max_image_size: { height: 4096, width: 4096 }, + max_images_per_request: 1, + max_reference_images: 14, + modes: ['txt2img'], + resolution_presets: ['1K', '2K'].flatMap((imageSize) => + [ + ['1:1', 1, 1], + ['16:9', 16, 9], + ['9:16', 9, 16], + ].map(([ratio, w, h]) => { + const base = imageSize === '1K' ? 1024 : 2048; + const width = w >= h ? base : Math.round((base * w) / h); + const height = w >= h ? Math.round((base * h) / w) : base; + return { + aspect_ratio: ratio, + height, + image_size: imageSize, + label: `${ratio} (${imageSize}) — ${width}×${height}`, + width, + }; + }) + ), + supports_reference_images: true, + supports_seed: true, + }, + cover_image: null, + default_settings: { height: 1024, num_images: 1, width: 1024 }, + description: 'Fixture hosted model with named resolution presets and sampling options.', + file_size: 0, + format: 'external_api', + hash: 'external:gemini:fixture-gemini-image', + key: 'fixture-external-gemini', + name: 'Fixture Gemini Image', + path: 'external://gemini/fixture-gemini-image', + provider_id: 'gemini', + provider_model_id: 'fixture-gemini-image', + source: 'external://gemini/fixture-gemini-image', + source_type: 'external', + source_url: null, + trigger_phrases: null, + type: 'external_image_generator', + }, + { + base: 'external', + capabilities: { + allowed_aspect_ratios: ['1:1', '3:2', '2:3'], + aspect_ratio_sizes: { + '1:1': { height: 1024, width: 1024 }, + '3:2': { height: 1024, width: 1536 }, + '2:3': { height: 1536, width: 1024 }, + }, + max_images_per_request: 10, + modes: ['txt2img', 'img2img', 'inpaint'], + supports_reference_images: true, + }, + cover_image: null, + default_settings: { height: 1024, num_images: 1, width: 1024 }, + description: 'Fixture hosted model with fixed ratio sizes and canvas edit modes.', + file_size: 0, + format: 'external_api', + hash: 'external:openai:fixture-gpt-image', + key: 'fixture-external-openai', + name: 'Fixture GPT Image', + path: 'external://openai/fixture-gpt-image', + provider_id: 'openai', + provider_model_id: 'fixture-gpt-image', + source: 'external://openai/fixture-gpt-image', + source_type: 'external', + source_url: null, + trigger_phrases: null, + type: 'external_image_generator', + }, + { + base: 'external', + capabilities: { + allowed_aspect_ratios: ['1:1', '16:9'], + aspect_ratio_sizes: { '1:1': { height: 2048, width: 2048 }, '16:9': { height: 1600, width: 2848 } }, + max_images_per_request: 15, + max_reference_images: 14, + modes: ['txt2img', 'img2img'], + supports_reference_images: true, + }, + cover_image: null, + default_settings: { height: 2048, num_images: 1, width: 2048 }, + description: 'Fixture hosted model with provider toggles.', + file_size: 0, + format: 'external_api', + hash: 'external:seedream:fixture-seedream', + key: 'fixture-external-seedream', + name: 'Fixture Seedream', + path: 'external://seedream/fixture-seedream', + provider_id: 'seedream', + provider_model_id: 'fixture-seedream', + source: 'external://seedream/fixture-seedream', + source_type: 'external', + source_url: null, + trigger_phrases: null, + type: 'external_image_generator', + }, +]; + +/** The last slots of the model library hold the hosted-provider records so the profile counts stay as declared. */ +const createModels = (count) => [ + ...createLocalModels(Math.max(0, count - EXTERNAL_MODEL_FIXTURES.length)), + ...EXTERNAL_MODEL_FIXTURES.slice(0, count).map((model) => structuredClone(model)), +]; + +const createLocalModels = (count) => range(count, (index) => { const id = ordinal(index); const type = MODEL_TYPES[index % MODEL_TYPES.length]; diff --git a/invokeai/frontend/webv2/scripts/mock-backend.mjs b/invokeai/frontend/webv2/scripts/mock-backend.mjs index 74836005032..89ca19c97c2 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend.mjs @@ -1242,7 +1242,18 @@ export const startMockBackend = async (port, { profile = 'empty' } = {}) => { const key = decodeURIComponent(modelMatch[1]); const model = state.models.get(key); - return model ? json(200, model) : json(404, { detail: 'Model not found' }); + if (!model) { + return json(404, { detail: 'Model not found' }); + } + if (method === 'PATCH') { + // Mirrors ModelRecordService.update_model: each submitted field replaces the stored one wholesale. + const changes = await readJsonBody(request); + const updated = { ...model, ...changes }; + state.models.set(key, updated); + return json(200, updated); + } + + return json(200, model); } if (method === 'GET' && (path === '/api/v2/models' || path === '/api/v2/models/')) { diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts index 68d069e8e72..0cf3dbb37a7 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts @@ -575,6 +575,27 @@ describe('component policies', () => { ); }); + it('validates the external mode being compiled against the modes the model advertises', () => { + const model: GenerateModelConfig = { + ...externalModel, + capabilities: { modes: ['txt2img', 'inpaint'] }, + }; + + expect(getGenerationValidationReasons(model, createSettings(model))).toEqual([]); + expect(getGenerationValidationReasons(model, createSettings(model), 'inpaint')).toEqual([]); + expect(getGenerationValidationReasons(model, createSettings(model), 'img2img')).toContain( + 'External Model does not support image-to-image generation.' + ); + // Modes default to txt2img only when the record omits them. + expect( + getGenerationValidationReasons({ ...externalModel, capabilities: null }, createSettings(externalModel), 'inpaint') + ).toContain('External Model does not support inpainting.'); + }); + + it('accepts provider sizes off the local model grids for external generators', () => { + expect(getGenerationValidationReasons(externalModel, createSettings(externalModel, { width: 1365 }))).toEqual([]); + }); + it('renders no component slots for bases without extra requirements', () => { expect(getComponentSectionPolicy(createModel('cogview4'), createSettings(createModel('cogview4'))).slots).toEqual( [] diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts index e3e849062f2..5dcf4cd2f33 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts @@ -18,6 +18,7 @@ import { import { SEED_MAX } from '@platform/core/seed'; import type { + ExternalGenerationMode, GenerateModelConfig, GenerateReferenceImage, GenerateReferenceImageAsset, @@ -192,11 +193,12 @@ const getBaseGenerationConfig = ( return getArchitectureGenerationConfig(model.base, model.variant) ?? FALLBACK_GENERATION_CONFIG; }; -/** Use variant-aware grids; return null for undescribed local architectures. External providers use a generic grid. */ +/** Pixel-exact: hosted providers bucket requests to their own size tables, which are not grid-aligned. */ +const EXTERNAL_DIMENSION_GRID = 1; + +/** Use variant-aware grids; return null for undescribed local architectures. External providers have no grid. */ export const getDimensionGrid = (base: string, variant?: unknown): number | null => - base === 'external' - ? FALLBACK_GENERATION_CONFIG.dimensions.grid - : (getArchitectureFeatures(base, variant)?.dimension_grid ?? null); + base === 'external' ? EXTERNAL_DIMENSION_GRID : (getArchitectureFeatures(base, variant)?.dimension_grid ?? null); /** A served architecture row does not imply frontend graph support. */ export const isArchitectureDescribed = (model: Pick): boolean => @@ -219,7 +221,7 @@ export const getGenerationDimensions = ( const { grid, optimalSide } = getPidDimensionOverrides( pidMode, model?.base, - config.dimensions.grid, + model?.type === 'external_image_generator' ? EXTERNAL_DIMENSION_GRID : config.dimensions.grid, config.dimensions.optimalSide ); @@ -477,6 +479,14 @@ export const getDefaultGenerateSettings = (model?: GenerateModelConfig): Generat pidDecoderModel: null, gemma2EncoderModel: null, pidSteps: DEFAULT_PID_STEPS, + externalImageSize: null, + openaiQuality: 'auto', + openaiBackground: 'auto', + openaiInputFidelity: null, + geminiTemperature: null, + geminiThinkingLevel: null, + seedreamWatermark: false, + seedreamOptimizePrompt: false, referenceImages: [], scheduler: defaults.scheduler, seamlessXAxis: false, @@ -1669,7 +1679,24 @@ const getModelFamilyValidationReasons = (model: MainModelConfig, settings: Gener return reasons; }; -export const getGenerationValidationReasons = (model: GenerateModelConfig, settings: GenerateSettings): string[] => { +const EXTERNAL_MODE_LABELS: Record = { + img2img: 'image-to-image generation', + inpaint: 'inpainting', + txt2img: 'text-to-image generation', +}; + +/** Why an external model cannot run `mode`, or null when it advertises it. */ +export const getExternalModeReason = (model: GenerateModelConfig, mode: ExternalGenerationMode): string | null => + model.type === 'external_image_generator' && !(model.capabilities?.modes ?? ['txt2img']).includes(mode) + ? `${model.name} does not support ${EXTERNAL_MODE_LABELS[mode]}.` + : null; + +/** `mode` is the external generation mode being compiled; local models are validated the same way for every mode. */ +export const getGenerationValidationReasons = ( + model: GenerateModelConfig, + settings: GenerateSettings, + mode: ExternalGenerationMode = 'txt2img' +): string[] => { // All compilers share this validator; check capabilities before model-specific policy. if (!hasArchitectureCapabilities()) { // Keep core independent of the load-state store; failures direct callers to retry. @@ -1694,8 +1721,10 @@ export const getGenerationValidationReasons = (model: GenerateModelConfig, setti ]; if (model.type === 'external_image_generator') { - if (model.capabilities?.modes && !model.capabilities.modes.includes('txt2img')) { - reasons.push(`${model.name} does not support text-to-image generation.`); + const modeReason = getExternalModeReason(model, mode); + + if (modeReason) { + reasons.push(modeReason); } if (!getExternalProviderNodeType(model.provider_id)) { diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index f0c5d3d695a..80610f788cd 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -406,15 +406,96 @@ describe('compileCanvasGraph', () => { ); }); - describe('unsupported models', () => { - it('rejects external image generators for txt2img', () => { - expect(() => compile(externalModel, 'txt2img')).toThrow('does not support canvas generation'); + describe('external image generators', () => { + const editModel: GenerateModelConfig = { + ...externalModel, + capabilities: { modes: ['txt2img', 'img2img', 'inpaint'], supports_seed: true }, + }; + const smallBbox: Rect = { height: 512, width: 640, x: 10, y: 20 }; + + it('generates txt2img from an empty bbox on one provider node sized to the bbox, with no encode or resize', () => { + const { backendGraph, mode } = compile(externalModel, 'txt2img', { + bbox: smallBbox, + compositeImageName: null, + scaling: { height: null, method: 'auto', width: null }, + }); + const output = backendGraph.nodes.canvas_output; + + expect(mode).toBe('txt2img'); + expect(output).toMatchObject({ + height: 512, + is_intermediate: true, + mode: 'txt2img', + type: 'openai_image_generation', + width: 640, + }); + expect(output).not.toHaveProperty('init_image'); + expect(output).not.toHaveProperty('mask_image'); + expect(getEdge(backendGraph, 'canvas_output', 'prompt')?.source.node_id).toBe('positive_prompt'); + expect(getEdge(backendGraph, 'canvas_output', 'seed')?.source.node_id).toBe('seed'); + expect(backendGraph.nodes.canvas_i2l).toBeUndefined(); + expect(backendGraph.nodes.canvas_processing_output).toBeUndefined(); + expect(Object.values(backendGraph.nodes).map((node) => node.type)).not.toContain('img_resize'); + // The provider returns one image at the bbox size, so metadata records the bbox too. + expect(getNodeByType(backendGraph, 'core_metadata')).toMatchObject({ height: 512, width: 640 }); }); - it('rejects external image generators for img2img', () => { - expect(() => compile(externalModel, 'img2img')).toThrow('does not support canvas generation'); + it('sends the composited bbox as the init image for img2img when the model advertises it', () => { + const { backendGraph } = compile(editModel, 'img2img', { compositeImageName: 'bbox.png' }); + + expect(backendGraph.nodes.canvas_output).toMatchObject({ + init_image: { image_name: 'bbox.png' }, + mode: 'img2img', + }); + expect(backendGraph.nodes.canvas_output).not.toHaveProperty('mask_image'); + // No local denoise path is grafted: the provider edits the whole image. + expect(backendGraph.nodes.denoise_latents).toBeUndefined(); + expect(backendGraph.nodes.create_gradient_mask).toBeUndefined(); + }); + + it('sends the composite and the grayscale mask for inpaint', () => { + const { backendGraph } = compile(editModel, 'inpaint', { + compositeImageName: 'bbox.png', + maskImageName: 'mask.png', + }); + + expect(backendGraph.nodes.canvas_output).toMatchObject({ + init_image: { image_name: 'bbox.png' }, + mask_image: { image_name: 'mask.png' }, + mode: 'inpaint', + }); + // The provider composites its own edit, so no blend/apply-mask tail follows it. + expect(backendGraph.nodes.invokeai_img_blend).toBeUndefined(); + expect(backendGraph.nodes.expand_mask).toBeUndefined(); }); + it('still requires the composite and the mask the mode needs', () => { + expect(() => compile(editModel, 'img2img', { compositeImageName: null })).toThrow( + 'Canvas generation requires a composited source image.' + ); + expect(() => compile(editModel, 'inpaint', { maskImageName: null })).toThrow( + 'Canvas inpainting requires an inpaint mask.' + ); + }); + + it('rejects the edit modes a model does not advertise, naming the mode', () => { + expect(() => compile(externalModel, 'img2img')).toThrow( + 'OpenAI Image does not support image-to-image generation.' + ); + expect(() => compile(externalModel, 'inpaint')).toThrow('OpenAI Image does not support inpainting.'); + expect(() => + compile({ ...externalModel, capabilities: { modes: ['img2img'] } }, 'txt2img', { compositeImageName: null }) + ).toThrow('OpenAI Image does not support text-to-image generation.'); + }); + + it('rejects outpaint even for a model that advertises every provider mode', () => { + expect(() => compile(editModel, 'outpaint', { maskImageName: null })).toThrow( + 'OpenAI Image does not support outpainting on the canvas.' + ); + }); + }); + + describe('unsupported models', () => { it('rejects PiD with an actionable message rather than an internal VAE error', () => { // PiD lacks the VAE seam expected by canvas compilation, so reject it explicitly. for (const mode of ['txt2img', 'img2img'] as const) { @@ -844,12 +925,6 @@ describe('compileCanvasGraph — outpaint per base', () => { expect(backendGraph.nodes.create_gradient_mask?.minimum_denoise).toBe(0.3); expect(backendGraph.nodes.expand_mask?.fade_size_px).toBe(24); }); - - it('rejects an external model for every image mode', () => { - expect(() => compile(externalModel, 'outpaint', { maskImageName: null })).toThrow( - 'does not support canvas generation' - ); - }); }); // Exercise control grafting through the real compiler entry point. diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts index 8155895bd1e..3ef293b735e 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts @@ -1,6 +1,6 @@ /** - * Pure canvas compiler over uploaded inputs. Resize through the processing grid and restore the bbox; reject - * external models and unsupported encode modes. + * Pure canvas compiler over uploaded inputs. Resize through the processing grid and restore the bbox; hosted + * providers take the bbox composite as-is in the modes they advertise; reject unsupported encode modes. */ import type { SupportedGenerateBase } from '@features/generation/core/baseGenerationPolicies'; @@ -8,7 +8,8 @@ import type { BackendGraphContract, BackendInvocationContract } from '@features/ import type { GenerateModelConfig, GenerateSettings } from '@features/generation/core/types'; import { getGenerationValidationReasons } from '@features/generation/core/baseGenerationPolicies'; -import { GRAPH_BUILDERS } from '@features/generation/core/graph'; +import { isExternalGenerationMode } from '@features/generation/core/externalProvider'; +import { buildExternalGraph, GRAPH_BUILDERS } from '@features/generation/core/graph'; import { addEdge, addNode, toGraphContract } from '@features/generation/core/graphBuilder'; import { addKrea2ConditioningEnhancers } from '@features/generation/core/krea2Conditioning'; import { getIsPidSupportedBase } from '@features/generation/core/pid'; @@ -62,9 +63,10 @@ const getCanvasValidationReasons = (input: CompileCanvasGraphInput): string[] => const { bbox, compositeImageName, maskImageName, mode, model, strength } = input; const reasons: string[] = []; - if (model.type === 'external_image_generator') { - reasons.push(`${model.name} does not support canvas generation.`); - return reasons; + // Providers take a whole image to edit; a partially covered bbox has nothing to send for the empty area. The + // shared validator reports the other modes the model does not advertise. + if (model.type === 'external_image_generator' && mode === 'outpaint') { + reasons.push(`${model.name} does not support outpainting on the canvas.`); } // Reject canvas PiD because its graph lacks the expected VAE/decode seam. @@ -232,9 +234,6 @@ const addInputResizeNode = ( /** Resolves the encode node type for a supported base (throws otherwise). */ const requireI2lType = (model: GenerateModelConfig): string => { - if (model.type === 'external_image_generator') { - throw new Error(`${model.name} does not support canvas generation.`); - } const i2lType = CANVAS_I2L_NODE_TYPES[model.base as SupportedGenerateBase]; if (!i2lType) { throw new Error(`Canvas generation is not supported for ${model.name}.`); @@ -611,27 +610,66 @@ const graftOutpaint = ( /** Throw the first validation reason. */ export const compileCanvasGraph = (input: CompileCanvasGraphInput): CompiledCanvasGraph => { const { bbox, compositeImageName, destination, mode, model, projectSettings, strength } = input; - const processingSize = resolveCanvasProcessingSize(model, input.settings.pidMode, bbox, input.scaling); + // A provider buckets the request to its own size table and returns one image, so it is asked for the bbox itself. + const processingSize = + model.type === 'external_image_generator' + ? { height: bbox.height, width: bbox.width } + : resolveCanvasProcessingSize(model, input.settings.pidMode, bbox, input.scaling); const settings = withProcessingDimensions(input.settings, processingSize); - const validationReasons = [...getCanvasValidationReasons(input), ...getGenerationValidationReasons(model, settings)]; + const validationReasons = [ + ...getCanvasValidationReasons(input), + ...getGenerationValidationReasons(model, settings, isExternalGenerationMode(mode) ? mode : 'txt2img'), + ]; if (validationReasons.length > 0) { throw new Error(validationReasons[0]); } - const builder = GRAPH_BUILDERS[model.base as SupportedGenerateBase]; - - if (!builder || model.type === 'external_image_generator') { - throw new Error(`${model.name} does not support canvas generation.`); - } - // Canvas outputs are intermediate; gallery outputs are durable. const outputIsIntermediate = destination === 'canvas'; const runtimeProjectSettings = { ...projectSettings, randDevice: input.randDevice ?? (projectSettings.useCpuNoise ? 'cpu' : 'cuda'), }; + + if (model.type === 'external_image_generator') { + if (!isExternalGenerationMode(mode)) { + throw new Error(`${model.name} does not support outpainting on the canvas.`); + } + + // Validation above guarantees the composite for the edit modes and the mask for inpaint. The provider returns + // the edited image whole, so no encode, mask tail, control, or regional grafts apply. + const initImageName = compositeImageName as string; + const backendGraph = buildExternalGraph( + settings, + model, + outputIsIntermediate, + runtimeProjectSettings, + mode === 'txt2img' + ? { mode } + : mode === 'img2img' + ? { initImageName, mode } + : { initImageName, maskImageName: input.maskImageName as string, mode } + ); + setCanvasMetadataDimensions(backendGraph, bbox); + + return { + backendGraph, + graph: toGraphContract(backendGraph, `${model.name} ${mode}`), + mode, + negativePromptNodeId: 'negative_prompt', + positivePromptNodeId: 'positive_prompt', + seedNodeId: 'seed', + }; + } + + const builder = GRAPH_BUILDERS[model.base as SupportedGenerateBase]; + + if (!builder) { + throw new Error(`Canvas generation is not supported for ${model.name}.`); + } + const backendGraph = builder(settings, model, outputIsIntermediate, runtimeProjectSettings); const { compositing } = input; diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts new file mode 100644 index 00000000000..e23290caf76 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; + +import type { ExternalImageGeneratorModelConfig, GenerateModelConfig } from './types'; + +import { + findExternalResolutionPreset, + getExternalAspectRatioSizes, + getExternalProviderOptionKeys, + resolveExternalImageSize, +} from './externalProvider'; + +const external = ( + providerId: string, + capabilities: ExternalImageGeneratorModelConfig['capabilities'] +): GenerateModelConfig => ({ + base: 'external', + capabilities, + format: 'external_api', + key: `${providerId}-model`, + name: providerId, + provider_id: providerId, + type: 'external_image_generator', +}); + +const presets = [ + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }, + { aspect_ratio: '16:9', height: 1152, image_size: '2K', label: '16:9 (2K)', width: 2048 }, +]; + +describe('external provider options', () => { + it('offers each provider only the options its node declares', () => { + expect(getExternalProviderOptionKeys(external('gemini', { modes: ['txt2img'] }))).toEqual([ + 'geminiTemperature', + 'geminiThinkingLevel', + ]); + expect(getExternalProviderOptionKeys(external('seedream', { modes: ['txt2img'] }))).toEqual([ + 'seedreamWatermark', + 'seedreamOptimizePrompt', + ]); + // Alibaba Cloud and Atlas Cloud nodes add nothing beyond the shared fields. + expect(getExternalProviderOptionKeys(external('alibabacloud', { modes: ['txt2img'] }))).toEqual([]); + expect(getExternalProviderOptionKeys({ base: 'sdxl', key: 'sdxl', name: 'SDXL', type: 'main' })).toEqual([]); + expect(getExternalProviderOptionKeys(undefined)).toEqual([]); + }); + + it('offers OpenAI input fidelity only to models that take image inputs', () => { + expect(getExternalProviderOptionKeys(external('openai', { modes: ['txt2img'] }))).toEqual([ + 'openaiQuality', + 'openaiBackground', + ]); + expect(getExternalProviderOptionKeys(external('openai', { modes: ['txt2img', 'img2img'] }))).toContain( + 'openaiInputFidelity' + ); + expect( + getExternalProviderOptionKeys(external('openai', { modes: ['txt2img'], supports_reference_images: true })) + ).toContain('openaiInputFidelity'); + }); + + it('resolves a named size only from the selected model’s own presets', () => { + const gemini = external('gemini', { modes: ['txt2img'], resolution_presets: presets }); + + expect(resolveExternalImageSize({ externalImageSize: '2K' }, gemini)).toBe('2K'); + expect(resolveExternalImageSize({ externalImageSize: '4K' }, gemini)).toBeNull(); + expect( + resolveExternalImageSize({ externalImageSize: '2K' }, external('openai', { modes: ['txt2img'] })) + ).toBeNull(); + }); + + it('matches the preset select to the exact size and named size, so a hand-typed size reads as custom', () => { + const gemini = external('gemini', { modes: ['txt2img'], resolution_presets: presets }); + + expect(findExternalResolutionPreset({ externalImageSize: '2K', height: 1152, width: 2048 }, gemini)?.label).toBe( + '16:9 (2K)' + ); + expect(findExternalResolutionPreset({ externalImageSize: '2K', height: 1152, width: 2000 }, gemini)).toBeNull(); + expect(findExternalResolutionPreset({ externalImageSize: null, height: 1024, width: 1024 }, gemini)).toBeNull(); + }); + + it('lists fixed ratio sizes in the provider’s order', () => { + const openai = external('openai', { + aspect_ratio_sizes: { '1:1': { height: 1024, width: 1024 }, '3:2': { height: 1024, width: 1536 } }, + modes: ['txt2img'], + }); + + expect(getExternalAspectRatioSizes(openai)).toEqual([ + ['1:1', { height: 1024, width: 1024 }], + ['3:2', { height: 1024, width: 1536 }], + ]); + expect(getExternalAspectRatioSizes(external('gemini', { modes: ['txt2img'] }))).toEqual([]); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts new file mode 100644 index 00000000000..84f81fce724 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts @@ -0,0 +1,188 @@ +/** + * Pure policy for hosted image providers: which provider-specific options a selected external model exposes, the + * named sizes and fixed-size ratios it advertises, and the invocation fields those settings compile to. + */ + +import type { + ExternalGenerationMode, + ExternalImageSize, + ExternalModelCapabilities, + ExternalResolutionPreset, + GeminiThinkingLevel, + GenerateModelConfig, + GenerateSettings, + OpenAIImageBackground, + OpenAIImageQuality, + OpenAIInputFidelity, +} from './types'; + +export const OPENAI_IMAGE_QUALITIES: readonly OpenAIImageQuality[] = ['auto', 'high', 'medium', 'low']; +export const OPENAI_IMAGE_BACKGROUNDS: readonly OpenAIImageBackground[] = ['auto', 'transparent', 'opaque']; +export const OPENAI_INPUT_FIDELITIES: readonly OpenAIInputFidelity[] = ['low', 'high']; +export const GEMINI_THINKING_LEVELS: readonly GeminiThinkingLevel[] = ['minimal', 'high']; +/** `GeminiImageGenerationInvocation.temperature` is `ge=0, le=2`; the provider default is 1. */ +export const GEMINI_TEMPERATURE_MIN = 0; +export const GEMINI_TEMPERATURE_MAX = 2; +export const DEFAULT_GEMINI_TEMPERATURE = 1; + +export const isOpenAIImageQuality = (value: unknown): value is OpenAIImageQuality => + OPENAI_IMAGE_QUALITIES.includes(value as OpenAIImageQuality); +export const isOpenAIImageBackground = (value: unknown): value is OpenAIImageBackground => + OPENAI_IMAGE_BACKGROUNDS.includes(value as OpenAIImageBackground); +export const isOpenAIInputFidelity = (value: unknown): value is OpenAIInputFidelity => + OPENAI_INPUT_FIDELITIES.includes(value as OpenAIInputFidelity); +export const isGeminiThinkingLevel = (value: unknown): value is GeminiThinkingLevel => + GEMINI_THINKING_LEVELS.includes(value as GeminiThinkingLevel); +export const isExternalGenerationMode = (value: unknown): value is ExternalGenerationMode => + value === 'txt2img' || value === 'img2img' || value === 'inpaint'; + +export type ExternalProviderOptionKey = keyof Pick< + GenerateSettings, + | 'openaiQuality' + | 'openaiBackground' + | 'openaiInputFidelity' + | 'geminiTemperature' + | 'geminiThinkingLevel' + | 'seedreamWatermark' + | 'seedreamOptimizePrompt' +>; + +const getCapabilities = (model: GenerateModelConfig | undefined): ExternalModelCapabilities | null => + model?.type === 'external_image_generator' ? (model.capabilities ?? null) : null; + +const getProviderId = (model: GenerateModelConfig | undefined): string | null => + model?.type === 'external_image_generator' && typeof model.provider_id === 'string' ? model.provider_id : null; + +/** True when the model accepts image inputs in any form (an edit mode or reference images). */ +const acceptsImageInputs = (capabilities: ExternalModelCapabilities | null): boolean => + capabilities?.supports_reference_images === true || + (capabilities?.modes ?? []).some((mode) => mode === 'img2img' || mode === 'inpaint'); + +/** + * The provider options the selected model's invocation accepts, in display order. Each provider's node declares its + * own fields, so the provider decides; OpenAI's input fidelity is only consulted for image inputs, so a model that + * takes none does not offer it. + */ +export const getExternalProviderOptionKeys = (model: GenerateModelConfig | undefined): ExternalProviderOptionKey[] => { + const capabilities = getCapabilities(model); + + switch (getProviderId(model)) { + case 'openai': + return [ + 'openaiQuality', + 'openaiBackground', + ...(acceptsImageInputs(capabilities) ? (['openaiInputFidelity'] as const) : []), + ]; + case 'gemini': + return ['geminiTemperature', 'geminiThinkingLevel']; + case 'seedream': + return ['seedreamWatermark', 'seedreamOptimizePrompt']; + default: + return []; + } +}; + +export const getExternalResolutionPresets = (model: GenerateModelConfig | undefined): ExternalResolutionPreset[] => { + const presets = getCapabilities(model)?.resolution_presets; + + return Array.isArray(presets) ? presets : []; +}; + +/** Fixed sizes per ratio, as `[ratio, size]` pairs in the provider's order; empty for free-form providers. */ +export const getExternalAspectRatioSizes = ( + model: GenerateModelConfig | undefined +): Array<[ratio: string, size: ExternalImageSize]> => { + const sizes = getCapabilities(model)?.aspect_ratio_sizes; + + return sizes ? Object.entries(sizes) : []; +}; + +/** The named size to send: only one the model lists, so a stale pick from another model never reaches the provider. */ +export const resolveExternalImageSize = ( + settings: Pick, + model: GenerateModelConfig | undefined +): string | null => + settings.externalImageSize !== null && + getExternalResolutionPresets(model).some((preset) => preset.image_size === settings.externalImageSize) + ? settings.externalImageSize + : null; + +/** The preset matching the current size and named size exactly, for the preset select's value. */ +export const findExternalResolutionPreset = ( + settings: Pick, + model: GenerateModelConfig | undefined +): ExternalResolutionPreset | null => + getExternalResolutionPresets(model).find( + (preset) => + preset.image_size === settings.externalImageSize && + preset.width === settings.width && + preset.height === settings.height + ) ?? null; + +/** + * Provider-specific invocation fields for the selected model, keyed as the node declares them. Optional fields are + * omitted when unset so the node applies its own default. + */ +export const getExternalProviderInvocationOptions = ( + settings: GenerateSettings, + model: GenerateModelConfig +): Record => { + const keys = new Set(getExternalProviderOptionKeys(model)); + const options: Record = {}; + + if (keys.has('openaiQuality')) { + options.quality = settings.openaiQuality; + } + if (keys.has('openaiBackground')) { + options.background = settings.openaiBackground; + } + if (keys.has('openaiInputFidelity') && settings.openaiInputFidelity !== null) { + options.input_fidelity = settings.openaiInputFidelity; + } + if (keys.has('geminiTemperature') && settings.geminiTemperature !== null) { + options.temperature = settings.geminiTemperature; + } + if (keys.has('geminiThinkingLevel') && settings.geminiThinkingLevel !== null) { + options.thinking_level = settings.geminiThinkingLevel; + } + if (keys.has('seedreamWatermark')) { + options.watermark = settings.seedreamWatermark; + } + if (keys.has('seedreamOptimizePrompt')) { + options.optimize_prompt = settings.seedreamOptimizePrompt; + } + + return options; +}; + +/** Metadata keys mirror what the provider nodes record themselves, so Generate and Canvas outputs recall alike. */ +export const EXTERNAL_METADATA_KEYS: Record = { + geminiTemperature: 'gemini_temperature', + geminiThinkingLevel: 'gemini_thinking_level', + openaiBackground: 'openai_background', + openaiInputFidelity: 'openai_input_fidelity', + openaiQuality: 'openai_quality', + seedreamOptimizePrompt: 'seedream_optimize_prompt', + seedreamWatermark: 'seedream_watermark', +}; + +export const getExternalProviderMetadata = ( + settings: GenerateSettings, + model: GenerateModelConfig +): Record => { + const metadata: Record = {}; + const imageSize = resolveExternalImageSize(settings, model); + + if (imageSize !== null) { + metadata.image_size = imageSize; + } + for (const key of getExternalProviderOptionKeys(model)) { + const value = settings[key]; + + if (value !== null) { + metadata[EXTERNAL_METADATA_KEYS[key]] = value; + } + } + + return metadata; +}; diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts index 789d05819c4..04e1853ba11 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import type { ComponentModelConfig, + ExternalImageGeneratorModelConfig, GenerateModelConfig, GenerateSettings, LoraModelConfig, @@ -600,6 +601,119 @@ describe('compileGenerateGraph', () => { expect(graph.nodes.canvas_output?.reference_images).toEqual([{ image_name: 'reference.png' }]); }); + describe('external provider options', () => { + const providerSettings: Partial = { + externalImageSize: '2K', + geminiTemperature: 0.4, + geminiThinkingLevel: 'high', + openaiBackground: 'transparent', + openaiInputFidelity: 'high', + openaiQuality: 'low', + seedreamOptimizePrompt: true, + seedreamWatermark: true, + }; + const providerModel = ( + providerId: string, + capabilities: ExternalImageGeneratorModelConfig['capabilities'] + ): GenerateModelConfig => ({ ...externalModel, capabilities, provider_id: providerId }) as GenerateModelConfig; + + it('compiles OpenAI quality, background and input fidelity, recording them in metadata', () => { + const model = providerModel('openai', { modes: ['txt2img', 'img2img'] }); + const graph = compile(model, providerSettings); + + expect(graph.nodes.canvas_output).toMatchObject({ + background: 'transparent', + input_fidelity: 'high', + quality: 'low', + }); + // Fields of other providers' nodes never reach this one. + expect(graph.nodes.canvas_output).not.toHaveProperty('temperature'); + expect(graph.nodes.canvas_output).not.toHaveProperty('watermark'); + expect(getNodeByType(graph, 'core_metadata')).toMatchObject({ + openai_background: 'transparent', + openai_input_fidelity: 'high', + openai_quality: 'low', + }); + }); + + it('omits OpenAI input fidelity for a model with no image inputs, and when unset', () => { + const noInputs = compile(providerModel('openai', { modes: ['txt2img'] }), providerSettings); + + expect(noInputs.nodes.canvas_output).toMatchObject({ background: 'transparent', quality: 'low' }); + expect(noInputs.nodes.canvas_output).not.toHaveProperty('input_fidelity'); + + const unset = compile(providerModel('openai', { modes: ['txt2img', 'img2img'] }), { + ...providerSettings, + openaiInputFidelity: null, + }); + + expect(unset.nodes.canvas_output).not.toHaveProperty('input_fidelity'); + expect(getNodeByType(unset, 'core_metadata')).not.toHaveProperty('openai_input_fidelity'); + }); + + it('compiles Gemini temperature and thinking level only when set', () => { + const model = providerModel('gemini', { modes: ['txt2img'] }); + const graph = compile(model, providerSettings); + + expect(graph.nodes.canvas_output).toMatchObject({ temperature: 0.4, thinking_level: 'high' }); + expect(graph.nodes.canvas_output).not.toHaveProperty('quality'); + expect(getNodeByType(graph, 'core_metadata')).toMatchObject({ + gemini_temperature: 0.4, + gemini_thinking_level: 'high', + }); + + const defaults = compile(model, { ...providerSettings, geminiTemperature: null, geminiThinkingLevel: null }); + + expect(defaults.nodes.canvas_output).not.toHaveProperty('temperature'); + expect(defaults.nodes.canvas_output).not.toHaveProperty('thinking_level'); + }); + + it('compiles Seedream watermark and prompt optimization as explicit booleans', () => { + const model = providerModel('seedream', { modes: ['txt2img'] }); + + expect(compile(model, providerSettings).nodes.canvas_output).toMatchObject({ + optimize_prompt: true, + watermark: true, + }); + expect( + compile(model, { ...providerSettings, seedreamOptimizePrompt: false, seedreamWatermark: false }).nodes + .canvas_output + ).toMatchObject({ optimize_prompt: false, watermark: false }); + expect(getNodeByType(compile(model, providerSettings), 'core_metadata')).toMatchObject({ + seedream_optimize_prompt: true, + seedream_watermark: true, + }); + }); + + it('sends the named size only when the model lists it in a resolution preset', () => { + const presets = [ + { aspect_ratio: '1:1', height: 2048, image_size: '2K', label: '1:1 (2K)', width: 2048 }, + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }, + ]; + const listed = compile(providerModel('gemini', { modes: ['txt2img'], resolution_presets: presets }), { + ...providerSettings, + height: 2048, + width: 2048, + }); + + expect(listed.nodes.canvas_output).toMatchObject({ height: 2048, image_size: '2K', width: 2048 }); + expect(getNodeByType(listed, 'core_metadata')).toMatchObject({ image_size: '2K' }); + + // A size picked for another model, or a provider without named sizes, falls back to the provider default. + const unlisted = compile(providerModel('gemini', { modes: ['txt2img'] }), providerSettings); + + expect(unlisted.nodes.canvas_output?.image_size).toBeNull(); + expect(getNodeByType(unlisted, 'core_metadata')).not.toHaveProperty('image_size'); + }); + + it('accepts provider sizes that are not multiples of a local model grid', () => { + const model = providerModel('gemini', { modes: ['txt2img'] }); + const graph = compile(model, { height: 2048, width: 1365 }); + + expect(graph.nodes.canvas_output).toMatchObject({ height: 2048, width: 1365 }); + }); + }); + it('wires SD IP Adapter reference images into denoise', () => { const graph = compile(sd1Model, { referenceImages: [ diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.ts index 02be3aae118..e11ea63c51f 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.ts @@ -21,6 +21,7 @@ import type { import { coerceSchedulerForGraph, + getExternalModeReason, getExternalProviderNodeType, getFlux2DiffusersComponentSource, getGenerationDimensions, @@ -42,6 +43,11 @@ import { isVaeCompatibleWithGenerateModel, type GenerateComponentFilter, } from './componentCompatibility'; +import { + getExternalProviderInvocationOptions, + getExternalProviderMetadata, + resolveExternalImageSize, +} from './externalProvider'; import { addEdge, addMetadata, @@ -1430,18 +1436,32 @@ const buildWanGraph = ( return graph; }; -const buildExternalGraph = ( +/** Canvas inputs per external mode: the composited bbox for the edit modes and its grayscale mask for inpaint. */ +export type ExternalCanvasInputs = + | { mode: 'txt2img' } + | { mode: 'img2img'; initImageName: string } + | { mode: 'inpaint'; initImageName: string; maskImageName: string }; + +/** + * Hosted providers take the whole request on one node: prompt, seed, size, provider options, and (on the canvas) the + * init image and mask. The provider composites its own edits, so the node's image is the canvas output as-is. + */ +export const buildExternalGraph = ( settings: GenerateSettings, model: GenerateModelConfig, outputIsIntermediate: boolean, - projectSettings: GenerationProjectSettings + projectSettings: GenerationProjectSettings, + canvas: ExternalCanvasInputs | null = null ): BackendGraphContract => { if (model.type !== 'external_image_generator') { throw new Error('Selected model is not an external image generator.'); } - if (model.capabilities?.modes && !model.capabilities.modes.includes('txt2img')) { - throw new Error(`${model.name} does not support text-to-image generation.`); + const mode = canvas?.mode ?? 'txt2img'; + const modeReason = getExternalModeReason(model, mode); + + if (modeReason) { + throw new Error(modeReason); } const providerId = typeof model.provider_id === 'string' ? model.provider_id : ''; @@ -1456,14 +1476,17 @@ const buildExternalGraph = ( const output = addNode(graph, { height: settings.height, id: 'canvas_output', - image_size: null, + image_size: resolveExternalImageSize(settings, model), is_intermediate: outputIsIntermediate, - mode: 'txt2img', + mode, model, num_images: 1, type: nodeType, use_cache: false, width: settings.width, + ...getExternalProviderInvocationOptions(settings, model), + ...(canvas && canvas.mode !== 'txt2img' ? { init_image: { image_name: canvas.initImageName } } : {}), + ...(canvas?.mode === 'inpaint' ? { mask_image: { image_name: canvas.maskImageName } } : {}), }); addEdge(graph, positivePrompt, 'value', output, 'prompt'); @@ -1484,7 +1507,10 @@ const buildExternalGraph = ( (output as Record).reference_images = referenceImages; } } - addMetadata(graph, output, settings, model, null, projectSettings, { scheduler: undefined }); + addMetadata(graph, output, settings, model, null, projectSettings, { + scheduler: undefined, + ...getExternalProviderMetadata(settings, model), + }); addReferenceImageMetadata(graph, output, settings); return graph; diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts index bd621bc83ae..ceb0fea3975 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts @@ -279,6 +279,54 @@ describe('normalizeGenerateSettings', () => { expect(normalized?.vaePrecision).toBe('fp32'); }); + it('keeps valid hosted-provider options and drops the rest back to provider defaults', () => { + const kept = normalizeGenerateSettings({ + ...legacyStoredValues, + externalImageSize: '2K', + geminiTemperature: 0.5, + geminiThinkingLevel: 'minimal', + openaiBackground: 'opaque', + openaiInputFidelity: 'high', + openaiQuality: 'medium', + seedreamOptimizePrompt: true, + seedreamWatermark: true, + }); + + expect(kept).toMatchObject({ + externalImageSize: '2K', + geminiTemperature: 0.5, + geminiThinkingLevel: 'minimal', + openaiBackground: 'opaque', + openaiInputFidelity: 'high', + openaiQuality: 'medium', + seedreamOptimizePrompt: true, + seedreamWatermark: true, + }); + + const dropped = normalizeGenerateSettings({ + ...legacyStoredValues, + externalImageSize: '', + geminiTemperature: 2.5, + geminiThinkingLevel: 'max', + openaiBackground: 'clear', + openaiInputFidelity: 'medium', + openaiQuality: 'ultra', + seedreamOptimizePrompt: 'yes', + seedreamWatermark: 1, + }); + + expect(dropped).toMatchObject({ + externalImageSize: null, + geminiTemperature: null, + geminiThinkingLevel: null, + openaiBackground: 'auto', + openaiInputFidelity: null, + openaiQuality: 'auto', + seedreamOptimizePrompt: false, + seedreamWatermark: false, + }); + }); + it('drops malformed persisted model identifiers missing a base', () => { const normalized = normalizeGenerateSettings({ ...legacyStoredValues, @@ -508,6 +556,8 @@ describe('dimension helpers', () => { expect(getGenerationDimensions({ base: 'flux2', type: 'main' }).grid).toBe(16); expect(getGenerationDimensions({ base: 'qwen-image', type: 'main' }).grid).toBe(16); expect(getGenerationDimensions({ base: 'cogview4', type: 'main' }).grid).toBe(32); + // Hosted providers bucket sizes themselves, so their presets (e.g. 1365 wide) are kept pixel-exact. + expect(getGenerationDimensions({ base: 'external', type: 'external_image_generator' }).grid).toBe(1); expect(clampDimension(888, getGenerationDimensions({ base: 'flux2', type: 'main' }).grid)).toBe(896); }); }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.ts index 4b5cf3c4809..28601d09f43 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.ts @@ -28,6 +28,14 @@ import { KREA2_REBALANCE_WEIGHT_COUNT, } from './conditioningRebalance'; import { isDynamicPromptsSeedBehaviour, sanitizeMaxPrompts, sanitizeSampleSeed } from './dynamicPrompts'; +import { + GEMINI_TEMPERATURE_MAX, + GEMINI_TEMPERATURE_MIN, + isGeminiThinkingLevel, + isOpenAIImageBackground, + isOpenAIImageQuality, + isOpenAIInputFidelity, +} from './externalProvider'; import { clampPidSteps, DEFAULT_PID_STEPS, isPidMode } from './pid'; import { isCanonicalPromptTemplateSnapshot, sanitizePromptTemplateSnapshot } from './promptTemplates'; import { cloneCroppableImage, normalizeCroppableImage } from './referenceImage'; @@ -794,6 +802,20 @@ export const normalizeGenerateSettings = (values: unknown): GenerateSettings | n pidDecoderModel: getModelIdentifierOrNull(values.pidDecoderModel), gemma2EncoderModel: getModelIdentifierOrNull(values.gemma2EncoderModel), pidSteps: clampPidSteps(hasFiniteNumber(values, 'pidSteps') ? (values.pidSteps as number) : DEFAULT_PID_STEPS), + externalImageSize: + typeof values.externalImageSize === 'string' && values.externalImageSize !== '' ? values.externalImageSize : null, + openaiQuality: isOpenAIImageQuality(values.openaiQuality) ? values.openaiQuality : 'auto', + openaiBackground: isOpenAIImageBackground(values.openaiBackground) ? values.openaiBackground : 'auto', + openaiInputFidelity: isOpenAIInputFidelity(values.openaiInputFidelity) ? values.openaiInputFidelity : null, + geminiTemperature: + hasFiniteNumber(values, 'geminiTemperature') && + (values.geminiTemperature as number) >= GEMINI_TEMPERATURE_MIN && + (values.geminiTemperature as number) <= GEMINI_TEMPERATURE_MAX + ? (values.geminiTemperature as number) + : null, + geminiThinkingLevel: isGeminiThinkingLevel(values.geminiThinkingLevel) ? values.geminiThinkingLevel : null, + seedreamWatermark: values.seedreamWatermark === true, + seedreamOptimizePrompt: values.seedreamOptimizePrompt === true, width, }; }; diff --git a/invokeai/frontend/webv2/src/features/generation/core/types.ts b/invokeai/frontend/webv2/src/features/generation/core/types.ts index d32d37543de..b3f11764a50 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/types.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/types.ts @@ -30,18 +30,43 @@ export type MainModelConfig = ModelIdentifierConfig & { } | null; }; +export type ExternalGenerationMode = 'txt2img' | 'img2img' | 'inpaint'; + +export interface ExternalImageSize { + width: number; + height: number; +} + +/** A provider's named size at one aspect ratio, e.g. Gemini's `1K` at `16:9`. */ +export interface ExternalResolutionPreset { + label: string; + aspect_ratio: string; + image_size: string; + width: number; + height: number; +} + +/** The capability matrix the backend records for an external model (`ExternalModelCapabilities`). */ +export interface ExternalModelCapabilities { + modes?: string[]; + supports_seed?: boolean; + supports_negative_prompt?: boolean; + supports_reference_images?: boolean; + max_reference_images?: number | null; + max_images_per_request?: number | null; + max_image_size?: ExternalImageSize | null; + allowed_aspect_ratios?: string[] | null; + /** Fixed output size per aspect ratio, for providers that accept no free-form dimensions. */ + aspect_ratio_sizes?: Record | null; + resolution_presets?: ExternalResolutionPreset[] | null; +} + export type ExternalImageGeneratorModelConfig = ModelIdentifierConfig & { base: 'external'; type: 'external_image_generator'; format: 'external_api'; provider_id?: string; - capabilities?: { - modes?: string[]; - supports_seed?: boolean; - supports_negative_prompt?: boolean; - supports_reference_images?: boolean; - max_reference_images?: number | null; - } | null; + capabilities?: ExternalModelCapabilities | null; default_settings?: { width?: number | null; height?: number | null; @@ -269,8 +294,24 @@ export interface GenerateSettings { /** The shared Gemma-2 caption encoder PiD conditions its decode on. */ gemma2EncoderModel: ComponentModelConfig | null; pidSteps: number; + /** A provider's named size (e.g. `1K`); sent only when the selected model lists it in a resolution preset. */ + externalImageSize: string | null; + openaiQuality: OpenAIImageQuality; + openaiBackground: OpenAIImageBackground; + /** Null lets OpenAI pick; only meaningful with image inputs. */ + openaiInputFidelity: OpenAIInputFidelity | null; + /** Null uses the provider default. */ + geminiTemperature: number | null; + geminiThinkingLevel: GeminiThinkingLevel | null; + seedreamWatermark: boolean; + seedreamOptimizePrompt: boolean; } +export type OpenAIImageQuality = 'auto' | 'high' | 'medium' | 'low'; +export type OpenAIImageBackground = 'auto' | 'transparent' | 'opaque'; +export type OpenAIInputFidelity = 'low' | 'high'; +export type GeminiThinkingLevel = 'minimal' | 'high'; + /** * - `off`: ordinary VAE decode. * - `fit`: generate at the requested size, decode 4x, downscale back to it. diff --git a/invokeai/frontend/webv2/src/features/generation/settings.ts b/invokeai/frontend/webv2/src/features/generation/settings.ts index 8561bbc1a9b..192b0535b9a 100644 --- a/invokeai/frontend/webv2/src/features/generation/settings.ts +++ b/invokeai/frontend/webv2/src/features/generation/settings.ts @@ -54,6 +54,16 @@ export { type GenerateModelSelectionResult, } from './core/baseGenerationPolicies'; export { hasArchitectureCapabilities } from './core/architectureCapabilities'; +export { + GEMINI_TEMPERATURE_MAX, + GEMINI_TEMPERATURE_MIN, + getExternalProviderOptionKeys, + getExternalResolutionPresets, + isGeminiThinkingLevel, + isOpenAIImageBackground, + isOpenAIImageQuality, + isOpenAIInputFidelity, +} from './core/externalProvider'; export { resolveGenerateWidgetValues, type ResolvedGenerateWidgetValues, diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx index 174c5ad2bf1..53f0e66bc65 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx @@ -26,6 +26,7 @@ import { useGenerationUi, } from './GenerationUiContext'; import { AspectRatioLockButton, AspectRatioSelect } from './shared/AspectRatioSelect'; +import { ExternalSizeSelect } from './shared/ExternalSizeSelect'; import { GenerateCollapsibleSection } from './shared/GenerateCollapsibleSection'; import { GenerateFieldContextMenu } from './shared/GenerateFieldContextMenu'; @@ -247,6 +248,7 @@ const selectDimensionSettings = pickGenerateSettings([ 'aspectRatioId', 'aspectRatioIsLocked', 'aspectRatioValue', + 'externalImageSize', 'height', 'width', ]); @@ -519,6 +521,17 @@ export const GenerateDimensionFields = ({ + {selectedModel?.type === 'external_image_generator' ? ( + + ) : null} ({ + useGenerationUi: () => ({ + project: { invocationSourceId: 'generate' }, + sectionPreferences: { sectionsOpen: { providerOptions: true }, setSectionOpen: vi.fn() }, + }), +})); + +const i18n = createInstance(); +void i18n.use(initReactI18next).init({ + fallbackLng: 'en', + initAsync: false, + lng: 'en', + resources: { + en: { + translation: { + common: { scrubber: { editValue: 'Edit {{label}}' } }, + widgets: { + generate: { + override: 'Override', + overridesCount_one: '{{count}} override', + overridesCount_other: '{{count}} overrides', + providerOptions: { + geminiTemperature: 'Temperature', + geminiThinkingLevel: 'Thinking level', + geminiThinkingLevelOptions: { high: 'High', minimal: 'Minimal' }, + openaiBackground: 'Background', + openaiBackgroundOptions: { auto: 'Auto', opaque: 'Opaque', transparent: 'Transparent' }, + openaiInputFidelity: 'Input fidelity', + openaiInputFidelityOptions: { high: 'High', low: 'Low' }, + openaiQuality: 'Quality', + openaiQualityOptions: { auto: 'Auto', high: 'High', low: 'Low', medium: 'Medium' }, + providerDefault: 'Provider default', + seedreamOptimizePrompt: 'Optimize prompt', + seedreamWatermark: 'Watermark', + title: 'Provider options', + }, + }, + }, + }, + }, + }, +}); + +const external = (providerId: string, capabilities: Record): GenerateModelConfig => + ({ + base: 'external', + capabilities, + format: 'external_api', + key: `${providerId}-model`, + name: providerId, + provider_id: providerId, + type: 'external_image_generator', + }) as GenerateModelConfig; +const sd1Model: MainModelConfig = { base: 'sd-1', key: 'sd1', name: 'SD 1.5', type: 'main' }; + +let host: HTMLDivElement | null = null; +let root: Root | null = null; +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const settle = (action: () => void): Promise => + act(async () => { + action(); + await new Promise((resolve) => { + globalThis.setTimeout(resolve, 50); + }); + }); + +const render = async (model: GenerateModelConfig, settings: Partial = {}) => { + host = document.createElement('div'); + document.body.append(host); + root = createRoot(host); + const onCommit = vi.fn(); + + await settle(() => { + root?.render( + + + + + + ); + }); + + return onCommit; +}; + +const selectTriggers = () => [ + ...(host?.querySelectorAll('[data-scope="select"][data-part="trigger"]') ?? []), +]; +const selectNames = () => + selectTriggers().map((trigger) => + (trigger.getAttribute('aria-labelledby') ?? '') + .split(' ') + .map((id) => document.getElementById(id)?.textContent?.trim() ?? '') + .join(' ') + .trim() + ); +const switches = () => [...(host?.querySelectorAll('input[type="checkbox"]') ?? [])]; +const option = (label: string) => + [...document.querySelectorAll('[role="option"]')].find((item) => item.textContent === label) ?? null; + +afterEach(async () => { + await settle(() => root?.unmount()); + host?.remove(); + host = null; + root = null; +}); + +describe('GenerateProviderOptionsSection', () => { + it('renders nothing for local models and providers without options', async () => { + await render(sd1Model); + expect(host?.childElementCount).toBe(0); + + await settle(() => root?.unmount()); + await render(external('alibabacloud', { modes: ['txt2img'] })); + expect(host?.childElementCount).toBe(0); + }); + + it('offers OpenAI quality and background, adding input fidelity only for image inputs', async () => { + await render(external('openai', { modes: ['txt2img'] })); + expect(selectNames()).toEqual(['Quality', 'Background']); + + await settle(() => root?.unmount()); + const onCommit = await render(external('openai', { modes: ['txt2img', 'img2img'] })); + expect(selectNames()).toEqual(['Quality', 'Background', 'Input fidelity']); + + await settle(() => selectTriggers()[1]?.click()); + await settle(() => option('Transparent')?.click()); + expect(onCommit).toHaveBeenCalledWith({ openaiBackground: 'transparent' }); + + await settle(() => selectTriggers()[2]?.click()); + await settle(() => option('High')?.click()); + expect(onCommit).toHaveBeenCalledWith({ openaiInputFidelity: 'high' }); + }); + + it('lets Gemini temperature be overridden and thinking level return to the provider default', async () => { + const onCommit = await render(external('gemini', { modes: ['txt2img'] }), { geminiThinkingLevel: 'high' }); + + expect(selectNames()).toEqual(['Thinking level']); + // Temperature starts at the provider default: no slider until overridden. + expect(host?.querySelector('input[aria-label="Temperature"]')).toBeNull(); + + await settle(() => switches()[0]?.click()); + expect(onCommit).toHaveBeenCalledWith({ geminiTemperature: 1 }); + + await settle(() => selectTriggers()[0]?.click()); + await settle(() => option('Provider default')?.click()); + expect(onCommit).toHaveBeenCalledWith({ geminiThinkingLevel: null }); + }); + + it('toggles the Seedream flags and counts them as overrides', async () => { + const onCommit = await render(external('seedream', { modes: ['txt2img'] }), { seedreamWatermark: true }); + + expect(host?.textContent).toContain('1 override'); + const [watermark, optimizePrompt] = switches(); + + expect(watermark?.checked).toBe(true); + expect(optimizePrompt?.checked).toBe(false); + + await settle(() => optimizePrompt?.click()); + expect(onCommit).toHaveBeenCalledWith({ seedreamOptimizePrompt: true }); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.tsx new file mode 100644 index 00000000000..56a12cdfdb7 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.tsx @@ -0,0 +1,260 @@ +/* oxlint-disable react-perf/jsx-no-new-object-as-prop, react-perf/jsx-no-new-function-as-prop, react-perf/jsx-no-new-array-as-prop, react-perf/jsx-no-jsx-as-prop */ +import type { GenerateModelConfig, GenerateSettings } from '@features/generation/core/types'; + +import { Badge, createListCollection, Stack } from '@chakra-ui/react'; +import { + DEFAULT_GEMINI_TEMPERATURE, + GEMINI_TEMPERATURE_MAX, + GEMINI_TEMPERATURE_MIN, + GEMINI_THINKING_LEVELS, + getExternalProviderOptionKeys, + isGeminiThinkingLevel, + isOpenAIImageBackground, + isOpenAIImageQuality, + isOpenAIInputFidelity, + OPENAI_IMAGE_BACKGROUNDS, + OPENAI_IMAGE_QUALITIES, + OPENAI_INPUT_FIDELITIES, +} from '@features/generation/core/externalProvider'; +import { useExternalStoreSelector } from '@platform/state/selectors'; +import { Field } from '@platform/ui/Field'; +import { ScrubberField } from '@platform/ui/ScrubberField'; +import { Select } from '@platform/ui/Select'; +import { useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { type GenerateDraft, pickGenerateSettings } from './generateDebounce'; +import { GenerateCollapsibleSection } from './shared/GenerateCollapsibleSection'; +import { GenerateToggleSwitch } from './shared/GenerateToggleSwitch'; + +interface GenerateProviderOptionsSectionProps { + draft: GenerateDraft; + selectedModel: GenerateModelConfig | undefined; + onCommit: (patch: Partial) => void; +} + +const selectProviderSettings = pickGenerateSettings([ + 'geminiTemperature', + 'geminiThinkingLevel', + 'openaiBackground', + 'openaiInputFidelity', + 'openaiQuality', + 'seedreamOptimizePrompt', + 'seedreamWatermark', +]); + +/** The empty value stands for "let the provider decide" on the optional selects. */ +const PROVIDER_DEFAULT = ''; + +/** Options a hosted provider's node accepts beyond the shared prompt/size/seed fields; hidden for local models. */ +export const GenerateProviderOptionsSection = ({ + draft, + onCommit, + selectedModel, +}: GenerateProviderOptionsSectionProps) => { + const { t } = useTranslation(); + const settings = useExternalStoreSelector(draft.subscribe, draft.getSnapshot, selectProviderSettings); + const optionKeys = getExternalProviderOptionKeys(selectedModel); + const collections = useMemo( + () => ({ + background: createListCollection({ + items: OPENAI_IMAGE_BACKGROUNDS.map((value) => ({ + label: t(`widgets.generate.providerOptions.openaiBackgroundOptions.${value}`), + value, + })), + }), + inputFidelity: createListCollection({ + items: [ + { label: t('widgets.generate.providerOptions.providerDefault'), value: PROVIDER_DEFAULT }, + ...OPENAI_INPUT_FIDELITIES.map((value) => ({ + label: t(`widgets.generate.providerOptions.openaiInputFidelityOptions.${value}`), + value, + })), + ], + }), + quality: createListCollection({ + items: OPENAI_IMAGE_QUALITIES.map((value) => ({ + label: t(`widgets.generate.providerOptions.openaiQualityOptions.${value}`), + value, + })), + }), + thinkingLevel: createListCollection({ + items: [ + { label: t('widgets.generate.providerOptions.providerDefault'), value: PROVIDER_DEFAULT }, + ...GEMINI_THINKING_LEVELS.map((value) => ({ + label: t(`widgets.generate.providerOptions.geminiThinkingLevelOptions.${value}`), + value, + })), + ], + }), + }), + [t] + ); + + if (optionKeys.length === 0) { + return null; + } + + // Count decisions away from the provider defaults, as the Advanced section does. + const overrideCount = optionKeys.filter((key) => { + switch (key) { + case 'openaiQuality': + case 'openaiBackground': + return settings[key] !== 'auto'; + case 'seedreamWatermark': + case 'seedreamOptimizePrompt': + return settings[key]; + default: + return settings[key] !== null; + } + }).length; + + return ( + 0 ? {t('widgets.generate.overridesCount', { count: overrideCount })} : null + } + sectionId="providerOptions" + > + + {optionKeys.includes('openaiQuality') ? ( + + { + if (isOpenAIImageBackground(value[0])) { + onCommit({ openaiBackground: value[0] }); + } + }} + /> + + ) : null} + {optionKeys.includes('openaiInputFidelity') ? ( + + { + const level = value[0]; + + if (level === PROVIDER_DEFAULT) { + onCommit({ geminiThinkingLevel: null }); + } else if (isGeminiThinkingLevel(level)) { + onCommit({ geminiThinkingLevel: level }); + } + }} + /> + + ) : null} + {optionKeys.includes('seedreamWatermark') ? ( + + onCommit({ seedreamWatermark: checked })} + /> + + ) : null} + {optionKeys.includes('seedreamOptimizePrompt') ? ( + + onCommit({ seedreamOptimizePrompt: checked })} + /> + + ) : null} + + + ); +}; diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx index 86d79bbe0c8..e7b8c4283bf 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx @@ -22,6 +22,7 @@ import { useRegisterGenerateDraftFlusher } from './generateDraftRegistry'; import { getSettingsWithLatestPromptFields } from './generateFormViewModel'; import { GenerateGuidanceSection } from './GenerateGuidanceSection'; import { GenerateModelCard } from './GenerateModelCard'; +import { GenerateProviderOptionsSection } from './GenerateProviderOptionsSection'; import { GenerateRenderSection } from './GenerateRenderSection'; import { GeneratePromptFields } from './promptFields'; @@ -278,6 +279,8 @@ export const GenerateSettingsForm = ({ onCommitImmediate={commitPatchImmediately} /> + + ): GenerateModelConfig => + ({ + base: 'external', + capabilities, + format: 'external_api', + key: 'external', + name: 'External', + provider_id: 'gemini', + type: 'external_image_generator', + }) as GenerateModelConfig; + +const presetModel = external({ + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }, + { aspect_ratio: '16:9', height: 1152, image_size: '2K', label: '16:9 (2K)', width: 2048 }, + ], +}); +const ratioModel = external({ + aspect_ratio_sizes: { '1:1': { height: 1024, width: 1024 }, '7:4': { height: 1024, width: 1792 } }, + modes: ['txt2img'], +}); + +let host: HTMLDivElement | null = null; +let root: Root | null = null; +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const settle = (action: () => void): Promise => + act(async () => { + action(); + await new Promise((resolve) => { + globalThis.setTimeout(resolve, 50); + }); + }); + +const render = async ( + model: GenerateModelConfig, + settings: { externalImageSize: string | null; height: number; width: number } +) => { + host = document.createElement('div'); + document.body.append(host); + root = createRoot(host); + const onCommit = vi.fn(); + + await settle(() => { + root?.render( + + + + + + ); + }); + + return onCommit; +}; + +const trigger = () => host?.querySelector('[data-scope="select"][data-part="trigger"]') ?? null; +const option = (label: string) => + [...document.querySelectorAll('[role="option"]')].find((item) => item.textContent === label) ?? null; + +afterEach(async () => { + await settle(() => root?.unmount()); + host?.remove(); + host = null; + root = null; +}); + +describe('ExternalSizeSelect', () => { + it('renders nothing for providers without a size table', async () => { + await render(external({ modes: ['txt2img'] }), { externalImageSize: null, height: 1024, width: 1024 }); + + expect(host?.childElementCount).toBe(0); + }); + + it('applies a named preset as a locked size with its image size', async () => { + const onCommit = await render(presetModel, { externalImageSize: '1K', height: 1024, width: 1024 }); + + expect(trigger()?.textContent).toContain('1:1 (1K)'); + await settle(() => trigger()?.click()); + await settle(() => option('16:9 (2K)')?.click()); + + expect(onCommit).toHaveBeenCalledWith({ + aspectRatioId: '16:9', + aspectRatioIsLocked: true, + aspectRatioValue: 2048 / 1152, + externalImageSize: '2K', + height: 1152, + width: 2048, + }); + }); + + it('shows a hand-typed size as custom', async () => { + await render(presetModel, { externalImageSize: '1K', height: 1000, width: 1024 }); + + expect(trigger()?.textContent).toContain('Custom'); + }); + + it('applies a fixed ratio size, falling back to a free ratio id for shapes outside the preset list', async () => { + const onCommit = await render(ratioModel, { externalImageSize: null, height: 1024, width: 1024 }); + + expect(trigger()?.textContent).toContain('1:1 (1024×1024)'); + await settle(() => trigger()?.click()); + await settle(() => option('7:4 (1792×1024)')?.click()); + + expect(onCommit).toHaveBeenCalledWith({ + aspectRatioId: 'Free', + aspectRatioIsLocked: true, + aspectRatioValue: 1792 / 1024, + externalImageSize: null, + height: 1024, + width: 1792, + }); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx new file mode 100644 index 00000000000..ce62b7caf6a --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx @@ -0,0 +1,101 @@ +/* oxlint-disable react-perf/jsx-no-new-object-as-prop, react-perf/jsx-no-new-function-as-prop, react-perf/jsx-no-new-array-as-prop */ +import type { GenerateModelConfig, GenerateSettings } from '@features/generation/core/types'; + +import { createListCollection } from '@chakra-ui/react'; +import { + findExternalResolutionPreset, + getExternalAspectRatioSizes, + getExternalResolutionPresets, +} from '@features/generation/core/externalProvider'; +import { isAspectRatioId } from '@features/generation/core/settings'; +import { Field } from '@platform/ui/Field'; +import { Select } from '@platform/ui/Select'; +import { useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; + +type SizeSettings = Pick; + +interface ExternalSizeSelectProps { + model: GenerateModelConfig | undefined; + settings: SizeSettings; + onCommit: (patch: Partial) => void; +} + +const presetKey = (preset: { aspect_ratio: string; image_size: string }): string => + `${preset.aspect_ratio}|${preset.image_size}`; + +/** The size patch a provider entry implies; its ratio is locked so the width/height fields keep it. */ +const sizePatch = (ratio: string, width: number, height: number): Partial => ({ + aspectRatioId: isAspectRatioId(ratio) ? ratio : 'Free', + aspectRatioIsLocked: true, + aspectRatioValue: height > 0 ? width / height : 1, + height, + width, +}); + +/** + * A hosted provider's own size table: named sizes per ratio (Gemini 3's `1K`/`2K`/`4K`), or one fixed size per + * ratio. Renders nothing for free-form providers and local models. A size typed by hand shows as custom. + */ +export const ExternalSizeSelect = ({ model, onCommit, settings }: ExternalSizeSelectProps) => { + const { t } = useTranslation(); + const presets = getExternalResolutionPresets(model); + const ratioSizes = getExternalAspectRatioSizes(model); + const collection = useMemo( + () => + createListCollection({ + items: + presets.length > 0 + ? presets.map((preset) => ({ label: preset.label, value: presetKey(preset) })) + : ratioSizes.map(([ratio, size]) => ({ label: `${ratio} (${size.width}×${size.height})`, value: ratio })), + }), + [presets, ratioSizes] + ); + + if (presets.length === 0 && ratioSizes.length === 0) { + return null; + } + + const currentPreset = presets.length > 0 ? findExternalResolutionPreset(settings, model) : null; + const currentRatio = + presets.length > 0 + ? null + : (ratioSizes.find(([, size]) => size.width === settings.width && size.height === settings.height)?.[0] ?? null); + const value = currentPreset ? [presetKey(currentPreset)] : currentRatio ? [currentRatio] : []; + + return ( + + setField('providerId', event.currentTarget.value)} + /> + + + setField('providerModelId', event.currentTarget.value)} + /> + + + + + {t('models.externalFields.supportedModes')} + + + {MODE_FLAGS.map(({ key, labelKey }) => ( + setField(key, event.checked === true)} + > + + + {t(labelKey)} + + ))} + + + {t('models.externalFields.supportedModesHelp')} + + + + {SUPPORT_FLAGS.map(({ key, labelKey }) => ( + + + + {t(labelKey)} + + setField(key, event.checked)} + > + + + + + {t(labelKey)} + + + + ))} + + + {LIMIT_FIELDS.map((field) => numberField(field, 'models.externalFields.unlimited'))} + + + {DEFAULT_FIELDS.map((field) => numberField(field, 'models.externalFields.providerDefault'))} + + + ); +}; diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx index a354aa5f32e..178633378b2 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx @@ -28,6 +28,8 @@ import { useTranslation } from 'react-i18next'; import { CpuOnlySetting, supportsCpuOnlySetting } from './CpuOnlySetting'; import { supportsDefaultSettings, type DefaultSettingsModel } from './defaultSettingsFields'; import { DefaultSettingsSection } from './DefaultSettingsSection'; +import { supportsExternalModelSettings, type ExternalModelSettingsModel } from './externalModelFields'; +import { ExternalModelSection } from './ExternalModelSection'; import { MissingFileBadge, ModelBaseBadge, ModelFormatBadge } from './ModelBadges'; import { ModelEditForm } from './ModelEditForm'; import { ModelImageUpload } from './ModelImageUpload'; @@ -122,6 +124,22 @@ const selectDefaultSettingsModel = (snapshot: ModelsSnapshot, modelKey: string): : null; }; +const selectExternalModel = (snapshot: ModelsSnapshot, modelKey: string): ExternalModelSettingsModel | null => { + const model = findModel(snapshot, modelKey); + + return model + ? { + capabilities: model.capabilities, + default_settings: model.default_settings, + key: model.key, + name: model.name, + provider_id: model.provider_id, + provider_model_id: model.provider_model_id, + type: model.type, + } + : null; +}; + const selectTriggerPhrasesModel = (snapshot: ModelsSnapshot, modelKey: string): TriggerPhrasesModel | null => { const model = findModel(snapshot, modelKey); @@ -174,6 +192,13 @@ export const ModelDetail = ({ ) : null} + {supportsExternalModelSettings(model) ? ( + <> + + + + ) : null} + {supportsDefaultSettings(model) ? ( <> @@ -358,6 +383,24 @@ const DefaultSettingsSectionContainer = memo(function DefaultSettingsSectionCont ); }); +const ExternalModelSectionContainer = memo(function ExternalModelSectionContainer({ modelKey }: { modelKey: string }) { + const notify = useNotify(); + const { t } = useTranslation(); + const model = useModelsSelector((snapshot) => selectExternalModel(snapshot, modelKey)); + + if (!model || !supportsExternalModelSettings(model)) { + return null; + } + + return ( + notify.error(t('models.externalProvider'), message)} + onSaved={() => notify.success(t('models.externalSettingsSaved'))} + /> + ); +}); + const CpuOnlySettingContainer = memo(function CpuOnlySettingContainer({ modelKey }: { modelKey: string }) { const notify = useNotify(); const { t } = useTranslation(); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.test.ts b/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.test.ts new file mode 100644 index 00000000000..e7f239db5ea --- /dev/null +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.test.ts @@ -0,0 +1,120 @@ +import type { TFunction } from 'i18next'; + +import { describe, expect, it } from 'vitest'; + +import type { ExternalModelSettingsModel } from './externalModelFields'; + +import { toExternalModelChanges, toExternalModelValues, validateExternalModel } from './externalModelFields'; + +const t = ((key: string) => key) as unknown as TFunction; + +const geminiModel: ExternalModelSettingsModel = { + capabilities: { + allowed_aspect_ratios: ['1:1', '16:9'], + aspect_ratio_sizes: { '1:1': { height: 1024, width: 1024 } }, + mask_format: 'none', + max_image_size: { height: 4096, width: 4096 }, + max_images_per_request: 1, + max_reference_images: 14, + modes: ['txt2img'], + resolution_presets: [{ aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }], + supports_reference_images: true, + supports_seed: true, + }, + default_settings: { height: 1024, num_images: 1, width: 1024 }, + key: 'gemini-3-pro', + name: 'Gemini 3 Pro', + provider_id: 'gemini', + provider_model_id: 'gemini-3-pro-image-preview', + type: 'external_image_generator', +}; + +describe('external model record mapping', () => { + it('reads the record into flat editor values, defaulting the fields the backend defaults', () => { + expect(toExternalModelValues(geminiModel)).toEqual({ + defaultHeight: 1024, + defaultNumImages: 1, + defaultWidth: 1024, + maxImageHeight: 4096, + maxImageWidth: 4096, + maxImagesPerRequest: 1, + maxReferenceImages: 14, + modeImg2img: false, + modeInpaint: false, + modeTxt2img: true, + providerId: 'gemini', + providerModelId: 'gemini-3-pro-image-preview', + supportsNegativePrompt: true, + supportsReferenceImages: true, + supportsSeed: true, + }); + // A record with no capability matrix reads as the backend's defaults: txt2img, negative prompt, nothing else. + expect( + toExternalModelValues({ ...geminiModel, capabilities: null, default_settings: null, provider_id: undefined }) + ).toMatchObject({ + defaultWidth: null, + maxImagesPerRequest: null, + modeTxt2img: true, + providerId: '', + supportsNegativePrompt: true, + supportsReferenceImages: false, + supportsSeed: false, + }); + }); + + it('writes a whole capability matrix, keeping the tables the editor does not show', () => { + const changes = toExternalModelChanges( + { + ...toExternalModelValues(geminiModel), + maxImageHeight: null, + maxImageWidth: null, + maxReferenceImages: null, + modeImg2img: true, + modeInpaint: true, + providerId: ' gemini ', + supportsSeed: false, + }, + geminiModel + ); + + expect(changes.provider_id).toBe('gemini'); + expect(changes.capabilities).toEqual({ + allowed_aspect_ratios: ['1:1', '16:9'], + aspect_ratio_sizes: { '1:1': { height: 1024, width: 1024 } }, + mask_format: 'none', + max_image_size: null, + max_images_per_request: 1, + max_reference_images: null, + modes: ['txt2img', 'img2img', 'inpaint'], + resolution_presets: [{ aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }], + supports_negative_prompt: true, + supports_reference_images: true, + supports_seed: false, + }); + expect(changes.default_settings).toEqual({ height: 1024, num_images: 1, width: 1024 }); + }); + + it('clears the defaults record when every default is lifted', () => { + const changes = toExternalModelChanges( + { ...toExternalModelValues(geminiModel), defaultHeight: null, defaultNumImages: null, defaultWidth: null }, + geminiModel + ); + + expect(changes.default_settings).toBeNull(); + }); + + it('rejects an empty route, no modes, half a max size, and non-positive limits', () => { + const values = toExternalModelValues(geminiModel); + + expect(validateExternalModel(values, t)).toBeNull(); + expect(validateExternalModel({ ...values, providerModelId: ' ' }, t)).toBe('Provider model ID is required.'); + expect(validateExternalModel({ ...values, modeTxt2img: false }, t)).toBe('Choose at least one supported mode.'); + expect(validateExternalModel({ ...values, maxImageHeight: null }, t)).toBe( + 'Set both a max width and a max height, or neither.' + ); + expect(validateExternalModel({ ...values, maxImagesPerRequest: 0 }, t)).toBe( + 'Images per request must be greater than 0.' + ); + expect(validateExternalModel({ ...values, defaultWidth: 1024.5 }, t)).toBe('Default width must be a whole number.'); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.ts b/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.ts new file mode 100644 index 00000000000..3dfd3f36986 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.ts @@ -0,0 +1,83 @@ +import type { ExternalModelFormValues } from '@features/models/core/schemas'; +import type { ExternalModelCapabilities, ModelConfig, ModelRecordChanges } from '@features/models/core/types'; +import type { TFunction } from 'i18next'; + +import { externalModelSchema } from '@features/models/core/schemas'; + +export type ExternalModelSettingsModel = Pick< + ModelConfig, + 'capabilities' | 'default_settings' | 'key' | 'name' | 'provider_id' | 'provider_model_id' | 'type' +>; + +export const supportsExternalModelSettings = (model: Pick): boolean => + model.type === 'external_image_generator'; + +const toOptionalInteger = (value: number | null | undefined): number | null => + typeof value === 'number' && Number.isFinite(value) ? value : null; + +/** The editor's flat values for a record: what a draft of the external surface is a difference from. */ +export const toExternalModelValues = (model: ExternalModelSettingsModel): ExternalModelFormValues => { + const capabilities = model.capabilities ?? {}; + const modes = new Set(capabilities.modes ?? ['txt2img']); + + return { + defaultHeight: toOptionalInteger(model.default_settings?.height), + defaultNumImages: toOptionalInteger(model.default_settings?.num_images), + defaultWidth: toOptionalInteger(model.default_settings?.width), + maxImageHeight: toOptionalInteger(capabilities.max_image_size?.height), + maxImageWidth: toOptionalInteger(capabilities.max_image_size?.width), + maxImagesPerRequest: toOptionalInteger(capabilities.max_images_per_request), + maxReferenceImages: toOptionalInteger(capabilities.max_reference_images), + modeImg2img: modes.has('img2img'), + modeInpaint: modes.has('inpaint'), + modeTxt2img: modes.has('txt2img'), + providerId: model.provider_id ?? '', + providerModelId: model.provider_model_id ?? '', + supportsNegativePrompt: capabilities.supports_negative_prompt ?? true, + supportsReferenceImages: capabilities.supports_reference_images === true, + supportsSeed: capabilities.supports_seed === true, + }; +}; + +/** + * The PATCH body for saved values. The backend replaces `capabilities` as a whole, so the fields this editor does not + * show (aspect-ratio tables, presets, mask format) are carried over from the current record. + */ +export const toExternalModelChanges = ( + values: ExternalModelFormValues, + model: ExternalModelSettingsModel +): Pick => { + const capabilities: ExternalModelCapabilities = { + ...model.capabilities, + max_image_size: + values.maxImageWidth !== null && values.maxImageHeight !== null + ? { height: values.maxImageHeight, width: values.maxImageWidth } + : null, + max_images_per_request: values.maxImagesPerRequest, + max_reference_images: values.maxReferenceImages, + modes: [ + ...(values.modeTxt2img ? (['txt2img'] as const) : []), + ...(values.modeImg2img ? (['img2img'] as const) : []), + ...(values.modeInpaint ? (['inpaint'] as const) : []), + ], + supports_negative_prompt: values.supportsNegativePrompt, + supports_reference_images: values.supportsReferenceImages, + supports_seed: values.supportsSeed, + }; + const hasDefaults = values.defaultWidth !== null || values.defaultHeight !== null || values.defaultNumImages !== null; + + return { + capabilities, + default_settings: hasDefaults + ? { height: values.defaultHeight, num_images: values.defaultNumImages, width: values.defaultWidth } + : null, + provider_id: values.providerId.trim(), + provider_model_id: values.providerModelId.trim(), + }; +}; + +export const validateExternalModel = (values: ExternalModelFormValues, t: TFunction): string | null => { + const result = externalModelSchema.safeParse(values); + + return result.success ? null : (result.error.issues[0]?.message ?? t('models.invalidExternalSettings')); +}; diff --git a/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts b/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts index ae4cff9f30e..6997cd4c513 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts +++ b/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts @@ -47,6 +47,21 @@ describe('model drafts store', () => { expect(getModelDraft('m')?.identity?.fields).toEqual({ name: 'Draft 2' }); }); + it('counts hosted-provider record edits as unsaved changes and discards them like defaults', () => { + recordModelDraftFields( + 'm', + 'external', + { modeInpaint: true, providerId: 'openai' }, + { modeInpaint: false, providerId: 'openai' } + ); + + expect(getModelDraft('m')?.external?.fields).toEqual({ modeInpaint: true }); + expect(hasUnsavedModelChanges(getModelDraft('m'))).toBe(true); + + discardModelDraft('m', 'external'); + expect(getModelDraft('m')).toBeUndefined(); + }); + it('treats unset and null default settings as the same value', () => { recordModelDraftFields('m', 'defaults', { cfgScale: null, steps: 30 }, { steps: 30 }); diff --git a/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.ts b/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.ts index 1265343f89a..49cf108c888 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.ts +++ b/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.ts @@ -2,6 +2,7 @@ import type { ModelEditFormValues } from '@features/models/core/schemas'; import type { ModelConfig } from '@features/models/core/types'; import { getModelsSnapshot, subscribeModels } from '@features/models/data/modelsStore'; +import { toExternalModelValues } from '@features/models/ui/detail/externalModelFields'; import { registerAccountOwnedResource } from '@platform/state/accountLifecycle'; import { createKeyedTransientStore } from '@platform/state/externalStore'; import { useExternalStoreSelector, type EqualityFn } from '@platform/state/selectors'; @@ -36,6 +37,8 @@ export interface ModelIdentityDraft extends ModelFieldsDraft { export interface ModelDraft { /** Default-settings edits; absent when nothing differs from the server and no save is in flight. */ readonly defaults?: ModelFieldsDraft; + /** Hosted-provider record edits, kept like `defaults`. */ + readonly external?: ModelFieldsDraft; /** Present exactly while the identity editor is open, even before anything is edited. */ readonly identity?: ModelIdentityDraft; } @@ -68,9 +71,12 @@ export const toModelIdentityValues = (model: ModelIdentitySource): ModelEditForm const SERVER_VALUES: Record object> = { defaults: (model) => model.default_settings ?? {}, + external: toExternalModelValues, identity: toModelIdentityValues, }; +const FIELD_SURFACES = ['identity', 'defaults', 'external'] as const satisfies readonly ModelDraftSurface[]; + // Unset and null are the same "no value" to the backend and to every form here. const isSameValue = (left: unknown, right: unknown): boolean => Object.is(left ?? null, right ?? null); @@ -93,7 +99,7 @@ export const hasModelDraftConflict = (server: object, draft: ModelFieldsDraft | ); export const hasUnsavedModelChanges = (draft: ModelDraft | undefined): boolean => - hasDraftFields(draft?.identity) || hasDraftFields(draft?.defaults); + FIELD_SURFACES.some((surface) => hasDraftFields(draft?.[surface])); /** * Whether a user value is still an edit: it differs from the server, or from a save in flight (a field typed back @@ -123,7 +129,7 @@ const evictBeyondBound = (): void => { }; const writeDraft = (modelKey: string, draft: ModelDraft, edited = false): void => { - if (draft.identity === undefined && draft.defaults === undefined) { + if (FIELD_SURFACES.every((surface) => draft[surface] === undefined)) { deleteDraft(modelKey); return; } @@ -224,7 +230,7 @@ const syncWithLibrary = (): void => { let draft = store.get(key); - for (const surface of ['identity', 'defaults'] as const) { + for (const surface of FIELD_SURFACES) { const surfaceDraft = draft?.[surface]; if (surfaceDraft !== undefined) { diff --git a/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts b/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts index 54dd28668cd..d3b8429764d 100644 --- a/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts +++ b/invokeai/frontend/webv2/src/workbench/canvas-operations/importGalleryImages.test.ts @@ -120,6 +120,14 @@ const setModel = (project: Project, base: GenerateWidgetValues['model']['base']) ideogram4GuidanceScale: null, ideogram4Mu: null, ideogram4ColorPalette: [], + externalImageSize: null, + openaiQuality: 'auto', + openaiBackground: 'auto', + openaiInputFidelity: null, + geminiTemperature: null, + geminiThinkingLevel: null, + seedreamWatermark: false, + seedreamOptimizePrompt: false, krea2RebalanceEnabled: false, krea2RebalanceMultiplier: 4, krea2RebalanceWeights: '1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.5,5.0,1.1,4.0,1.0', diff --git a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts index 83d8be78063..da30143cb74 100644 --- a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts +++ b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts @@ -143,6 +143,14 @@ const createValues = (overrides: Partial = {}): GenerateWi ideogram4GuidanceScale: null, ideogram4Mu: null, ideogram4ColorPalette: [], + externalImageSize: null, + openaiQuality: 'auto', + openaiBackground: 'auto', + openaiInputFidelity: null, + geminiTemperature: null, + geminiThinkingLevel: null, + seedreamWatermark: false, + seedreamOptimizePrompt: false, krea2RebalanceEnabled: false, krea2RebalanceMultiplier: 4, krea2RebalanceWeights: '1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.5,5.0,1.1,4.0,1.0', @@ -567,8 +575,9 @@ describe('image recall', () => { expect(result?.values.aspectRatioId).toBe('2:3'); }); - it('recalls the size for an external generator, which has no architecture row to wait for', () => { - // External providers have no capability row and must remain eligible for dimension recall. + it('recalls the exact size for an external generator, which has no architecture row or grid', () => { + // External providers have no capability row and must remain eligible for dimension recall; their providers + // bucket sizes themselves, so the image's pixel size is kept rather than snapped to a local model grid. const externalModel = { base: 'external', capabilities: { modes: ['txt2img'], supports_seed: true }, @@ -589,11 +598,11 @@ describe('image recall', () => { vaeModels: [], }); - expect(recall()?.values).toMatchObject({ height: 768, width: 512 }); + expect(recall()?.values).toMatchObject({ height: 770, width: 513 }); resetArchitectureCapabilities(); try { - expect(recall()?.values).toMatchObject({ height: 768, width: 512 }); + expect(recall()?.values).toMatchObject({ height: 770, width: 513 }); } finally { setArchitectureCapabilities(architectureCapabilitiesFixture); } @@ -1178,6 +1187,137 @@ describe('image recall', () => { expect(capabilities.remix).toBe(true); }); }); + + describe('external provider options', () => { + const openaiModel = { + base: 'external', + capabilities: { modes: ['txt2img', 'img2img'], supports_reference_images: true }, + format: 'external_api', + key: 'gpt-image', + name: 'GPT Image', + provider_id: 'openai', + type: 'external_image_generator', + } as GenerateWidgetValues['model']; + const geminiModel = { + ...openaiModel, + capabilities: { + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }, + { aspect_ratio: '1:1', height: 2048, image_size: '2K', label: '1:1 (2K)', width: 2048 }, + ], + }, + key: 'gemini-3', + name: 'Gemini 3', + provider_id: 'gemini', + } as GenerateWidgetValues['model']; + const seedreamModel = { + ...openaiModel, + capabilities: { modes: ['txt2img'] }, + key: 'seedream', + name: 'Seedream', + provider_id: 'seedream', + } as GenerateWidgetValues['model']; + // What the provider nodes record (`_build_output_provider_metadata`) plus the named size the base node records. + const providerMetadata = { + gemini_temperature: 0.4, + gemini_thinking_level: 'high', + image_size: '2K', + openai_background: 'transparent', + openai_input_fidelity: 'high', + openai_quality: 'low', + seedream_optimize_prompt: true, + seedream_watermark: true, + }; + const recall = (model: GenerateWidgetValues['model'], metadata: Record, kind: 'all' | 'remix') => + buildImageRecallSettings({ + currentValues: createValues(), + image, + kind, + metadata: { ...metadata, model: { key: model.key } }, + models: [], + supportedModels: [model], + vaeModels: [], + }); + + it('restores only the options the recalled OpenAI model accepts', () => { + const result = recall(openaiModel, providerMetadata, 'all'); + + expect(result?.values).toMatchObject({ + openaiBackground: 'transparent', + openaiInputFidelity: 'high', + openaiQuality: 'low', + }); + // Another provider's keys and a named size the model does not list stay at the current values. + expect(result?.values).toMatchObject({ + externalImageSize: null, + geminiTemperature: null, + geminiThinkingLevel: null, + seedreamOptimizePrompt: false, + seedreamWatermark: false, + }); + expect(result?.fields).toContain('externalProvider'); + }); + + it('restores Gemini sampling and the named size the model lists, on a remix too', () => { + const result = recall(geminiModel, providerMetadata, 'remix'); + + expect(result?.values).toMatchObject({ + externalImageSize: '2K', + geminiTemperature: 0.4, + geminiThinkingLevel: 'high', + openaiQuality: 'auto', + }); + expect(result?.fields).toContain('externalProvider'); + }); + + it('restores Seedream toggles and drops values the node would reject', () => { + const result = recall( + seedreamModel, + { ...providerMetadata, seedream_optimize_prompt: 'yes', seedream_watermark: false }, + 'all' + ); + + expect(result?.values.seedreamWatermark).toBe(false); + expect(result?.values.seedreamOptimizePrompt).toBe(false); + expect(result?.fields).toContain('externalProvider'); + + const invalid = recall(geminiModel, { gemini_temperature: 9, gemini_thinking_level: 'max' }, 'all'); + + expect(invalid?.values.geminiTemperature).toBeNull(); + expect(invalid?.values.geminiThinkingLevel).toBeNull(); + expect(invalid?.fields).not.toContain('externalProvider'); + }); + + it('skips provider metadata when the image was made with a local model', () => { + const result = buildImageRecallSettings({ + currentValues: createValues(), + image, + kind: 'all', + metadata: { ...providerMetadata, model: { key: sdxlModel.key } }, + models: [], + supportedModels: [sdxlModel], + vaeModels: [], + }); + + expect(result?.values.openaiQuality).toBe('auto'); + expect(result?.fields).not.toContain('externalProvider'); + }); + + it('offers recall for an image whose only recallable delta is a provider option', () => { + const capabilities = getImageRecallCapabilities({ + currentValues: createValues({ model: seedreamModel, modelKey: seedreamModel.key }), + image, + metadata: { seedream_watermark: true }, + models: [], + supportedModels: [], + vaeModels: [], + }); + + expect(capabilities.all).toBe(true); + expect(capabilities.remix).toBe(true); + }); + }); }); /** Remix from the metadata a fully filled graph records, starting from SDXL defaults. */ diff --git a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts index 829d4119327..97f02443269 100644 --- a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts +++ b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts @@ -15,6 +15,8 @@ import { deriveAspectRatioId, getCompatibleReferenceImages, getDimensionGrid, + getExternalProviderOptionKeys, + getExternalResolutionPresets, getGenerateModelSelectionResult, getGenerationUiPolicy, isComponentCompatibleWithModel, @@ -28,7 +30,13 @@ import { isMainModelConfig, isModelIdentifierConfig, isVaeCompatibleWithGenerateModel, + isGeminiThinkingLevel, + isOpenAIImageBackground, + isOpenAIImageQuality, + isOpenAIInputFidelity, isValidKrea2RebalanceWeights, + GEMINI_TEMPERATURE_MAX, + GEMINI_TEMPERATURE_MIN, MAX_HIDIFFUSION_RATIO, MIN_HIDIFFUSION_T1_RATIO, normalizeReferenceImages, @@ -73,7 +81,8 @@ type RecalledField = | 'components' | 'loras' | 'referenceImages' - | 'krea2Rebalance'; + | 'krea2Rebalance' + | 'externalProvider'; export type ImageRecallSkipReason = | 'ambiguous' @@ -419,6 +428,92 @@ const getMetadataKrea2Rebalance = ( const hasKrea2Rebalance = (metadata: unknown, model: GenerateModelConfig | undefined): boolean => Object.keys(getMetadataKrea2Rebalance(metadata, model)).length > 0; +/** + * Provider options and the named size, gated on what the effective model's provider accepts: the keys are recorded + * by the provider nodes (`openai_quality`, `gemini_temperature`, ...), so a value only applies to the same provider. + */ +const getMetadataExternalProvider = ( + metadata: unknown, + model: GenerateModelConfig | undefined +): Partial => { + if (model?.type !== 'external_image_generator') { + return {}; + } + + const patch: Partial = {}; + const imageSize = getString(metadata, 'image_size'); + + if (imageSize !== null && getExternalResolutionPresets(model).some((preset) => preset.image_size === imageSize)) { + patch.externalImageSize = imageSize; + } + + for (const key of getExternalProviderOptionKeys(model)) { + switch (key) { + case 'openaiQuality': { + const value = getString(metadata, 'openai_quality'); + + if (isOpenAIImageQuality(value)) { + patch.openaiQuality = value; + } + break; + } + case 'openaiBackground': { + const value = getString(metadata, 'openai_background'); + + if (isOpenAIImageBackground(value)) { + patch.openaiBackground = value; + } + break; + } + case 'openaiInputFidelity': { + const value = getString(metadata, 'openai_input_fidelity'); + + if (isOpenAIInputFidelity(value)) { + patch.openaiInputFidelity = value; + } + break; + } + case 'geminiTemperature': { + const value = getNumber(metadata, 'gemini_temperature'); + + if (value !== null && value >= GEMINI_TEMPERATURE_MIN && value <= GEMINI_TEMPERATURE_MAX) { + patch.geminiTemperature = value; + } + break; + } + case 'geminiThinkingLevel': { + const value = getString(metadata, 'gemini_thinking_level'); + + if (isGeminiThinkingLevel(value)) { + patch.geminiThinkingLevel = value; + } + break; + } + case 'seedreamWatermark': { + const value = getBoolean(metadata, 'seedream_watermark'); + + if (value !== null) { + patch.seedreamWatermark = value; + } + break; + } + case 'seedreamOptimizePrompt': { + const value = getBoolean(metadata, 'seedream_optimize_prompt'); + + if (value !== null) { + patch.seedreamOptimizePrompt = value; + } + break; + } + } + } + + return patch; +}; + +const hasExternalProvider = (metadata: unknown, model: GenerateModelConfig | undefined): boolean => + Object.keys(getMetadataExternalProvider(metadata, model)).length > 0; + type RecalledComponentSetting = keyof Pick< GenerateWidgetValues, | 'clipEmbedModel' @@ -707,6 +802,7 @@ export const getImageRecallCapabilities = ({ const hasReferenceImages = getMetadataReferenceImages(metadata).length > 0; const hasModel = supportedMetadataModel !== null; const hasRebalance = hasKrea2Rebalance(metadata, clipSkipModel); + const hasProviderOptions = hasExternalProvider(metadata, clipSkipModel); const hasConcepts = (getMetadataConcepts(metadata, models, clipSkipModel, supportedMetadataModel).loras?.length ?? 0) > 0; const hasAnyMetadata = @@ -721,7 +817,8 @@ export const getImageRecallCapabilities = ({ hasComponents || hasReferenceImages || hasConcepts || - hasRebalance; + hasRebalance || + hasProviderOptions; const hasNonSeedMetadata = hasModel || hasVae || @@ -733,7 +830,8 @@ export const getImageRecallCapabilities = ({ hasComponents || hasReferenceImages || hasConcepts || - hasRebalance; + hasRebalance || + hasProviderOptions; return { all: hasAnyMetadata && isImageRecallKindAvailable('all'), @@ -803,6 +901,13 @@ export const buildImageRecallSettings = ({ fields.push('krea2Rebalance'); } + const providerPatch = getMetadataExternalProvider(metadata, values.model); + + if (Object.keys(providerPatch).length > 0) { + values = { ...values, ...providerPatch }; + fields.push('externalProvider'); + } + const vae = getMetadataVae(metadata, values.model, vaeModels); if (vae !== undefined) { diff --git a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts index c9db763eea2..a419e62ff81 100644 --- a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts +++ b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts @@ -248,6 +248,14 @@ const createGenerateValues = (overrides: Partial = {}): Ge ideogram4GuidanceScale: null, ideogram4Mu: null, ideogram4ColorPalette: [], + externalImageSize: null, + openaiQuality: 'auto', + openaiBackground: 'auto', + openaiInputFidelity: null, + geminiTemperature: null, + geminiThinkingLevel: null, + seedreamWatermark: false, + seedreamOptimizePrompt: false, krea2RebalanceEnabled: false, krea2RebalanceMultiplier: 4, krea2RebalanceWeights: '1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.5,5.0,1.1,4.0,1.0', From 5a3ccc8cea587526a057e34e961d36e312736167 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 11 Oct 2026 02:33:15 -0600 Subject: [PATCH 02/10] fix(generation): keep hosted-provider sizes, edits, and canvas inputs consistent Starter overlays fill only the capability, default-settings, and panel-schema fields a record never set, so saved external model edits persist through responses and generation; the router and service share one helper. Canvas asks a provider for the bounding box itself and hides render size and denoising strength for it; control layers and regional guidance are rejected by name. A named provider size is sent only when the current dimensions match its preset; the provider id is a select over the registered providers, falling back to text while the list is unavailable. --- .../Users Guide/03.Models/01.introduction.mdx | 8 +- .../03.Models/External Models/openai.mdx | 2 +- .../03.Models/External Models/seedream.mdx | 2 +- invokeai/app/api/routers/model_manager.py | 20 +---- .../external_generation_default.py | 19 +---- .../external_generation/starter_overrides.py | 30 ++++++++ .../frontend/webv2/public/locales/en.json | 1 - .../webv2/scripts/mock-backend-fixtures.mjs | 5 +- .../core/canvas/canvasProcessingSize.test.ts | 13 ++++ .../core/canvas/canvasProcessingSize.ts | 6 +- .../core/canvas/compileCanvasGraph.test.ts | 52 +++++++++++++ .../core/canvas/compileCanvasGraph.ts | 29 ++++--- .../generation/core/externalProvider.test.ts | 13 +++- .../generation/core/externalProvider.ts | 19 +++-- .../features/generation/core/graph.test.ts | 10 +++ .../GenerateDimensionFields.browser.test.tsx | 49 ++++++++++-- .../generation/ui/GenerateDimensionFields.tsx | 3 +- ...ateProviderOptionsSection.browser.test.tsx | 13 ++-- .../ui/GenerateRenderSection.browser.test.tsx | 20 ++++- .../generation/ui/GenerateRenderSection.tsx | 5 +- .../ui/shared/ExternalSizeSelect.tsx | 18 +++-- .../externalModelFields.test.ts | 16 ++-- .../detail => core}/externalModelFields.ts | 12 +-- .../ExternalModelSection.browser.test.tsx | 49 +++++++++++- .../models/ui/detail/ExternalModelSection.tsx | 76 ++++++++++++++++--- .../features/models/ui/detail/ModelDetail.tsx | 5 +- .../features/models/ui/modelDraftsStore.ts | 2 +- tests/app/routers/test_model_manager.py | 48 ++++++++++++ .../test_external_generation_service.py | 49 ++++++++++++ 29 files changed, 472 insertions(+), 122 deletions(-) create mode 100644 invokeai/app/services/external_generation/starter_overrides.py rename invokeai/frontend/webv2/src/features/models/{ui/detail => core}/externalModelFields.test.ts (92%) rename invokeai/frontend/webv2/src/features/models/{ui/detail => core}/externalModelFields.ts (91%) diff --git a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx index a61e2869594..caa505307cf 100644 --- a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx @@ -99,12 +99,16 @@ been provided, the list of models hosted by that service will appear as selections in the Image Generation panel. Opening an external model in the Model Manager shows an **External -Provider** section where you can change the provider and provider model -ID it routes to, the modes (text to image, image to image, inpaint) and +Provider** section where you can change the provider (one of the +registered providers) and provider model ID it routes to, the modes +(text to image, image to image, inpaint) and inputs (seed, reference images, negative prompt) it advertises, its request limits, and the size and image count the provider is asked for by default. The Generate panel and the Canvas only offer what the record advertises, so keep it in step with the provider's documentation. +A starter model's record follows the shipped definition until you save +an edit to its capabilities or defaults; from then on the record's own +values are used. :::note[Gated Downloads] The API Keys section is also used to provide keys for services that diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx index adbb53c771c..9e20bb9c052 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx @@ -56,7 +56,7 @@ The **Size** section lists the provider's fixed sizes per aspect ratio (**Provid ## Canvas -GPT Image models that advertise `img2img` can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. Inpainting and outpainting are not available for OpenAI models. +GPT Image models that advertise `img2img` can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider edits the composite whole at the bounding box's own size, so the render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available for OpenAI models. ## How Requests Are Routed diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx index efd57170120..2ce2095d2f7 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx @@ -57,7 +57,7 @@ Both are recorded in the image metadata and restored by **Recall All** and **Rem ## Canvas -Seedream models advertise `img2img`, so they can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. Inpainting and outpainting are not available. +Seedream models advertise `img2img`, so they can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider edits the composite whole at the bounding box's own size, so the render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available. **Seed** and **guidance scale** are not accepted by the 4.x / 5.x family. diff --git a/invokeai/app/api/routers/model_manager.py b/invokeai/app/api/routers/model_manager.py index 2cc77dfd5f8..e6a00736fad 100644 --- a/invokeai/app/api/routers/model_manager.py +++ b/invokeai/app/api/routers/model_manager.py @@ -25,6 +25,7 @@ from invokeai.app.api.auth_dependencies import AdminUserOrDefault, CurrentUserOrDefault from invokeai.app.api.dependencies import ApiDependencies +from invokeai.app.services.external_generation.starter_overrides import apply_external_starter_overrides from invokeai.app.services.model_images.model_images_common import ( ModelImageFileDeleteException, ModelImageFileNotFoundException, @@ -188,26 +189,11 @@ def add_cover_image_to_model_config(config: AnyModelConfig, dependencies: Type[A def apply_external_starter_model_overrides(config: AnyModelConfig) -> AnyModelConfig: - """Overlay starter-model metadata onto installed external model configs.""" + """Fill an installed external model config's unset fields from its starter definition.""" if not isinstance(config, ExternalApiModelConfig): return config - starter_match = next((starter for starter in STARTER_MODELS if starter.source == config.source), None) - if starter_match is None: - return config - - model_updates: dict[str, object] = {} - if starter_match.capabilities is not None: - model_updates["capabilities"] = starter_match.capabilities - if starter_match.default_settings is not None: - model_updates["default_settings"] = starter_match.default_settings - if starter_match.panel_schema is not None: - model_updates["panel_schema"] = starter_match.panel_schema - - if not model_updates: - return config - - return config.model_copy(update=model_updates) + return apply_external_starter_overrides(config) def prepare_model_config_for_response(config: AnyModelConfig, dependencies: Type[ApiDependencies]) -> AnyModelConfig: diff --git a/invokeai/app/services/external_generation/external_generation_default.py b/invokeai/app/services/external_generation/external_generation_default.py index 506c0020fdc..1e1305a2509 100644 --- a/invokeai/app/services/external_generation/external_generation_default.py +++ b/invokeai/app/services/external_generation/external_generation_default.py @@ -26,8 +26,8 @@ ExternalGenerationResult, ExternalProviderStatus, ) +from invokeai.app.services.external_generation.starter_overrides import apply_external_starter_overrides from invokeai.backend.model_manager.configs.external_api import ExternalApiModelConfig, ExternalImageSize -from invokeai.backend.model_manager.starter_models import STARTER_MODELS if TYPE_CHECKING: from invokeai.app.services.model_records import ModelRecordServiceBase @@ -188,7 +188,7 @@ def _refresh_model_capabilities(self, request: ExternalGenerationRequest) -> Ext if record.provider_model_id != request.model.provider_model_id: return request - record = _apply_starter_overrides(record) + record = apply_external_starter_overrides(record) if record == request.model: return request @@ -402,18 +402,3 @@ def _resize_result_images(result: ExternalGenerationResult, width: int, height: provider_metadata=result.provider_metadata, content_filters=result.content_filters, ) - - -def _apply_starter_overrides(model: ExternalApiModelConfig) -> ExternalApiModelConfig: - source = model.source or f"external://{model.provider_id}/{model.provider_model_id}" - starter_match = next((starter for starter in STARTER_MODELS if starter.source == source), None) - if starter_match is None: - return model - updates: dict[str, object] = {} - if starter_match.capabilities is not None: - updates["capabilities"] = starter_match.capabilities - if starter_match.default_settings is not None: - updates["default_settings"] = starter_match.default_settings - if not updates: - return model - return model.model_copy(update=updates) diff --git a/invokeai/app/services/external_generation/starter_overrides.py b/invokeai/app/services/external_generation/starter_overrides.py new file mode 100644 index 00000000000..b1a3ddac67f --- /dev/null +++ b/invokeai/app/services/external_generation/starter_overrides.py @@ -0,0 +1,30 @@ +from invokeai.backend.model_manager.configs.external_api import ExternalApiModelConfig, ExternalModelCapabilities +from invokeai.backend.model_manager.starter_models import STARTER_MODELS + +_UNSET_CAPABILITIES = ExternalModelCapabilities() + + +def apply_external_starter_overrides(config: ExternalApiModelConfig) -> ExternalApiModelConfig: + """Fill a starter-sourced record's unset fields from its starter definition. + + A record that never set a field (the default capability matrix, no default settings, no panel schema) + follows the shipped starter definition, so definition updates reach it. A field the record sets, by an + install that copied the definition or by an edit in the model editor, is the record's own and wins. + """ + source = config.source or f"external://{config.provider_id}/{config.provider_model_id}" + starter_match = next((starter for starter in STARTER_MODELS if starter.source == source), None) + if starter_match is None: + return config + + updates: dict[str, object] = {} + if starter_match.capabilities is not None and config.capabilities == _UNSET_CAPABILITIES: + updates["capabilities"] = starter_match.capabilities + if starter_match.default_settings is not None and config.default_settings is None: + updates["default_settings"] = starter_match.default_settings + if starter_match.panel_schema is not None and config.panel_schema is None: + updates["panel_schema"] = starter_match.panel_schema + + if not updates: + return config + + return config.model_copy(update=updates) diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 45c2750e3a5..9346ae161b1 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -1343,7 +1343,6 @@ "saveExternalSettings": "Save Provider Settings", "externalSettingsSaved": "Provider settings saved", "failedToSaveExternalSettings": "Failed to save provider settings.", - "invalidExternalSettings": "Invalid provider settings.", "externalModes": { "txt2img": "Text to image", "img2img": "Image to image", diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs index 800e354d78d..d5b3cdb2a6f 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs @@ -399,6 +399,7 @@ const MODEL_TYPES = ['main', 'lora', 'controlnet', 'vae', 'embedding']; /** * Hosted-provider records with the capability shapes the Generate panel, Canvas, and Model Manager read: a named * resolution table (Gemini), fixed sizes per ratio with an edit mode (OpenAI), and provider toggles (Seedream). + * No shipped provider accepts a mask, so none advertises inpaint. */ const EXTERNAL_MODEL_FIXTURES = [ { @@ -457,12 +458,12 @@ const EXTERNAL_MODEL_FIXTURES = [ '2:3': { height: 1536, width: 1024 }, }, max_images_per_request: 10, - modes: ['txt2img', 'img2img', 'inpaint'], + modes: ['txt2img', 'img2img'], supports_reference_images: true, }, cover_image: null, default_settings: { height: 1024, num_images: 1, width: 1024 }, - description: 'Fixture hosted model with fixed ratio sizes and canvas edit modes.', + description: 'Fixture hosted model with fixed ratio sizes and canvas image-to-image.', file_size: 0, format: 'external_api', hash: 'external:openai:fixture-gpt-image', diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts index b29d7bf78b3..37326ca3cee 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts @@ -9,8 +9,21 @@ seedArchitectureCapabilities(); const sd1 = { base: 'sd-1', type: 'main' } as GenerateModelConfig; const sdxl = { base: 'sdxl', type: 'main' } as GenerateModelConfig; +const external = { base: 'external', type: 'external_image_generator' } as GenerateModelConfig; describe('resolveCanvasProcessingSize', () => { + it('asks a hosted provider for the bbox itself, whatever the scaling method', () => { + const bbox = { height: 300, width: 500 }; + + expect(resolveCanvasProcessingSize(external, 'off', bbox, undefined)).toEqual(bbox); + expect(resolveCanvasProcessingSize(external, 'off', bbox, { height: null, method: 'auto', width: null })).toEqual( + bbox + ); + expect(resolveCanvasProcessingSize(external, 'off', bbox, { height: 2048, method: 'manual', width: 2048 })).toEqual( + bbox + ); + }); + it('keeps the bbox on the model grid when scaling is off or unset', () => { expect(resolveCanvasProcessingSize(sd1, 'off', { height: 300, width: 500 }, undefined)).toEqual({ height: 304, diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts index a3ea9431b43..b3976bc406b 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts @@ -1,6 +1,7 @@ /** * None snaps the bbox to the model grid; auto grows undersized content toward optimal area at its aspect ratio; - * manual uses the selected size. Restore final output to the exact bbox. + * manual uses the selected size. Restore final output to the exact bbox. Hosted providers are asked for the bbox + * itself: they bucket the request to their own size table and return one image, so there is nothing to scale. */ import type { GenerateModelConfig, PidMode } from '@features/generation/core/types'; @@ -80,6 +81,9 @@ export const resolveCanvasProcessingSize = ( bbox: CanvasSize, scaling: CanvasScalingSettings | undefined ): CanvasSize => { + if (model.type === 'external_image_generator') { + return { height: bbox.height, width: bbox.width }; + } const { grid, optimal } = getGenerationDimensions(model, pidMode); switch (scaling?.method) { case 'auto': { diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index 80610f788cd..ea9ba34ef0a 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -493,6 +493,58 @@ describe('compileCanvasGraph', () => { 'OpenAI Image does not support outpainting on the canvas.' ); }); + + it('rejects control layers and regional guidance by name instead of failing inside a graft', () => { + const layer: ControlLayerGraphInput = { + beginEndStepPct: [0, 1], + controlMode: 'balanced', + id: 'control', + imageName: 'control.png', + kind: 'controlnet', + model: { base: 'sd-1', key: 'canny', name: 'Canny', type: 'controlnet' }, + weight: 1, + }; + + expect(() => compile(editModel, 'txt2img', { compositeImageName: null, controlLayers: [layer] })).toThrow( + 'OpenAI Image does not support control layers.' + ); + expect(() => + compile(editModel, 'txt2img', { + compositeImageName: null, + regionalGuidance: [ + { + autoNegative: false, + id: 'r1', + maskImageName: 'r1-mask.png', + negativePrompt: null, + positivePrompt: 'a cat', + referenceImages: [], + }, + ], + }) + ).toThrow('OpenAI Image does not support regional guidance.'); + }); + + it('ignores the denoising strength: the provider edits at its own, so none is validated, sent or recorded', () => { + const { backendGraph } = compile(editModel, 'img2img', { compositeImageName: 'bbox.png', strength: 0 }); + const metadata = getNodeByType(backendGraph, 'core_metadata'); + + expect(backendGraph.nodes.canvas_output).not.toHaveProperty('strength'); + expect(backendGraph.nodes.canvas_output).not.toHaveProperty('denoising_start'); + expect(metadata).not.toHaveProperty('strength'); + expect(metadata).not.toHaveProperty('generation_mode'); + }); + + it('asks for the bbox itself even when the render size is set to auto or a custom size', () => { + const custom = compile(editModel, 'img2img', { + bbox: smallBbox, + compositeImageName: 'bbox.png', + scaling: { height: 2048, method: 'manual', width: 2048 }, + }); + + expect(custom.backendGraph.nodes.canvas_output).toMatchObject({ height: 512, width: 640 }); + expect(Object.values(custom.backendGraph.nodes).map((node) => node.type)).not.toContain('img_resize'); + }); }); describe('unsupported models', () => { diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts index 3ef293b735e..7d86c2dd3bc 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts @@ -63,10 +63,19 @@ const getCanvasValidationReasons = (input: CompileCanvasGraphInput): string[] => const { bbox, compositeImageName, maskImageName, mode, model, strength } = input; const reasons: string[] = []; - // Providers take a whole image to edit; a partially covered bbox has nothing to send for the empty area. The - // shared validator reports the other modes the model does not advertise. - if (model.type === 'external_image_generator' && mode === 'outpaint') { - reasons.push(`${model.name} does not support outpainting on the canvas.`); + if (model.type === 'external_image_generator') { + // A provider takes a whole image to edit and nothing else: a partially covered bbox has nothing to send for the + // empty area, and the request carries no conditioning beyond the prompt, the init image and the mask. The + // shared validator reports the other modes the model does not advertise. + if (mode === 'outpaint') { + reasons.push(`${model.name} does not support outpainting on the canvas.`); + } + if (input.controlLayers && input.controlLayers.length > 0) { + reasons.push(`${model.name} does not support control layers.`); + } + if (input.regionalGuidance && input.regionalGuidance.length > 0) { + reasons.push(`${model.name} does not support regional guidance.`); + } } // Reject canvas PiD because its graph lacks the expected VAE/decode seam. @@ -85,7 +94,8 @@ const getCanvasValidationReasons = (input: CompileCanvasGraphInput): string[] => reasons.push('Canvas generation requires a composited source image.'); } - if (!Number.isFinite(strength) || strength <= 0 || strength > 1) { + // A provider edits at its own strength; only the local denoise reads this one. + if (model.type !== 'external_image_generator' && (!Number.isFinite(strength) || strength <= 0 || strength > 1)) { reasons.push('Canvas denoising strength must be greater than 0 and at most 1.'); } } @@ -610,11 +620,7 @@ const graftOutpaint = ( /** Throw the first validation reason. */ export const compileCanvasGraph = (input: CompileCanvasGraphInput): CompiledCanvasGraph => { const { bbox, compositeImageName, destination, mode, model, projectSettings, strength } = input; - // A provider buckets the request to its own size table and returns one image, so it is asked for the bbox itself. - const processingSize = - model.type === 'external_image_generator' - ? { height: bbox.height, width: bbox.width } - : resolveCanvasProcessingSize(model, input.settings.pidMode, bbox, input.scaling); + const processingSize = resolveCanvasProcessingSize(model, input.settings.pidMode, bbox, input.scaling); const settings = withProcessingDimensions(input.settings, processingSize); const validationReasons = [ @@ -639,7 +645,8 @@ export const compileCanvasGraph = (input: CompileCanvasGraphInput): CompiledCanv } // Validation above guarantees the composite for the edit modes and the mask for inpaint. The provider returns - // the edited image whole, so no encode, mask tail, control, or regional grafts apply. + // the edited image whole, so no encode, mask tail, control, or regional grafts apply, and no denoising strength + // is sent or recorded. Metadata carries no `generation_mode`: the backend's mode list has no provider variants. const initImageName = compositeImageName as string; const backendGraph = buildExternalGraph( settings, diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts index e23290caf76..5c1d627f2cd 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts @@ -56,13 +56,18 @@ describe('external provider options', () => { ).toContain('openaiInputFidelity'); }); - it('resolves a named size only from the selected model’s own presets', () => { + it('resolves a named size only from the selected model’s own presets, at the preset’s exact size', () => { const gemini = external('gemini', { modes: ['txt2img'], resolution_presets: presets }); - expect(resolveExternalImageSize({ externalImageSize: '2K' }, gemini)).toBe('2K'); - expect(resolveExternalImageSize({ externalImageSize: '4K' }, gemini)).toBeNull(); + expect(resolveExternalImageSize({ externalImageSize: '2K', height: 1152, width: 2048 }, gemini)).toBe('2K'); + expect(resolveExternalImageSize({ externalImageSize: '4K', height: 1152, width: 2048 }, gemini)).toBeNull(); + // A size typed by hand after picking a preset is custom: the provider buckets the dimensions instead. + expect(resolveExternalImageSize({ externalImageSize: '2K', height: 1152, width: 2000 }, gemini)).toBeNull(); expect( - resolveExternalImageSize({ externalImageSize: '2K' }, external('openai', { modes: ['txt2img'] })) + resolveExternalImageSize( + { externalImageSize: '2K', height: 1152, width: 2048 }, + external('openai', { modes: ['txt2img'] }) + ) ).toBeNull(); }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts index 84f81fce724..6951bfc77a4 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts @@ -97,16 +97,6 @@ export const getExternalAspectRatioSizes = ( return sizes ? Object.entries(sizes) : []; }; -/** The named size to send: only one the model lists, so a stale pick from another model never reaches the provider. */ -export const resolveExternalImageSize = ( - settings: Pick, - model: GenerateModelConfig | undefined -): string | null => - settings.externalImageSize !== null && - getExternalResolutionPresets(model).some((preset) => preset.image_size === settings.externalImageSize) - ? settings.externalImageSize - : null; - /** The preset matching the current size and named size exactly, for the preset select's value. */ export const findExternalResolutionPreset = ( settings: Pick, @@ -119,6 +109,15 @@ export const findExternalResolutionPreset = ( preset.height === settings.height ) ?? null; +/** + * The named size to send: the selected model's preset the current size matches exactly, so a hand-typed size or a + * pick left over from another model never overrides the dimensions at the provider. + */ +export const resolveExternalImageSize = ( + settings: Pick, + model: GenerateModelConfig | undefined +): string | null => findExternalResolutionPreset(settings, model)?.image_size ?? null; + /** * Provider-specific invocation fields for the selected model, keyed as the node declares them. Optional fields are * omitted when unset so the node applies its own default. diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts index 04e1853ba11..70f141f1c71 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts @@ -704,6 +704,16 @@ describe('compileGenerateGraph', () => { expect(unlisted.nodes.canvas_output?.image_size).toBeNull(); expect(getNodeByType(unlisted, 'core_metadata')).not.toHaveProperty('image_size'); + + // Dimensions typed after the pick are custom: the named size would otherwise override them at the provider. + const typed = compile(providerModel('gemini', { modes: ['txt2img'], resolution_presets: presets }), { + ...providerSettings, + height: 2048, + width: 1536, + }); + + expect(typed.nodes.canvas_output).toMatchObject({ height: 2048, image_size: null, width: 1536 }); + expect(getNodeByType(typed, 'core_metadata')).not.toHaveProperty('image_size'); }); it('accepts provider sizes that are not multiples of a local model grid', () => { diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx index d8fd6865b3f..deb161a88a9 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx @@ -1,5 +1,5 @@ /* oxlint-disable react-perf/jsx-no-new-object-as-prop */ -import type { GenerateSettings, MainModelConfig } from '@features/generation/core/types'; +import type { GenerateModelConfig, GenerateSettings, MainModelConfig } from '@features/generation/core/types'; import { ChakraProvider } from '@chakra-ui/react'; import { getDefaultGenerateSettings } from '@features/generation/core/baseGenerationPolicies'; @@ -11,13 +11,23 @@ import { createRoot, type Root } from 'react-dom/client'; import { I18nextProvider, initReactI18next } from 'react-i18next'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { CanvasRenderSizeProps } from './GenerationUiContext'; + import { GenerateDimensionFields } from './GenerateDimensionFields'; +const harness = vi.hoisted(() => ({ invocationSourceId: 'generate' })); + vi.mock('./GenerationUiContext', () => ({ useGenerationQueueInsights: (select: (insights: unknown) => unknown) => select({ secondsPerRun: null, seedHistory: [] }), useGenerationUi: () => ({ - project: { invocationSourceId: 'generate' }, + // Stands in for the canvas's render-size slot, which lays out the Size section around its controls. + CanvasRenderSize: ({ children, frame }: CanvasRenderSizeProps) => + children({ + controls:
, + size: { height: frame.height * 2, width: frame.width * 2 }, + }), + project: { invocationSourceId: harness.invocationSourceId }, sectionPreferences: { sectionsOpen: { dimensions: true }, setSectionOpen: vi.fn() }, }), })); @@ -49,6 +59,15 @@ void i18n.use(initReactI18next).init({ }); const sd1Model: MainModelConfig = { base: 'sd-1', key: 'sd1', name: 'SD 1.5', type: 'main' }; +const gptImage: GenerateModelConfig = { + base: 'external', + capabilities: { modes: ['txt2img', 'img2img'] }, + format: 'external_api', + key: 'gpt-image', + name: 'GPT Image', + provider_id: 'openai', + type: 'external_image_generator', +}; let host: HTMLDivElement | null = null; let root: Root | null = null; @@ -62,7 +81,7 @@ const settle = (action: () => void): Promise => }); }); -const render = async (settings: Partial) => { +const render = async (settings: Partial, model: GenerateModelConfig = sd1Model) => { host = document.createElement('div'); document.body.append(host); root = createRoot(host); @@ -73,9 +92,9 @@ const render = async (settings: Partial) => { @@ -94,12 +113,32 @@ const bracket = () => host?.querySelector('[data-part="bracket"]')?.parentElemen const presetTrigger = () => host?.querySelector('[role="combobox"]') ?? null; afterEach(async () => { + harness.invocationSourceId = 'generate'; await settle(() => root?.unmount()); host?.remove(); host = null; root = null; }); +describe('GenerateDimensionFields canvas render size', () => { + it('lays out the render-size controls and the resized badge for a local model on the canvas', async () => { + harness.invocationSourceId = 'canvas'; + await render({ height: 512, width: 512 }); + + expect(host?.querySelector('[data-testid="render-size-controls"]')).not.toBeNull(); + expect(host?.textContent).toContain('512x512 → 1024x1024'); + }); + + it('shows a hosted provider only the frame, which is the size the provider is asked for', async () => { + harness.invocationSourceId = 'canvas'; + await render({ height: 512, width: 512 }, gptImage); + + expect(host?.querySelector('[data-testid="render-size-controls"]')).toBeNull(); + expect(host?.textContent).not.toContain('→'); + expect(host?.textContent).toContain('512x512'); + }); +}); + describe('GenerateDimensionFields aspect ratio', () => { it('reshapes the size to a chosen preset and locks it', async () => { const onCommit = await render({ aspectRatioId: 'Free', aspectRatioIsLocked: false, height: 1024, width: 1024 }); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx index 53f0e66bc65..477afd1f0e5 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx @@ -583,7 +583,8 @@ export const GenerateDimensionFields = ({ }; // Switching source remounts the section's subtree; that is deliberate, as its state lives in this component. - return project.invocationSourceId === 'canvas' ? ( + // A hosted provider is asked for the frame itself (it buckets to its own size table), so there is no render size. + return project.invocationSourceId === 'canvas' && selectedModel?.type !== 'external_image_generator' ? ( {renderSection} ) : ( renderSection(null) diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx index 0062617452d..89da813f69a 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx @@ -84,17 +84,15 @@ const render = async (model: GenerateModelConfig, settings: Partial) => draft.setSnapshot({ ...draft.getSnapshot(), ...patch })); await settle(() => { root?.render( - + ); @@ -157,10 +155,11 @@ describe('GenerateProviderOptionsSection', () => { expect(selectNames()).toEqual(['Thinking level']); // Temperature starts at the provider default: no slider until overridden. - expect(host?.querySelector('input[aria-label="Temperature"]')).toBeNull(); + expect(host?.querySelector('[role="slider"]')).toBeNull(); await settle(() => switches()[0]?.click()); expect(onCommit).toHaveBeenCalledWith({ geminiTemperature: 1 }); + expect(host?.querySelector('[role="slider"]')?.getAttribute('aria-valuenow')).toBe('1'); await settle(() => selectTriggers()[0]?.click()); await settle(() => option('Provider default')?.click()); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.browser.test.tsx index 7b97d4bf1ce..bc627e0c564 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.browser.test.tsx @@ -1,5 +1,5 @@ /* oxlint-disable react-perf/jsx-no-new-object-as-prop */ -import type { GenerateSettings, MainModelConfig } from '@features/generation/core/types'; +import type { GenerateModelConfig, GenerateSettings, MainModelConfig } from '@features/generation/core/types'; import { ChakraProvider } from '@chakra-ui/react'; import { seedArchitectureCapabilities } from '@features/generation/core/architectureCapabilities.testing'; @@ -141,7 +141,7 @@ const settle = (action: () => void): Promise => }); }); -const render = async (model: MainModelConfig, settings: Partial = {}) => { +const render = async (model: GenerateModelConfig, settings: Partial = {}) => { host = document.createElement('div'); document.body.append(host); root = createRoot(host); @@ -286,6 +286,22 @@ describe('GenerateRenderSection denoising strength', () => { expect(guidanceIndex).toBeGreaterThan(-1); expect(controls[guidanceIndex + 1]?.getAttribute('data-testid')).toBe('strength-field'); }); + + it('adds no strength for a hosted provider, which edits the canvas composite at its own', async () => { + harness.invocationSourceId = 'canvas'; + await render({ + base: 'external', + capabilities: { modes: ['txt2img', 'img2img'] }, + format: 'external_api', + key: 'gpt-image', + name: 'GPT Image', + provider_id: 'openai', + type: 'external_image_generator', + }); + + expect(host?.querySelector('[data-testid="strength-field"]')).toBeNull(); + expect(host?.querySelector('[data-testid="strength-badge"]')).toBeNull(); + }); }); /** Resolve names through both aria-labelledby and aria-label. */ diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.tsx index f565e7efb80..9e5d9645345 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRenderSection.tsx @@ -472,8 +472,9 @@ export const GenerateRenderSection = ({ ); - // Switching source remounts the section's subtree, as the Size section does. - return project.invocationSourceId === 'canvas' ? ( + // Switching source remounts the section's subtree, as the Size section does. A hosted provider edits the canvas + // composite at its own strength, so the canvas adds none for it. + return project.invocationSourceId === 'canvas' && selectedModel?.type !== 'external_image_generator' ? ( {renderSection} ) : ( renderSection(null) diff --git a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx index ce62b7caf6a..13b541013c8 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx @@ -39,18 +39,22 @@ const sizePatch = (ratio: string, width: number, height: number): Partial { const { t } = useTranslation(); - const presets = getExternalResolutionPresets(model); - const ratioSizes = getExternalAspectRatioSizes(model); - const collection = useMemo( - () => - createListCollection({ + // One table per model: the policy reads fresh arrays off the record, so the record is the memo key. + const { collection, presets, ratioSizes } = useMemo(() => { + const presets = getExternalResolutionPresets(model); + const ratioSizes = getExternalAspectRatioSizes(model); + + return { + collection: createListCollection({ items: presets.length > 0 ? presets.map((preset) => ({ label: preset.label, value: presetKey(preset) })) : ratioSizes.map(([ratio, size]) => ({ label: `${ratio} (${size.width}×${size.height})`, value: ratio })), }), - [presets, ratioSizes] - ); + presets, + ratioSizes, + }; + }, [model]); if (presets.length === 0 && ratioSizes.length === 0) { return null; diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.test.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts similarity index 92% rename from invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.test.ts rename to invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts index e7f239db5ea..22b4a53ea48 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.test.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts @@ -1,13 +1,9 @@ -import type { TFunction } from 'i18next'; - import { describe, expect, it } from 'vitest'; import type { ExternalModelSettingsModel } from './externalModelFields'; import { toExternalModelChanges, toExternalModelValues, validateExternalModel } from './externalModelFields'; -const t = ((key: string) => key) as unknown as TFunction; - const geminiModel: ExternalModelSettingsModel = { capabilities: { allowed_aspect_ratios: ['1:1', '16:9'], @@ -106,15 +102,15 @@ describe('external model record mapping', () => { it('rejects an empty route, no modes, half a max size, and non-positive limits', () => { const values = toExternalModelValues(geminiModel); - expect(validateExternalModel(values, t)).toBeNull(); - expect(validateExternalModel({ ...values, providerModelId: ' ' }, t)).toBe('Provider model ID is required.'); - expect(validateExternalModel({ ...values, modeTxt2img: false }, t)).toBe('Choose at least one supported mode.'); - expect(validateExternalModel({ ...values, maxImageHeight: null }, t)).toBe( + expect(validateExternalModel(values)).toBeNull(); + expect(validateExternalModel({ ...values, providerModelId: ' ' })).toBe('Provider model ID is required.'); + expect(validateExternalModel({ ...values, modeTxt2img: false })).toBe('Choose at least one supported mode.'); + expect(validateExternalModel({ ...values, maxImageHeight: null })).toBe( 'Set both a max width and a max height, or neither.' ); - expect(validateExternalModel({ ...values, maxImagesPerRequest: 0 }, t)).toBe( + expect(validateExternalModel({ ...values, maxImagesPerRequest: 0 })).toBe( 'Images per request must be greater than 0.' ); - expect(validateExternalModel({ ...values, defaultWidth: 1024.5 }, t)).toBe('Default width must be a whole number.'); + expect(validateExternalModel({ ...values, defaultWidth: 1024.5 })).toBe('Default width must be a whole number.'); }); }); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts similarity index 91% rename from invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.ts rename to invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts index 3dfd3f36986..bf146f074f1 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/externalModelFields.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts @@ -1,8 +1,7 @@ -import type { ExternalModelFormValues } from '@features/models/core/schemas'; -import type { ExternalModelCapabilities, ModelConfig, ModelRecordChanges } from '@features/models/core/types'; -import type { TFunction } from 'i18next'; +import type { ExternalModelFormValues } from './schemas'; +import type { ExternalModelCapabilities, ModelConfig, ModelRecordChanges } from './types'; -import { externalModelSchema } from '@features/models/core/schemas'; +import { externalModelSchema } from './schemas'; export type ExternalModelSettingsModel = Pick< ModelConfig, @@ -76,8 +75,9 @@ export const toExternalModelChanges = ( }; }; -export const validateExternalModel = (values: ExternalModelFormValues, t: TFunction): string | null => { +/** The first schema message, like the other record editors' messages, so the UI shows it as is. */ +export const validateExternalModel = (values: ExternalModelFormValues): string | null => { const result = externalModelSchema.safeParse(values); - return result.success ? null : (result.error.issues[0]?.message ?? t('models.invalidExternalSettings')); + return result.success ? null : (result.error.issues[0]?.message ?? 'Invalid provider settings.'); }; diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx index 372e62adca1..51bc8a8222d 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx @@ -1,3 +1,4 @@ +import type { ExternalModelSettingsModel } from '@features/models/core/externalModelFields'; import type * as ModelsApi from '@features/models/data/api'; import { ChakraProvider } from '@chakra-ui/react'; @@ -8,11 +9,15 @@ import { act } from 'react'; import { createRoot, type Root } from 'react-dom/client'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { ExternalModelSettingsModel } from './externalModelFields'; - import { ExternalModelSection } from './ExternalModelSection'; const api = vi.hoisted(() => ({ + getExternalProviderConfigs: vi.fn(() => + Promise.resolve([ + { api_key_configured: true, base_url: null, provider_id: 'openai' }, + { api_key_configured: false, base_url: null, provider_id: 'gemini' }, + ]) + ), getModelsDir: vi.fn(() => Promise.resolve('')), listMissingModels: vi.fn(() => Promise.resolve([])), listModels: vi.fn(() => Promise.resolve([])), @@ -87,8 +92,19 @@ describe('ExternalModelSection', () => { await Promise.resolve(); }); }; + // The select's menu renders after a frame; the provider list after its fetch settles. + const settle = () => + act( + () => + new Promise((resolve) => { + globalThis.setTimeout(resolve, 50); + }) + ); + // The provider is the section's only select; inside its Field it takes the field's label. + const providerTrigger = () => host.querySelector('[data-scope="select"][data-part="trigger"]'); beforeEach(() => { + api.getExternalProviderConfigs.mockClear(); api.updateModel.mockReset(); onError.mockReset(); onSaved.mockReset(); @@ -173,6 +189,35 @@ describe('ExternalModelSection', () => { expect(checkbox('models.externalModes.txt2img').checked).toBe(true); }); + it('offers the registered providers, keeping an unregistered id of the record selectable', async () => { + await render({ ...gptImage, provider_id: 'custom' }); + await settle(); + + expect(host.querySelector('input[aria-label="models.providerId"]')).toBeNull(); + expect(providerTrigger()?.textContent).toContain('custom'); + await click(providerTrigger()!); + await settle(); + + const options = [...document.querySelectorAll('[role="option"]')]; + expect(options.map((option) => option.textContent)).toEqual(['openai', 'gemini', 'custom']); + await click(options[1]!); + await settle(); + + expect(providerTrigger()?.textContent).toContain('gemini'); + expect(getModelDraft('gpt-image-1')?.external?.fields).toEqual({ providerId: 'gemini' }); + }); + + it('keeps the provider a text field while the provider list is unavailable', async () => { + api.getExternalProviderConfigs.mockRejectedValueOnce(new Error('offline')); + await render(gptImage); + await settle(); + + const providerInput = host.querySelector('input[aria-label="models.providerId"]')!; + expect(providerInput.value).toBe('openai'); + await type(providerInput, 'seedream'); + expect(getModelDraft('gpt-image-1')?.external?.fields).toEqual({ providerId: 'seedream' }); + }); + it('surfaces a failed save and keeps the edits for another attempt', async () => { api.updateModel.mockRejectedValue(new Error('Model not found')); await render(gptImage); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx index 4e958673d48..93b8ab57c64 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx @@ -1,8 +1,28 @@ /* eslint-disable react-perf/jsx-no-jsx-as-prop, react-perf/jsx-no-new-array-as-prop, react-perf/jsx-no-new-function-as-prop, react-perf/jsx-no-new-object-as-prop */ +import type { ExternalModelSettingsModel } from '@features/models/core/externalModelFields'; import type { ExternalModelFormValues } from '@features/models/core/schemas'; -import { Checkbox, Grid, HStack, Input, NumberInput, Stack, Switch, Text } from '@chakra-ui/react'; +import { + Checkbox, + createListCollection, + Grid, + HStack, + Input, + NumberInput, + Stack, + Switch, + Text, +} from '@chakra-ui/react'; +import { + toExternalModelChanges, + toExternalModelValues, + validateExternalModel, +} from '@features/models/core/externalModelFields'; import { updateModel } from '@features/models/data/api'; +import { + ensureExternalProvidersLoaded, + useExternalProvidersSelector, +} from '@features/models/data/externalProvidersStore'; import { replaceModelInStore } from '@features/models/data/modelsStore'; import { applyModelDraftFields, @@ -16,17 +36,16 @@ import { useModelDraft, type ModelDraft, } from '@features/models/ui/modelDraftsStore'; +import { useMountEffect } from '@platform/react/useMountEffect'; import { useScopedAction } from '@platform/react/useScopedAction'; import { assertAccountScopeCurrent } from '@platform/state/accountLifecycle'; import { Button } from '@platform/ui/Button'; import { Field, FieldLabel } from '@platform/ui/Field'; import { Panel } from '@platform/ui/Panel'; +import { Select } from '@platform/ui/Select'; import { useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; -import type { ExternalModelSettingsModel } from './externalModelFields'; - -import { toExternalModelChanges, toExternalModelValues, validateExternalModel } from './externalModelFields'; import { DraftConflictNotice, UnsavedChangesLabel } from './ModelDraftNotices'; /** @@ -120,10 +139,29 @@ export const ExternalModelSection = ({ const values = useMemo(() => applyModelDraftFields(saved, draft), [draft, saved]); const [error, setError] = useState(null); const { run } = useScopedAction(); + const providerConfigs = useExternalProvidersSelector((snapshot) => snapshot.configs); + // The registered providers are what requests can route to; a record's own id stays offered even when it is not + // among them, so an unknown provider reads as what it is rather than as nothing. + const providerCollection = useMemo( + () => + providerConfigs === null + ? null + : createListCollection({ + items: [...new Set([...providerConfigs.map((config) => config.provider_id), values.providerId])] + .filter((id) => id !== '') + .map((id) => ({ label: id, value: id })), + }), + [providerConfigs, values.providerId] + ); // From the draft, not this instance: a save started before the user navigated away may still be in flight. const isSaving = draft?.submitted !== undefined; const isDirty = hasDraftFields(draft); + useMountEffect(() => { + // The list is a convenience: while it is unavailable the provider stays a text field. + ensureExternalProvidersLoaded().catch(() => {}); + }); + const setField = (key: Key, value: ExternalModelFormValues[Key]) => { recordModelDraftFields(model.key, 'external', { ...values, [key]: value }, saved); setError(null); @@ -135,7 +173,7 @@ export const ExternalModelSection = ({ }; const handleSave = async () => { - const validationError = validateExternalModel(values, t); + const validationError = validateExternalModel(values); if (validationError) { setError(validationError); @@ -206,12 +244,28 @@ export const ExternalModelSection = ({ {hasModelDraftConflict(saved, draft) ? : null} - setField('providerId', event.currentTarget.value)} - /> + {providerCollection ? ( + setField('providerId', event.currentTarget.value)} + /> + )} None: + """A capability table or default settings the record sets are returned as stored, not the starter's.""" + config = ExternalApiModelConfig( + key="external_gemini_edited", + name="Gemini Edited", + provider_id="gemini", + provider_model_id="gemini-3.1-flash-image-preview", + capabilities=ExternalModelCapabilities(modes=["txt2img"]), + source="external://gemini/gemini-3.1-flash-image-preview", + ) + mm2_model_manager.store.add_model(config) + + services = type("Services", (), {})() + services.model_manager = mm2_model_manager + services.model_images = DummyModelImages() + services.configuration = mm2_app_config + services.logger = logging.getLogger("test") + + invoker = DummyInvoker(services) + monkeypatch.setattr("invokeai.app.api.routers.model_manager.ApiDependencies", MockApiDependencies(invoker)) + monkeypatch.setattr("invokeai.app.api.auth_dependencies.ApiDependencies", MockApiDependencies(invoker)) + + edited_capabilities = {"modes": ["txt2img", "img2img"], "max_reference_images": 3, "supports_seed": True} + edited_defaults = {"width": 2048, "height": 2048, "num_images": 1} + patch_response = client.patch( + "/api/v2/models/i/external_gemini_edited", + json={"capabilities": edited_capabilities, "default_settings": edited_defaults}, + ) + + assert patch_response.status_code == 200 + patched = patch_response.json() + assert patched["capabilities"]["modes"] == ["txt2img", "img2img"] + assert patched["capabilities"]["max_reference_images"] == 3 + assert patched["capabilities"]["allowed_aspect_ratios"] is None + assert patched["default_settings"] == edited_defaults + + get_response = client.get("/api/v2/models/i/external_gemini_edited") + + assert get_response.status_code == 200 + fetched = get_response.json() + assert fetched["capabilities"] == patched["capabilities"] + assert fetched["default_settings"] == edited_defaults + # Fields the record never set still follow the starter definition. + assert [control["name"] for control in fetched["panel_schema"]["prompts"]] == ["reference_images"] + + def _make_stats_services(ram_caches: dict) -> Any: class _Load: pass diff --git a/tests/app/services/external_generation/test_external_generation_service.py b/tests/app/services/external_generation/test_external_generation_service.py index 2b6ef3b6ca5..ea181a9582d 100644 --- a/tests/app/services/external_generation/test_external_generation_service.py +++ b/tests/app/services/external_generation/test_external_generation_service.py @@ -228,6 +228,55 @@ def test_generate_happy_path() -> None: assert provider.last_request == request +class _RecordStore: + def __init__(self, record: ExternalApiModelConfig) -> None: + self._record = record + + def get_model(self, key: str) -> ExternalApiModelConfig: + assert key == self._record.key + return self._record + + +def _gemini_record(capabilities: ExternalModelCapabilities) -> ExternalApiModelConfig: + return ExternalApiModelConfig( + key="gemini_flash", + name="Gemini Flash", + provider_id="gemini", + provider_model_id="gemini-2.5-flash-image", + capabilities=capabilities, + ) + + +def test_generate_fills_an_unset_starter_record_from_its_definition() -> None: + starter = next(model for model in STARTER_MODELS if model.source == "external://gemini/gemini-2.5-flash-image") + record = _gemini_record(ExternalModelCapabilities()) + provider = DummyProvider("gemini", configured=True, result=ExternalGenerationResult(images=[])) + service = ExternalGenerationService( + {"gemini": provider}, logging.getLogger("test"), record_store=_RecordStore(record) + ) + + service.generate(_build_request(model=record)) + + assert provider.last_request is not None + assert provider.last_request.model.capabilities == starter.capabilities + assert provider.last_request.model.default_settings == starter.default_settings + + +def test_generate_honours_a_starter_record_edited_past_its_definition() -> None: + # The shipped definition is txt2img only; the record was edited to allow img2img. + record = _gemini_record(ExternalModelCapabilities(modes=["txt2img", "img2img"], supports_seed=True)) + provider = DummyProvider("gemini", configured=True, result=ExternalGenerationResult(images=[])) + service = ExternalGenerationService( + {"gemini": provider}, logging.getLogger("test"), record_store=_RecordStore(record) + ) + + service.generate(_build_request(model=record, mode="img2img", init_image=_make_image())) + + assert provider.last_request is not None + assert provider.last_request.model.capabilities.modes == ["txt2img", "img2img"] + assert provider.last_request.model.capabilities.allowed_aspect_ratios is None + + def test_generate_resizes_inpaint_result_to_original_init_size() -> None: model = _build_model(ExternalModelCapabilities(modes=["inpaint"])) request = _build_request( From fffc35b19de07675ccae68d50a4365f1a0760b20 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 11 Oct 2026 03:17:33 -0600 Subject: [PATCH 03/10] fix(models): show starter-managed provider records read-only instead of overriding the catalog The backend keeps overlaying the starter catalog, so starter records save only their provider ids and show capabilities, limits and defaults read-only; custom records keep the full editor. Unit tests follow the pixel-exact external grid and the mode-specific canvas rejection. --- .../Users Guide/03.Models/01.introduction.mdx | 6 +- invokeai/app/api/routers/model_manager.py | 20 +- .../external_generation_default.py | 19 +- .../external_generation/starter_overrides.py | 30 --- .../frontend/webv2/public/locales/en.json | 5 +- .../models/core/externalModelFields.test.ts | 42 ++++- .../models/core/externalModelFields.ts | 29 ++- .../ExternalModelSection.browser.test.tsx | 40 ++++ .../models/ui/detail/ExternalModelSection.tsx | 174 ++++++++++++------ .../features/models/ui/detail/ModelDetail.tsx | 1 + .../workbench/widgets/canvas/bboxGrid.test.ts | 6 +- .../invoke/prepareCanvasInvocation.test.ts | 5 +- tests/app/routers/test_model_manager.py | 48 ----- .../test_external_generation_service.py | 49 ----- 14 files changed, 277 insertions(+), 197 deletions(-) delete mode 100644 invokeai/app/services/external_generation/starter_overrides.py diff --git a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx index caa505307cf..79f031fe038 100644 --- a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx @@ -106,9 +106,9 @@ inputs (seed, reference images, negative prompt) it advertises, its request limits, and the size and image count the provider is asked for by default. The Generate panel and the Canvas only offer what the record advertises, so keep it in step with the provider's documentation. -A starter model's record follows the shipped definition until you save -an edit to its capabilities or defaults; from then on the record's own -values are used. +A model installed from the starter catalog keeps the catalog's +capabilities and defaults, so those are shown read-only; its provider +and provider model ID remain editable. :::note[Gated Downloads] The API Keys section is also used to provide keys for services that diff --git a/invokeai/app/api/routers/model_manager.py b/invokeai/app/api/routers/model_manager.py index e6a00736fad..2cc77dfd5f8 100644 --- a/invokeai/app/api/routers/model_manager.py +++ b/invokeai/app/api/routers/model_manager.py @@ -25,7 +25,6 @@ from invokeai.app.api.auth_dependencies import AdminUserOrDefault, CurrentUserOrDefault from invokeai.app.api.dependencies import ApiDependencies -from invokeai.app.services.external_generation.starter_overrides import apply_external_starter_overrides from invokeai.app.services.model_images.model_images_common import ( ModelImageFileDeleteException, ModelImageFileNotFoundException, @@ -189,11 +188,26 @@ def add_cover_image_to_model_config(config: AnyModelConfig, dependencies: Type[A def apply_external_starter_model_overrides(config: AnyModelConfig) -> AnyModelConfig: - """Fill an installed external model config's unset fields from its starter definition.""" + """Overlay starter-model metadata onto installed external model configs.""" if not isinstance(config, ExternalApiModelConfig): return config - return apply_external_starter_overrides(config) + starter_match = next((starter for starter in STARTER_MODELS if starter.source == config.source), None) + if starter_match is None: + return config + + model_updates: dict[str, object] = {} + if starter_match.capabilities is not None: + model_updates["capabilities"] = starter_match.capabilities + if starter_match.default_settings is not None: + model_updates["default_settings"] = starter_match.default_settings + if starter_match.panel_schema is not None: + model_updates["panel_schema"] = starter_match.panel_schema + + if not model_updates: + return config + + return config.model_copy(update=model_updates) def prepare_model_config_for_response(config: AnyModelConfig, dependencies: Type[ApiDependencies]) -> AnyModelConfig: diff --git a/invokeai/app/services/external_generation/external_generation_default.py b/invokeai/app/services/external_generation/external_generation_default.py index 1e1305a2509..506c0020fdc 100644 --- a/invokeai/app/services/external_generation/external_generation_default.py +++ b/invokeai/app/services/external_generation/external_generation_default.py @@ -26,8 +26,8 @@ ExternalGenerationResult, ExternalProviderStatus, ) -from invokeai.app.services.external_generation.starter_overrides import apply_external_starter_overrides from invokeai.backend.model_manager.configs.external_api import ExternalApiModelConfig, ExternalImageSize +from invokeai.backend.model_manager.starter_models import STARTER_MODELS if TYPE_CHECKING: from invokeai.app.services.model_records import ModelRecordServiceBase @@ -188,7 +188,7 @@ def _refresh_model_capabilities(self, request: ExternalGenerationRequest) -> Ext if record.provider_model_id != request.model.provider_model_id: return request - record = apply_external_starter_overrides(record) + record = _apply_starter_overrides(record) if record == request.model: return request @@ -402,3 +402,18 @@ def _resize_result_images(result: ExternalGenerationResult, width: int, height: provider_metadata=result.provider_metadata, content_filters=result.content_filters, ) + + +def _apply_starter_overrides(model: ExternalApiModelConfig) -> ExternalApiModelConfig: + source = model.source or f"external://{model.provider_id}/{model.provider_model_id}" + starter_match = next((starter for starter in STARTER_MODELS if starter.source == source), None) + if starter_match is None: + return model + updates: dict[str, object] = {} + if starter_match.capabilities is not None: + updates["capabilities"] = starter_match.capabilities + if starter_match.default_settings is not None: + updates["default_settings"] = starter_match.default_settings + if not updates: + return model + return model.model_copy(update=updates) diff --git a/invokeai/app/services/external_generation/starter_overrides.py b/invokeai/app/services/external_generation/starter_overrides.py deleted file mode 100644 index b1a3ddac67f..00000000000 --- a/invokeai/app/services/external_generation/starter_overrides.py +++ /dev/null @@ -1,30 +0,0 @@ -from invokeai.backend.model_manager.configs.external_api import ExternalApiModelConfig, ExternalModelCapabilities -from invokeai.backend.model_manager.starter_models import STARTER_MODELS - -_UNSET_CAPABILITIES = ExternalModelCapabilities() - - -def apply_external_starter_overrides(config: ExternalApiModelConfig) -> ExternalApiModelConfig: - """Fill a starter-sourced record's unset fields from its starter definition. - - A record that never set a field (the default capability matrix, no default settings, no panel schema) - follows the shipped starter definition, so definition updates reach it. A field the record sets, by an - install that copied the definition or by an edit in the model editor, is the record's own and wins. - """ - source = config.source or f"external://{config.provider_id}/{config.provider_model_id}" - starter_match = next((starter for starter in STARTER_MODELS if starter.source == source), None) - if starter_match is None: - return config - - updates: dict[str, object] = {} - if starter_match.capabilities is not None and config.capabilities == _UNSET_CAPABILITIES: - updates["capabilities"] = starter_match.capabilities - if starter_match.default_settings is not None and config.default_settings is None: - updates["default_settings"] = starter_match.default_settings - if starter_match.panel_schema is not None and config.panel_schema is None: - updates["panel_schema"] = starter_match.panel_schema - - if not updates: - return config - - return config.model_copy(update=updates) diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 9346ae161b1..8375d3172bd 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -1362,7 +1362,10 @@ "defaultHeight": "Default height", "defaultNumImages": "Default image count", "unlimited": "No limit", - "providerDefault": "Provider default" + "providerDefault": "Provider default", + "supported": "Supported", + "notSupported": "Not supported", + "starterManaged": "Managed by the starter catalog; the provider IDs can still be edited." }, "queueAttentionCount_one": "{{count}} needs attention", "queueAttentionCount_other": "{{count}} need attention", diff --git a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts index 22b4a53ea48..09f09f8c083 100644 --- a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts @@ -2,7 +2,13 @@ import { describe, expect, it } from 'vitest'; import type { ExternalModelSettingsModel } from './externalModelFields'; -import { toExternalModelChanges, toExternalModelValues, validateExternalModel } from './externalModelFields'; +import { + isStarterManagedExternalModel, + toExternalModelChanges, + toExternalModelValues, + toExternalProviderChanges, + validateExternalModel, +} from './externalModelFields'; const geminiModel: ExternalModelSettingsModel = { capabilities: { @@ -22,6 +28,7 @@ const geminiModel: ExternalModelSettingsModel = { name: 'Gemini 3 Pro', provider_id: 'gemini', provider_model_id: 'gemini-3-pro-image-preview', + source: 'external://gemini/gemini-3-pro-image-preview', type: 'external_image_generator', }; @@ -113,4 +120,37 @@ describe('external model record mapping', () => { ); expect(validateExternalModel({ ...values, defaultWidth: 1024.5 })).toBe('Default width must be a whole number.'); }); + + it('recognises a record the starter catalog manages by its source, or by the source an install would record', () => { + const starters = [ + { source: 'external://openai/gpt-image-1' }, + { source: 'external://gemini/gemini-2.5-flash-image' }, + ]; + + expect( + isStarterManagedExternalModel( + { provider_id: 'openai', provider_model_id: 'renamed', source: 'external://openai/gpt-image-1' }, + starters + ) + ).toBe(true); + expect(isStarterManagedExternalModel({ provider_id: 'openai', provider_model_id: 'gpt-image-1' }, starters)).toBe( + true + ); + expect( + isStarterManagedExternalModel( + { provider_id: 'openai', provider_model_id: 'gpt-image-1', source: 'external://custom/gpt-image-1' }, + starters + ) + ).toBe(false); + expect(isStarterManagedExternalModel({ provider_id: 'custom', provider_model_id: 'mine' }, [])).toBe(false); + }); + + it('sends only the routing ids for a starter-managed record', () => { + const values = toExternalModelValues(geminiModel); + + expect(toExternalProviderChanges({ ...values, providerId: ' gemini ', providerModelId: 'flash ' })).toEqual({ + provider_id: 'gemini', + provider_model_id: 'flash', + }); + }); }); diff --git a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts index bf146f074f1..f1a1760c02c 100644 --- a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts @@ -1,16 +1,33 @@ import type { ExternalModelFormValues } from './schemas'; -import type { ExternalModelCapabilities, ModelConfig, ModelRecordChanges } from './types'; +import type { ExternalModelCapabilities, ModelConfig, ModelRecordChanges, StarterModel } from './types'; import { externalModelSchema } from './schemas'; export type ExternalModelSettingsModel = Pick< ModelConfig, - 'capabilities' | 'default_settings' | 'key' | 'name' | 'provider_id' | 'provider_model_id' | 'type' + 'capabilities' | 'default_settings' | 'key' | 'name' | 'provider_id' | 'provider_model_id' | 'source' | 'type' >; export const supportsExternalModelSettings = (model: Pick): boolean => model.type === 'external_image_generator'; +/** + * Whether the starter catalog manages the record's capabilities and defaults: the backend overlays the catalog's + * values on every response for a record whose source names a starter, so editing them would not persist. Records + * without a source are matched by the source an external install would record. + */ +export const isStarterManagedExternalModel = ( + model: Pick & { source?: string | null }, + starters: readonly Pick[] +): boolean => { + const source = + typeof model.source === 'string' && model.source !== '' + ? model.source + : `external://${model.provider_id ?? ''}/${model.provider_model_id ?? ''}`; + + return starters.some((starter) => starter.source === source); +}; + const toOptionalInteger = (value: number | null | undefined): number | null => typeof value === 'number' && Number.isFinite(value) ? value : null; @@ -75,6 +92,14 @@ export const toExternalModelChanges = ( }; }; +/** The PATCH body for a starter-managed record: only the routing ids persist, so only they are sent. */ +export const toExternalProviderChanges = ( + values: ExternalModelFormValues +): Pick => ({ + provider_id: values.providerId.trim(), + provider_model_id: values.providerModelId.trim(), +}); + /** The first schema message, like the other record editors' messages, so the UI shows it as is. */ export const validateExternalModel = (values: ExternalModelFormValues): string | null => { const result = externalModelSchema.safeParse(values); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx index 51bc8a8222d..ba0f425bbd4 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx @@ -19,6 +19,12 @@ const api = vi.hoisted(() => ({ ]) ), getModelsDir: vi.fn(() => Promise.resolve('')), + getStarterModels: vi.fn(() => + Promise.resolve({ + starter_bundles: {}, + starter_models: [{ source: 'external://gemini/gemini-2.5-flash-image' }], + }) + ), listMissingModels: vi.fn(() => Promise.resolve([])), listModels: vi.fn(() => Promise.resolve([])), updateModel: vi.fn(), @@ -44,6 +50,7 @@ const gptImage: ExternalModelSettingsModel = { name: 'GPT Image 1', provider_id: 'openai', provider_model_id: 'gpt-image-1', + source: 'external://openai/gpt-image-1', type: 'external_image_generator', }; @@ -60,6 +67,7 @@ describe('ExternalModelSection', () => { const onError = vi.fn(); const onSaved = vi.fn(); + // The editor is read-only until the starter catalog has answered, so every render waits for it. const render = async (model: ExternalModelSettingsModel) => { await act(() => { root.render( @@ -68,6 +76,7 @@ describe('ExternalModelSection', () => { ); }); + await settle(); }; const input = (label: string) => host.querySelector(`input[aria-label="${label}"]`)!; @@ -218,6 +227,37 @@ describe('ExternalModelSection', () => { expect(getModelDraft('gpt-image-1')?.external?.fields).toEqual({ providerId: 'seedream' }); }); + it('shows a starter-managed record read-only and saves only its routing ids', async () => { + const geminiFlash: ExternalModelSettingsModel = { + ...gptImage, + capabilities: { max_images_per_request: 1, modes: ['txt2img'], supports_reference_images: true }, + key: 'gemini-flash', + provider_id: 'gemini', + provider_model_id: 'gemini-2.5-flash-image', + source: 'external://gemini/gemini-2.5-flash-image', + }; + api.updateModel.mockImplementation((_key: string, changes: Record) => + Promise.resolve({ ...geminiFlash, ...changes }) + ); + await render(geminiFlash); + + expect(host.querySelector('[role="note"]')?.textContent).toBe('models.externalFields.starterManaged'); + expect(host.querySelectorAll('input[type="checkbox"]')).toHaveLength(0); + expect(host.querySelector('input[aria-label="models.externalFields.maxImagesPerRequest"]')).toBeNull(); + expect(host.textContent).toContain('models.externalModes.txt2img'); + expect(host.textContent).toContain('1024'); + + await type(host.querySelector('input[value="gemini-2.5-flash-image"]')!, 'gemini-3-pro'); + await click(button('models.saveExternalSettings')); + + expect(api.updateModel).toHaveBeenCalledWith( + 'gemini-flash', + { provider_id: 'gemini', provider_model_id: 'gemini-3-pro' }, + expect.anything() + ); + expect(onSaved).toHaveBeenCalledOnce(); + }); + it('surfaces a failed save and keeps the edits for another attempt', async () => { api.updateModel.mockRejectedValue(new Error('Model not found')); await render(gptImage); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx index 93b8ab57c64..0d500576c10 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx @@ -14,8 +14,10 @@ import { Text, } from '@chakra-ui/react'; import { + isStarterManagedExternalModel, toExternalModelChanges, toExternalModelValues, + toExternalProviderChanges, validateExternalModel, } from '@features/models/core/externalModelFields'; import { updateModel } from '@features/models/data/api'; @@ -24,6 +26,7 @@ import { useExternalProvidersSelector, } from '@features/models/data/externalProvidersStore'; import { replaceModelInStore } from '@features/models/data/modelsStore'; +import { ensureStartersLoaded, useStartersSelector } from '@features/models/data/startersStore'; import { applyModelDraftFields, beginModelDraftSave, @@ -50,8 +53,10 @@ import { DraftConflictNotice, UnsavedChangesLabel } from './ModelDraftNotices'; /** * Edits the hosted-provider record: which provider and model ID requests route to, the modes and inputs the model - * advertises, its limits, and the size/count the provider is asked for by default. Unsaved values live in the - * model's retained draft; `externalModelFields` owns the record mapping and validation. + * advertises, its limits, and the size/count the provider is asked for by default. A record the starter catalog + * manages shows its capabilities and defaults read-only, since the backend reapplies the catalog's on every + * response; its routing ids still persist. Unsaved values live in the model's retained draft; `externalModelFields` + * owns the record mapping and validation. */ const selectExternalDraft = (draft: ModelDraft | undefined) => draft?.external; @@ -88,6 +93,15 @@ const DEFAULT_FIELDS: readonly { key: OptionalNumberKey; labelKey: string }[] = { key: 'defaultNumImages', labelKey: 'models.externalFields.defaultNumImages' }, ]; +const ReadOnlyValue = ({ label, value }: { label: string; value: string }) => ( + + + {label} + + {value} + +); + const OptionalNumberField = ({ disabled, label, @@ -140,6 +154,12 @@ export const ExternalModelSection = ({ const [error, setError] = useState(null); const { run } = useScopedAction(); const providerConfigs = useExternalProvidersSelector((snapshot) => snapshot.configs); + const starters = useStartersSelector((snapshot) => snapshot.response?.starter_models ?? null); + const startersStatus = useStartersSelector((snapshot) => snapshot.status); + // Read-only until the catalog says otherwise, so an edit never flips under the user; a catalog that cannot be + // loaded lets the editor through rather than locking custom records. + const isStarterManaged = + starters === null ? startersStatus !== 'error' : isStarterManagedExternalModel(model, starters); // The registered providers are what requests can route to; a record's own id stays offered even when it is not // among them, so an unknown provider reads as what it is rather than as nothing. const providerCollection = useMemo( @@ -158,6 +178,7 @@ export const ExternalModelSection = ({ const isDirty = hasDraftFields(draft); useMountEffect(() => { + ensureStartersLoaded(); // The list is a convenience: while it is unavailable the provider stays a text field. ensureExternalProvidersLoaded().catch(() => {}); }); @@ -188,7 +209,11 @@ export const ExternalModelSection = ({ return; } - const updated = await updateModel(model.key, toExternalModelChanges(submitted, model), owner.signal); + const updated = await updateModel( + model.key, + isStarterManaged ? toExternalProviderChanges(submitted) : toExternalModelChanges(submitted, model), + owner.signal + ); assertAccountScopeCurrent(owner); replaceModelInStore(updated); @@ -276,59 +301,100 @@ export const ExternalModelSection = ({ /> - - - {t('models.externalFields.supportedModes')} - - - {MODE_FLAGS.map(({ key, labelKey }) => ( - setField(key, event.checked === true)} - > - - - {t(labelKey)} - - ))} - - - {t('models.externalFields.supportedModesHelp')} - - - - {SUPPORT_FLAGS.map(({ key, labelKey }) => ( - - - - {t(labelKey)} - - setField(key, event.checked)} - > - - - - - {t(labelKey)} - + {isStarterManaged ? ( + + {starters === null ? null : ( + + {t('models.externalFields.starterManaged')} + + )} + + saved[key]) + .map(({ labelKey }) => t(labelKey)) + .join(', ')} + /> + {SUPPORT_FLAGS.map(({ key, labelKey }) => ( + + ))} + {LIMIT_FIELDS.map(({ key, labelKey }) => ( + + ))} + {DEFAULT_FIELDS.map(({ key, labelKey }) => ( + + ))} + + + ) : ( + <> + + + {t('models.externalFields.supportedModes')} + + + {MODE_FLAGS.map(({ key, labelKey }) => ( + setField(key, event.checked === true)} + > + + + {t(labelKey)} + + ))} + + {t('models.externalFields.supportedModesHelp')} + - ))} - - - {LIMIT_FIELDS.map((field) => numberField(field, 'models.externalFields.unlimited'))} - - - {DEFAULT_FIELDS.map((field) => numberField(field, 'models.externalFields.providerDefault'))} - + + {SUPPORT_FLAGS.map(({ key, labelKey }) => ( + + + + {t(labelKey)} + + setField(key, event.checked)} + > + + + + + {t(labelKey)} + + + + ))} + + + {LIMIT_FIELDS.map((field) => numberField(field, 'models.externalFields.unlimited'))} + + + {DEFAULT_FIELDS.map((field) => numberField(field, 'models.externalFields.providerDefault'))} + + + )} ); }; diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx index 833ac871d36..85c1d6ccc2d 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ModelDetail.tsx @@ -138,6 +138,7 @@ const selectExternalModel = (snapshot: ModelsSnapshot, modelKey: string): Extern name: model.name, provider_id: model.provider_id, provider_model_id: model.provider_model_id, + source: model.source, type: model.type, } : null; diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/bboxGrid.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/bboxGrid.test.ts index 1dddb0ec5d0..c549ecc0b27 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/bboxGrid.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/bboxGrid.test.ts @@ -65,8 +65,10 @@ describe('resolveModelGrid before the capability table arrives', () => { it('answers for an external generator too, which will never get a row to wait for', () => { // A writer holds off on `null`, so answering it here would stop the bbox <-> dims sync for - // external providers for the whole session. - expect(resolveModelGrid('external')).toBe(DEFAULT_MODEL_GRID); + // external providers for the whole session. Providers bucket to their own size tables, so the + // bbox is pixel-exact rather than snapped to a local model grid. + expect(resolveModelGrid('external')).toBe(1); + expect(gridSizeForModelBase('external')).toBe(1); }); it('keeps the default for readers that must show something', () => { diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts index d157fd3f2f4..44d8b2cda8f 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts @@ -780,7 +780,8 @@ describe('runCanvasInvocation', () => { ); it('records a notice and dispatches no snapshot when the graph fails validation', async () => { - // External image generators are rejected by the graph compiler. + // The compiler rejects a hosted provider asked for a canvas mode its record does not advertise: the raster + // snapshot makes this an image-to-image run, and the record lists text-to-image only. const harness = makeHarness({ model: externalModel }); await runCanvasInvocation(harness.deps); @@ -788,7 +789,7 @@ describe('runCanvasInvocation', () => { expect(harness.submittedGraphs()).toHaveLength(0); const notices = harness.notices(); expect(notices).toHaveLength(1); - expect(notices[0]?.message).toContain('does not support canvas generation'); + expect(notices[0]?.message).toBe('OpenAI Image does not support image-to-image generation.'); expect(harness.dedupe.byKey).toHaveLength(0); expect(harness.dedupe.byHash).toHaveLength(0); expect(harness.releaseRasterSnapshot).toHaveBeenCalledTimes(1); diff --git a/tests/app/routers/test_model_manager.py b/tests/app/routers/test_model_manager.py index ab1a70ab9d1..90dbed3968b 100644 --- a/tests/app/routers/test_model_manager.py +++ b/tests/app/routers/test_model_manager.py @@ -194,54 +194,6 @@ def test_model_manager_gemini_starter_model_applies_reference_and_resolution_ove ] -def test_model_manager_external_record_edits_win_over_starter_overrides( - monkeypatch: Any, client: TestClient, mm2_model_manager: Any, mm2_app_config: Any -) -> None: - """A capability table or default settings the record sets are returned as stored, not the starter's.""" - config = ExternalApiModelConfig( - key="external_gemini_edited", - name="Gemini Edited", - provider_id="gemini", - provider_model_id="gemini-3.1-flash-image-preview", - capabilities=ExternalModelCapabilities(modes=["txt2img"]), - source="external://gemini/gemini-3.1-flash-image-preview", - ) - mm2_model_manager.store.add_model(config) - - services = type("Services", (), {})() - services.model_manager = mm2_model_manager - services.model_images = DummyModelImages() - services.configuration = mm2_app_config - services.logger = logging.getLogger("test") - - invoker = DummyInvoker(services) - monkeypatch.setattr("invokeai.app.api.routers.model_manager.ApiDependencies", MockApiDependencies(invoker)) - monkeypatch.setattr("invokeai.app.api.auth_dependencies.ApiDependencies", MockApiDependencies(invoker)) - - edited_capabilities = {"modes": ["txt2img", "img2img"], "max_reference_images": 3, "supports_seed": True} - edited_defaults = {"width": 2048, "height": 2048, "num_images": 1} - patch_response = client.patch( - "/api/v2/models/i/external_gemini_edited", - json={"capabilities": edited_capabilities, "default_settings": edited_defaults}, - ) - - assert patch_response.status_code == 200 - patched = patch_response.json() - assert patched["capabilities"]["modes"] == ["txt2img", "img2img"] - assert patched["capabilities"]["max_reference_images"] == 3 - assert patched["capabilities"]["allowed_aspect_ratios"] is None - assert patched["default_settings"] == edited_defaults - - get_response = client.get("/api/v2/models/i/external_gemini_edited") - - assert get_response.status_code == 200 - fetched = get_response.json() - assert fetched["capabilities"] == patched["capabilities"] - assert fetched["default_settings"] == edited_defaults - # Fields the record never set still follow the starter definition. - assert [control["name"] for control in fetched["panel_schema"]["prompts"]] == ["reference_images"] - - def _make_stats_services(ram_caches: dict) -> Any: class _Load: pass diff --git a/tests/app/services/external_generation/test_external_generation_service.py b/tests/app/services/external_generation/test_external_generation_service.py index ea181a9582d..2b6ef3b6ca5 100644 --- a/tests/app/services/external_generation/test_external_generation_service.py +++ b/tests/app/services/external_generation/test_external_generation_service.py @@ -228,55 +228,6 @@ def test_generate_happy_path() -> None: assert provider.last_request == request -class _RecordStore: - def __init__(self, record: ExternalApiModelConfig) -> None: - self._record = record - - def get_model(self, key: str) -> ExternalApiModelConfig: - assert key == self._record.key - return self._record - - -def _gemini_record(capabilities: ExternalModelCapabilities) -> ExternalApiModelConfig: - return ExternalApiModelConfig( - key="gemini_flash", - name="Gemini Flash", - provider_id="gemini", - provider_model_id="gemini-2.5-flash-image", - capabilities=capabilities, - ) - - -def test_generate_fills_an_unset_starter_record_from_its_definition() -> None: - starter = next(model for model in STARTER_MODELS if model.source == "external://gemini/gemini-2.5-flash-image") - record = _gemini_record(ExternalModelCapabilities()) - provider = DummyProvider("gemini", configured=True, result=ExternalGenerationResult(images=[])) - service = ExternalGenerationService( - {"gemini": provider}, logging.getLogger("test"), record_store=_RecordStore(record) - ) - - service.generate(_build_request(model=record)) - - assert provider.last_request is not None - assert provider.last_request.model.capabilities == starter.capabilities - assert provider.last_request.model.default_settings == starter.default_settings - - -def test_generate_honours_a_starter_record_edited_past_its_definition() -> None: - # The shipped definition is txt2img only; the record was edited to allow img2img. - record = _gemini_record(ExternalModelCapabilities(modes=["txt2img", "img2img"], supports_seed=True)) - provider = DummyProvider("gemini", configured=True, result=ExternalGenerationResult(images=[])) - service = ExternalGenerationService( - {"gemini": provider}, logging.getLogger("test"), record_store=_RecordStore(record) - ) - - service.generate(_build_request(model=record, mode="img2img", init_image=_make_image())) - - assert provider.last_request is not None - assert provider.last_request.model.capabilities.modes == ["txt2img", "img2img"] - assert provider.last_request.model.capabilities.allowed_aspect_ratios is None - - def test_generate_resizes_inpaint_result_to_original_init_size() -> None: model = _build_model(ExternalModelCapabilities(modes=["inpaint"])) request = _build_request( From 57273a4f5551ea53f9228735b67c58491a875b56 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 11 Oct 2026 03:21:37 -0600 Subject: [PATCH 04/10] chore(perf): re-record route baselines for the hosted-provider modules Request counts and chunk sets are unchanged; the provider core joins the editor's generation sources and the provider sections join the Generate widget's. --- .../performance/architecture-baseline.json | 23 +- .../webv2/performance/browser-baseline.json | 225 +++++++++--------- 2 files changed, 129 insertions(+), 119 deletions(-) diff --git a/invokeai/frontend/webv2/performance/architecture-baseline.json b/invokeai/frontend/webv2/performance/architecture-baseline.json index e8a49c4183f..eca2898b573 100644 --- a/invokeai/frontend/webv2/performance/architecture-baseline.json +++ b/invokeai/frontend/webv2/performance/architecture-baseline.json @@ -2,15 +2,15 @@ "build": { "launchpad": { "baseline": { - "brotliBytes": 794977, + "brotliBytes": 795865, "cssRawBytes": 2159, "fontRawBytes": 219480, - "gzipBytes": 793689, + "gzipBytes": 794662, "imageRawBytes": 44398, - "initialRawBytes": 2042114, - "largestAssetRawBytes": 749029, + "initialRawBytes": 2046539, + "largestAssetRawBytes": 752421, "otherAssetRawBytes": 0, - "ownedRawBytes": 98621, + "ownedRawBytes": 98845, "requestCount": 35, "scriptRequestCount": 19, "sourceOwners": [ @@ -346,15 +346,15 @@ }, "editor": { "baseline": { - "brotliBytes": 1171873, + "brotliBytes": 1175651, "cssRawBytes": 2159, "fontRawBytes": 219480, - "gzipBytes": 1165072, + "gzipBytes": 1168861, "imageRawBytes": 44398, - "initialRawBytes": 3208701, - "largestAssetRawBytes": 749029, + "initialRawBytes": 3222002, + "largestAssetRawBytes": 752421, "otherAssetRawBytes": 0, - "ownedRawBytes": 150716, + "ownedRawBytes": 151795, "requestCount": 72, "scriptRequestCount": 56, "sourceOwners": [ @@ -550,6 +550,7 @@ "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", "source:src/features/generation/core/dynamicPrompts.ts", + "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", "source:src/features/generation/core/graphBuilder.ts", "source:src/features/generation/core/krea2Conditioning.ts", @@ -1092,7 +1093,7 @@ "source": "src/app/WorkbenchApp.tsx" } }, - "capturedAt": "2026-10-09", + "capturedAt": "2026-10-11", "developmentInvalidation": { "platformUiBarrel": { "specifier": "@platform/ui", diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index 99f91f948c9..34d8535735e 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -1,6 +1,6 @@ { "browserExecutable": "/home/toast/.cache/ms-playwright/chromium-1243/chrome-linux64/chrome", - "capturedAt": "2026-10-09", + "capturedAt": "2026-10-11", "routes": [ { "activatedResourceBaseline": { @@ -14,13 +14,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 130.60000000149012, + "domContentLoadedMedianMs": 159.79999999701977, "id": "launchpad", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 130.79999999701977, - "longestTaskMaxMs": 82, + "loadMedianMs": 159.89999999850988, + "longestTaskMaxMs": 104, "owner": "app", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:launchpad", @@ -29,14 +29,14 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 0, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 30, - "scriptRawBytes": 1842561, + "scriptRawBytes": 1847465, "scriptRequestCount": 26, - "totalRawBytes": 2068864 + "totalRawBytes": 2078043 }, - "routeReadyMedianMs": 275.80000000447035, + "routeReadyMedianMs": 334.79999999701977, "scriptSourceOwnerSet": "launchpad-static", "stateProfile": "empty" }, @@ -52,13 +52,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 125.10000000149012, + "domContentLoadedMedianMs": 167.5, "id": "launchpad", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 125.20000000298023, - "longestTaskMaxMs": 81, + "loadMedianMs": 167.70000000298023, + "longestTaskMaxMs": 105, "owner": "app", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:launchpad", @@ -67,14 +67,14 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 0, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 30, - "scriptRawBytes": 1842561, + "scriptRawBytes": 1847465, "scriptRequestCount": 26, - "totalRawBytes": 2068864 + "totalRawBytes": 2078043 }, - "routeReadyMedianMs": 277, + "routeReadyMedianMs": 353.1000000014901, "scriptSourceOwnerSet": "launchpad-static", "stateProfile": "representative" }, @@ -90,29 +90,29 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 125.5, + "domContentLoadedMedianMs": 156.20000000298023, "id": "editor-minimal", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 125.70000000298023, - "longestTaskMaxMs": 173, + "loadMedianMs": 156.39999999850988, + "longestTaskMaxMs": 240, "owner": "workbench", - "projectSwitchMedianMs": 38.30000000447035, + "projectSwitchMedianMs": 55.19999999552965, "readyMark": "invokeai:ready:widget:center:preview", "remediationTicket": "deepen-widget-registry-loading", "resourceBaseline": { "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 88, - "scriptRawBytes": 4063609, + "scriptRawBytes": 4098968, "scriptRequestCount": 83, - "totalRawBytes": 4334310 + "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 430.19999999552965, + "routeReadyMedianMs": 549.5, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "empty" }, @@ -128,29 +128,29 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 129.60000000149012, + "domContentLoadedMedianMs": 152.60000000149012, "id": "editor-minimal", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 129.69999999552965, - "longestTaskMaxMs": 180, + "loadMedianMs": 152.79999999701977, + "longestTaskMaxMs": 232, "owner": "workbench", - "projectSwitchMedianMs": 44.099999994039536, + "projectSwitchMedianMs": 58.899999998509884, "readyMark": "invokeai:ready:widget:center:preview", "remediationTicket": "deepen-widget-registry-loading", "resourceBaseline": { "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 88, - "scriptRawBytes": 4063609, + "scriptRawBytes": 4098968, "scriptRequestCount": 83, - "totalRawBytes": 4334310 + "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 466.3999999985099, + "routeReadyMedianMs": 569, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "representative" }, @@ -159,20 +159,20 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 305162, + "largestAssetRawBytes": 305163, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 514518, + "scriptRawBytes": 515192, "scriptRequestCount": 9, - "totalRawBytes": 514518 + "totalRawBytes": 515192 }, - "domContentLoadedMedianMs": 125.5, + "domContentLoadedMedianMs": 146.89999999850988, "id": "editor-canvas", - "layoutAckMedianMs": 2.6999999955296516, - "layoutReturnSwitchMedianMs": 57.899999998509884, - "layoutSwitchMedianMs": 76.60000000149012, - "loadMedianMs": 125.60000000149012, - "longestTaskMaxMs": 179, + "layoutAckMedianMs": 3.399999998509884, + "layoutReturnSwitchMedianMs": 65.5, + "layoutSwitchMedianMs": 89.30000000447035, + "loadMedianMs": 147.09999999403954, + "longestTaskMaxMs": 211, "owner": "canvas", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:canvas", @@ -181,14 +181,14 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 97, - "scriptRawBytes": 4578127, + "scriptRawBytes": 4614160, "scriptRequestCount": 92, - "totalRawBytes": 4848828 + "totalRawBytes": 4889136 }, - "routeReadyMedianMs": 1153.7999999970198, + "routeReadyMedianMs": 1295.3999999985099, "scriptSourceOwnerSet": "editor-canvas-static", "stateProfile": "empty" }, @@ -197,20 +197,20 @@ "cssRawBytes": 0, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 305162, + "largestAssetRawBytes": 305163, "otherRawBytes": 0, "requestCount": 9, - "scriptRawBytes": 514518, + "scriptRawBytes": 515192, "scriptRequestCount": 9, - "totalRawBytes": 514518 + "totalRawBytes": 515192 }, - "domContentLoadedMedianMs": 126.59999999403954, + "domContentLoadedMedianMs": 126.19999999552965, "id": "editor-canvas", "layoutAckMedianMs": 3.100000001490116, - "layoutReturnSwitchMedianMs": 52.70000000298023, - "layoutSwitchMedianMs": 72.20000000298023, - "loadMedianMs": 126.69999999552965, - "longestTaskMaxMs": 196, + "layoutReturnSwitchMedianMs": 53.899999998509884, + "layoutSwitchMedianMs": 78.5, + "loadMedianMs": 126.29999999701977, + "longestTaskMaxMs": 205, "owner": "canvas", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:canvas", @@ -219,14 +219,14 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 97, - "scriptRawBytes": 4578127, + "scriptRawBytes": 4614160, "scriptRequestCount": 92, - "totalRawBytes": 4848828 + "totalRawBytes": 4889136 }, - "routeReadyMedianMs": 1147.1000000014901, + "routeReadyMedianMs": 1172, "scriptSourceOwnerSet": "editor-canvas-static", "stateProfile": "representative" }, @@ -235,20 +235,20 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 187094, + "largestAssetRawBytes": 187100, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 420919, + "scriptRawBytes": 420925, "scriptRequestCount": 6, - "totalRawBytes": 436332 + "totalRawBytes": 436338 }, - "domContentLoadedMedianMs": 125, + "domContentLoadedMedianMs": 126.89999999850988, "id": "editor-workflow", - "layoutAckMedianMs": 2.7000000029802322, - "layoutReturnSwitchMedianMs": 38.600000001490116, - "layoutSwitchMedianMs": 82.60000000149012, - "loadMedianMs": 125.09999999403954, - "longestTaskMaxMs": 174, + "layoutAckMedianMs": 3, + "layoutReturnSwitchMedianMs": 39.600000001490116, + "layoutSwitchMedianMs": 74.90000000596046, + "loadMedianMs": 127, + "longestTaskMaxMs": 172, "owner": "workflow", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:workflow", @@ -257,14 +257,14 @@ "cssRawBytes": 17572, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 95, - "scriptRawBytes": 4484528, + "scriptRawBytes": 4519893, "scriptRequestCount": 89, - "totalRawBytes": 4770642 + "totalRawBytes": 4810282 }, - "routeReadyMedianMs": 1171.6000000014901, + "routeReadyMedianMs": 1176.7999999970198, "scriptSourceOwnerSet": "editor-workflow-static", "stateProfile": "empty" }, @@ -273,20 +273,20 @@ "cssRawBytes": 15413, "fontRawBytes": 0, "imageRawBytes": 0, - "largestAssetRawBytes": 187094, + "largestAssetRawBytes": 187100, "otherRawBytes": 0, "requestCount": 7, - "scriptRawBytes": 420919, + "scriptRawBytes": 420925, "scriptRequestCount": 6, - "totalRawBytes": 436332 + "totalRawBytes": 436338 }, - "domContentLoadedMedianMs": 123.70000000298023, + "domContentLoadedMedianMs": 125, "id": "editor-workflow", - "layoutAckMedianMs": 2.6999999955296516, - "layoutReturnSwitchMedianMs": 41.80000000447035, - "layoutSwitchMedianMs": 128.80000000447035, - "loadMedianMs": 123.80000000447035, - "longestTaskMaxMs": 259, + "layoutAckMedianMs": 2.899999998509884, + "layoutReturnSwitchMedianMs": 43.100000001490116, + "layoutSwitchMedianMs": 132.20000000298023, + "loadMedianMs": 125, + "longestTaskMaxMs": 276, "owner": "workflow", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:workflow", @@ -295,14 +295,14 @@ "cssRawBytes": 17572, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 95, - "scriptRawBytes": 4484528, + "scriptRawBytes": 4519893, "scriptRequestCount": 89, - "totalRawBytes": 4770642 + "totalRawBytes": 4810282 }, - "routeReadyMedianMs": 1192.8999999985099, + "routeReadyMedianMs": 1216.2000000029802, "scriptSourceOwnerSet": "editor-workflow-static", "stateProfile": "representative" }, @@ -318,13 +318,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 125.79999999701977, + "domContentLoadedMedianMs": 126.59999999403954, "id": "editor-gallery", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, - "layoutSwitchMedianMs": 41.100000001490116, - "loadMedianMs": 125.89999999850988, - "longestTaskMaxMs": 174, + "layoutSwitchMedianMs": 39.79999999701977, + "loadMedianMs": 126.59999999403954, + "longestTaskMaxMs": 178, "owner": "gallery", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:gallery", @@ -333,14 +333,14 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 88, - "scriptRawBytes": 4063609, + "scriptRawBytes": 4098968, "scriptRequestCount": 83, - "totalRawBytes": 4334310 + "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 1431.3999999985099, + "routeReadyMedianMs": 1420.6000000014901, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "empty" }, @@ -356,13 +356,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 126.89999999850988, + "domContentLoadedMedianMs": 126.20000000298023, "id": "editor-gallery", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, - "layoutSwitchMedianMs": 45.399999998509884, - "loadMedianMs": 127.09999999403954, - "longestTaskMaxMs": 170, + "layoutSwitchMedianMs": 46.80000000447035, + "loadMedianMs": 126.39999999850988, + "longestTaskMaxMs": 171, "owner": "gallery", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:gallery", @@ -371,14 +371,14 @@ "cssRawBytes": 2159, "fontRawBytes": 23664, "imageRawBytes": 44398, - "largestAssetRawBytes": 749029, - "otherRawBytes": 200480, + "largestAssetRawBytes": 752421, + "otherRawBytes": 204755, "requestCount": 88, - "scriptRawBytes": 4063609, + "scriptRawBytes": 4098968, "scriptRequestCount": 83, - "totalRawBytes": 4334310 + "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 1396.3999999985099, + "routeReadyMedianMs": 1411.6000000014901, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "representative" } @@ -974,6 +974,7 @@ "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", "source:src/features/generation/core/dynamicPrompts.ts", + "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", "source:src/features/generation/core/graphBuilder.ts", "source:src/features/generation/core/krea2Conditioning.ts", @@ -1019,6 +1020,7 @@ "source:src/features/generation/ui/GenerateHeaderActions.tsx", "source:src/features/generation/ui/GenerateModelCard.tsx", "source:src/features/generation/ui/GeneratePresetsPopover.tsx", + "source:src/features/generation/ui/GenerateProviderOptionsSection.tsx", "source:src/features/generation/ui/GenerateRenderSection.tsx", "source:src/features/generation/ui/GenerateSettingsForm.tsx", "source:src/features/generation/ui/GenerateWidgetView.tsx", @@ -1070,6 +1072,7 @@ "source:src/features/generation/ui/shared/ConceptRow.tsx", "source:src/features/generation/ui/shared/ConditioningRebalanceBars.tsx", "source:src/features/generation/ui/shared/ConditioningRebalanceSparkline.tsx", + "source:src/features/generation/ui/shared/ExternalSizeSelect.tsx", "source:src/features/generation/ui/shared/GenerateCollapsibleSection.tsx", "source:src/features/generation/ui/shared/GenerateFieldContextMenu.tsx", "source:src/features/generation/ui/shared/GenerateToggleSwitch.tsx", @@ -2061,6 +2064,7 @@ "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", "source:src/features/generation/core/dynamicPrompts.ts", + "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", "source:src/features/generation/core/graphBuilder.ts", "source:src/features/generation/core/krea2Conditioning.ts", @@ -2106,6 +2110,7 @@ "source:src/features/generation/ui/GenerateHeaderActions.tsx", "source:src/features/generation/ui/GenerateModelCard.tsx", "source:src/features/generation/ui/GeneratePresetsPopover.tsx", + "source:src/features/generation/ui/GenerateProviderOptionsSection.tsx", "source:src/features/generation/ui/GenerateRenderSection.tsx", "source:src/features/generation/ui/GenerateSettingsForm.tsx", "source:src/features/generation/ui/GenerateWidgetView.tsx", @@ -2157,6 +2162,7 @@ "source:src/features/generation/ui/shared/ConceptRow.tsx", "source:src/features/generation/ui/shared/ConditioningRebalanceBars.tsx", "source:src/features/generation/ui/shared/ConditioningRebalanceSparkline.tsx", + "source:src/features/generation/ui/shared/ExternalSizeSelect.tsx", "source:src/features/generation/ui/shared/GenerateCollapsibleSection.tsx", "source:src/features/generation/ui/shared/GenerateFieldContextMenu.tsx", "source:src/features/generation/ui/shared/GenerateToggleSwitch.tsx", @@ -3307,6 +3313,7 @@ "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", "source:src/features/generation/core/dynamicPrompts.ts", + "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", "source:src/features/generation/core/graphBuilder.ts", "source:src/features/generation/core/krea2Conditioning.ts", @@ -3352,6 +3359,7 @@ "source:src/features/generation/ui/GenerateHeaderActions.tsx", "source:src/features/generation/ui/GenerateModelCard.tsx", "source:src/features/generation/ui/GeneratePresetsPopover.tsx", + "source:src/features/generation/ui/GenerateProviderOptionsSection.tsx", "source:src/features/generation/ui/GenerateRenderSection.tsx", "source:src/features/generation/ui/GenerateSettingsForm.tsx", "source:src/features/generation/ui/GenerateWidgetView.tsx", @@ -3403,6 +3411,7 @@ "source:src/features/generation/ui/shared/ConceptRow.tsx", "source:src/features/generation/ui/shared/ConditioningRebalanceBars.tsx", "source:src/features/generation/ui/shared/ConditioningRebalanceSparkline.tsx", + "source:src/features/generation/ui/shared/ExternalSizeSelect.tsx", "source:src/features/generation/ui/shared/GenerateCollapsibleSection.tsx", "source:src/features/generation/ui/shared/GenerateFieldContextMenu.tsx", "source:src/features/generation/ui/shared/GenerateToggleSwitch.tsx", From 7e4ae9b5d78294f7025795cf1c776598e82b655e Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sat, 17 Oct 2026 02:36:30 -0600 Subject: [PATCH 05/10] fix(canvas): restore hosted-provider output to the bounding box Canvas graphs for external models iterate the provider's image collection into an img_resize back to the bbox, carrying the provider's output metadata; inpaint is refused on Canvas because the keep-mask has no provider contract. Readiness and preparation validate the bbox, max_image_size, the Seedream image budget, and name unsupported layers; model switches select the nearest provider preset, typed preset sizes resolve their name, and recall resets omitted provider defaults and restores a named size only with unambiguous dimensions. --- invokeai/frontend/webv2/.oxfmtrc.json | 1 + .../__snapshots__/externalCanvasGraphs.json | 424 ++++++++++++++++++ .../core/baseGenerationPolicies.test.ts | 126 ++++++ .../generation/core/baseGenerationPolicies.ts | 61 +++ .../core/canvas/canvasProcessingSize.ts | 2 +- .../core/canvas/compileCanvasGraph.test.ts | 106 +++-- .../core/canvas/compileCanvasGraph.ts | 52 ++- .../generation/core/externalProvider.test.ts | 25 +- .../generation/core/externalProvider.ts | 38 +- .../src/features/generation/core/graph.ts | 12 +- .../features/generation/core/settings.test.ts | 10 + .../GenerateDimensionFields.browser.test.tsx | 60 +++ .../ExternalSizeSelect.browser.test.tsx | 6 + .../image-actions/imageRecall.test.ts | 107 ++++- .../workbench/image-actions/imageRecall.ts | 40 +- .../webv2/src/workbench/invocation.test.ts | 45 ++ .../webv2/src/workbench/invocation.ts | 39 +- .../invoke/prepareCanvasInvocation.test.ts | 85 ++++ .../canvas/invoke/prepareCanvasInvocation.ts | 16 +- .../test_frontend_external_canvas_graphs.py | 58 +++ 20 files changed, 1230 insertions(+), 83 deletions(-) create mode 100644 invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json create mode 100644 tests/app/invocations/test_frontend_external_canvas_graphs.py diff --git a/invokeai/frontend/webv2/.oxfmtrc.json b/invokeai/frontend/webv2/.oxfmtrc.json index b4548ee4cab..534184baaa0 100644 --- a/invokeai/frontend/webv2/.oxfmtrc.json +++ b/invokeai/frontend/webv2/.oxfmtrc.json @@ -10,6 +10,7 @@ "stats.html", "src/features/generation/core/__snapshots__/generateGraphNodeTypes.json", "src/features/generation/core/__snapshots__/animaLLLiteCanvasGraphs.json", + "src/features/generation/core/__snapshots__/externalCanvasGraphs.json", "src/features/video/core/__snapshots__/videoGraphNodeTypes.json", "src/features/upscale/core/__snapshots__/upscaleGraphNodeTypes.json", "src/features/upscale/core/__snapshots__/upscaleIndexClippedStarts.json", diff --git a/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json new file mode 100644 index 00000000000..4a4253cf5f3 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json @@ -0,0 +1,424 @@ +{ + "_comment": "Generated by compileCanvasGraph.test.ts; regenerate with vitest -u. Validated by tests/app/invocations/test_frontend_external_canvas_graphs.py.", + "txt2img": { + "edges": [ + { + "destination": { + "field": "prompt", + "node_id": "canvas_processing_output" + }, + "source": { + "field": "value", + "node_id": "positive_prompt" + } + }, + { + "destination": { + "field": "seed", + "node_id": "canvas_processing_output" + }, + "source": { + "field": "value", + "node_id": "seed" + } + }, + { + "destination": { + "field": "seed", + "node_id": "core_metadata_0_i" + }, + "source": { + "field": "value", + "node_id": "seed" + } + }, + { + "destination": { + "field": "positive_prompt", + "node_id": "core_metadata_0_i" + }, + "source": { + "field": "value", + "node_id": "positive_prompt" + } + }, + { + "destination": { + "field": "negative_prompt", + "node_id": "core_metadata_0_i" + }, + "source": { + "field": "value", + "node_id": "negative_prompt" + } + }, + { + "destination": { + "field": "metadata", + "node_id": "canvas_processing_output" + }, + "source": { + "field": "metadata", + "node_id": "core_metadata_0_i" + } + }, + { + "destination": { + "field": "collection", + "node_id": "canvas_provider_image" + }, + "source": { + "field": "collection", + "node_id": "canvas_processing_output" + } + }, + { + "destination": { + "field": "image", + "node_id": "canvas_output" + }, + "source": { + "field": "item", + "node_id": "canvas_provider_image" + } + }, + { + "destination": { + "field": "image", + "node_id": "canvas_provider_metadata" + }, + "source": { + "field": "item", + "node_id": "canvas_provider_image" + } + }, + { + "destination": { + "field": "metadata", + "node_id": "canvas_output" + }, + "source": { + "field": "metadata", + "node_id": "canvas_provider_metadata" + } + } + ], + "id": "external_graph_0_i", + "nodes": { + "negative_prompt": { + "is_intermediate": true, + "use_cache": true, + "id": "negative_prompt", + "type": "string" + }, + "positive_prompt": { + "is_intermediate": true, + "use_cache": true, + "id": "positive_prompt", + "type": "string" + }, + "seed": { + "is_intermediate": true, + "use_cache": true, + "id": "seed", + "type": "integer" + }, + "core_metadata_0_i": { + "is_intermediate": true, + "use_cache": true, + "cfg_scale": 7, + "cfg_rescale_multiplier": 0, + "height": 512, + "id": "core_metadata_0_i", + "model": { + "base": "external", + "capabilities": { + "modes": [ + "txt2img", + "img2img", + "inpaint" + ], + "supports_seed": true + }, + "format": "external_api", + "key": "external-model", + "name": "OpenAI Image", + "provider_id": "openai", + "type": "external_image_generator", + "hash": "blake3:external" + }, + "rand_device": "cpu", + "seamless_x": false, + "seamless_y": false, + "steps": 30, + "type": "core_metadata", + "width": 640, + "openai_quality": "auto", + "openai_background": "auto", + "clip_skip": 0 + }, + "canvas_processing_output": { + "is_intermediate": true, + "use_cache": false, + "height": 512, + "id": "canvas_processing_output", + "image_size": null, + "mode": "txt2img", + "model": { + "base": "external", + "capabilities": { + "modes": [ + "txt2img", + "img2img", + "inpaint" + ], + "supports_seed": true + }, + "format": "external_api", + "key": "external-model", + "name": "OpenAI Image", + "provider_id": "openai", + "type": "external_image_generator", + "hash": "blake3:external" + }, + "num_images": 1, + "type": "openai_image_generation", + "width": 640, + "quality": "auto", + "background": "auto" + }, + "canvas_provider_image": { + "is_intermediate": true, + "use_cache": false, + "id": "canvas_provider_image", + "type": "iterate" + }, + "canvas_provider_metadata": { + "is_intermediate": true, + "use_cache": true, + "id": "canvas_provider_metadata", + "type": "metadata_from_image" + }, + "canvas_output": { + "is_intermediate": true, + "use_cache": false, + "height": 512, + "id": "canvas_output", + "type": "img_resize", + "width": 640 + } + } + }, + "img2img": { + "edges": [ + { + "destination": { + "field": "prompt", + "node_id": "canvas_processing_output" + }, + "source": { + "field": "value", + "node_id": "positive_prompt" + } + }, + { + "destination": { + "field": "seed", + "node_id": "canvas_processing_output" + }, + "source": { + "field": "value", + "node_id": "seed" + } + }, + { + "destination": { + "field": "seed", + "node_id": "core_metadata_0_i" + }, + "source": { + "field": "value", + "node_id": "seed" + } + }, + { + "destination": { + "field": "positive_prompt", + "node_id": "core_metadata_0_i" + }, + "source": { + "field": "value", + "node_id": "positive_prompt" + } + }, + { + "destination": { + "field": "negative_prompt", + "node_id": "core_metadata_0_i" + }, + "source": { + "field": "value", + "node_id": "negative_prompt" + } + }, + { + "destination": { + "field": "metadata", + "node_id": "canvas_processing_output" + }, + "source": { + "field": "metadata", + "node_id": "core_metadata_0_i" + } + }, + { + "destination": { + "field": "collection", + "node_id": "canvas_provider_image" + }, + "source": { + "field": "collection", + "node_id": "canvas_processing_output" + } + }, + { + "destination": { + "field": "image", + "node_id": "canvas_output" + }, + "source": { + "field": "item", + "node_id": "canvas_provider_image" + } + }, + { + "destination": { + "field": "image", + "node_id": "canvas_provider_metadata" + }, + "source": { + "field": "item", + "node_id": "canvas_provider_image" + } + }, + { + "destination": { + "field": "metadata", + "node_id": "canvas_output" + }, + "source": { + "field": "metadata", + "node_id": "canvas_provider_metadata" + } + } + ], + "id": "external_graph_0_i", + "nodes": { + "negative_prompt": { + "is_intermediate": true, + "use_cache": true, + "id": "negative_prompt", + "type": "string" + }, + "positive_prompt": { + "is_intermediate": true, + "use_cache": true, + "id": "positive_prompt", + "type": "string" + }, + "seed": { + "is_intermediate": true, + "use_cache": true, + "id": "seed", + "type": "integer" + }, + "core_metadata_0_i": { + "is_intermediate": true, + "use_cache": true, + "cfg_scale": 7, + "cfg_rescale_multiplier": 0, + "height": 512, + "id": "core_metadata_0_i", + "model": { + "base": "external", + "capabilities": { + "modes": [ + "txt2img", + "img2img", + "inpaint" + ], + "supports_seed": true + }, + "format": "external_api", + "key": "external-model", + "name": "OpenAI Image", + "provider_id": "openai", + "type": "external_image_generator", + "hash": "blake3:external" + }, + "rand_device": "cpu", + "seamless_x": false, + "seamless_y": false, + "steps": 30, + "type": "core_metadata", + "width": 640, + "openai_quality": "auto", + "openai_background": "auto", + "clip_skip": 0 + }, + "canvas_processing_output": { + "is_intermediate": true, + "use_cache": false, + "height": 512, + "id": "canvas_processing_output", + "image_size": null, + "mode": "img2img", + "model": { + "base": "external", + "capabilities": { + "modes": [ + "txt2img", + "img2img", + "inpaint" + ], + "supports_seed": true + }, + "format": "external_api", + "key": "external-model", + "name": "OpenAI Image", + "provider_id": "openai", + "type": "external_image_generator", + "hash": "blake3:external" + }, + "num_images": 1, + "type": "openai_image_generation", + "width": 640, + "quality": "auto", + "background": "auto", + "init_image": { + "image_name": "canvas-composite.png" + } + }, + "canvas_provider_image": { + "is_intermediate": true, + "use_cache": false, + "id": "canvas_provider_image", + "type": "iterate" + }, + "canvas_provider_metadata": { + "is_intermediate": true, + "use_cache": true, + "id": "canvas_provider_metadata", + "type": "metadata_from_image" + }, + "canvas_output": { + "is_intermediate": true, + "use_cache": false, + "height": 512, + "id": "canvas_output", + "type": "img_resize", + "width": 640 + } + } + } +} diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts index fb7aac0c7ec..3d63b88fe79 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts @@ -615,6 +615,52 @@ describe('component policies', () => { expect(getGenerationValidationReasons(externalModel, createSettings(externalModel, { width: 1365 }))).toEqual([]); }); + it('rejects dimensions above the provider maximum on either axis', () => { + const model: GenerateModelConfig = { + ...externalModel, + capabilities: { modes: ['txt2img'], max_image_size: { width: 2048, height: 1536 } }, + }; + expect(getGenerationValidationReasons(model, createSettings(model, { width: 2048, height: 1536 }))).toEqual([]); + for (const dimensions of [ + { width: 2049, height: 1536 }, + { width: 2048, height: 1537 }, + ]) { + expect(getGenerationValidationReasons(model, createSettings(model, dimensions))).toContain( + 'External Model supports a maximum size of 2048×1536.' + ); + } + }); + + it('counts the canvas init and output against the Seedream total image budget', () => { + const model: GenerateModelConfig = { + ...externalModel, + provider_id: 'seedream', + provider_model_id: 'seedream-4-5-251128', + capabilities: { modes: ['txt2img', 'img2img'], supports_reference_images: true, max_reference_images: 14 }, + }; + const settings = createSettings(model, { + referenceImages: Array.from({ length: 14 }, (_, i) => ({ + id: `reference-${i}`, + isEnabled: true, + config: { + type: 'external_reference_image', + image: { + imageName: `reference-${i}.png`, + width: 512, + height: 512, + original: { image: { image_name: `reference-${i}.png`, width: 512, height: 512 } }, + }, + }, + })), + }); + expect(getGenerationValidationReasons(model, settings, 'txt2img')).toEqual([]); + expect(getGenerationValidationReasons(model, settings, 'img2img')).toEqual([ + 'External Model supports at most 15 images total (reference + init + output), got 16.', + ]); + settings.referenceImages[0]!.isEnabled = false; + expect(getGenerationValidationReasons(model, settings, 'img2img')).toEqual([]); + }); + it('renders no component slots for bases without extra requirements', () => { expect(getComponentSectionPolicy(createModel('cogview4'), createSettings(createModel('cogview4'))).slots).toEqual( [] @@ -663,6 +709,86 @@ describe('component policies', () => { expect(result.clearedLabels).toEqual(['LoRAs', 'T5 Encoder', 'CLIP skip', 'CFG rescale']); }); + it('initializes a provider size on model changes while preserving supported and same-model dimensions', () => { + const model: GenerateModelConfig = { + base: 'external', + format: 'external_api', + key: 'gemini', + name: 'Gemini', + provider_id: 'gemini', + type: 'external_image_generator', + capabilities: { + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: 'Square', width: 1024 }, + { aspect_ratio: '16:9', height: 1152, image_size: '2K', label: 'Landscape', width: 2048 }, + ], + }, + }; + const select = (settings: GenerateSettings) => + getGenerateModelSelectionResult({ currentValues: settings, model, models: [] }).settings; + + expect(select(createSettings(createModel('sd-1'), { height: 512, width: 512 }))).toMatchObject({ + externalImageSize: '1K', + height: 1024, + width: 1024, + }); + expect(select(createSettings(createModel('sdxl'), { height: 576, width: 1024 }))).toMatchObject({ + aspectRatioId: '16:9', + aspectRatioValue: 2048 / 1152, + externalImageSize: '2K', + height: 1152, + width: 2048, + }); + expect( + select(createSettings(createModel('sdxl'), { externalImageSize: 'stale', height: 1152, width: 2048 })) + ).toMatchObject({ + externalImageSize: '2K', + height: 1152, + width: 2048, + }); + expect(select(createSettings(model, { height: 1200, width: 1600 }))).toMatchObject({ + externalImageSize: null, + height: 1200, + width: 1600, + }); + const local = getGenerateModelSelectionResult({ + currentValues: createSettings(model, { externalImageSize: '2K', height: 1152, width: 2048 }), + model: createModel('sdxl'), + models: [], + }).settings; + + expect(local).toMatchObject({ externalImageSize: null, height: 1152, width: 2048 }); + }); + + it('prefers the nearer pixel budget among presets of one nominal aspect ratio', () => { + // Rounded 16:9 presets have slightly different pixel ratios (1.75 vs 1.78), so an exact ratio tie never occurs. + const model: GenerateModelConfig = { + base: 'external', + format: 'external_api', + key: 'gemini', + name: 'Gemini', + provider_id: 'gemini', + type: 'external_image_generator', + capabilities: { + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '16:9', height: 1152, image_size: '2K', label: '16:9 (2K)', width: 2048 }, + { aspect_ratio: '16:9', height: 768, image_size: '1K', label: '16:9 (1K)', width: 1344 }, + { aspect_ratio: '4:3', height: 768, image_size: '1K', label: '4:3 (1K)', width: 1024 }, + ], + }, + }; + + const result = getGenerateModelSelectionResult({ + currentValues: createSettings(createModel('sdxl'), { height: 576, width: 1024 }), + model, + models: [], + }); + + expect(result.settings).toMatchObject({ externalImageSize: '1K', height: 768, width: 1344 }); + }); + it('turns HiDiffusion off when changing to an unsupported model family', () => { const result = getGenerateModelSelectionResult({ currentValues: createSettings(createModel('sd-1'), { hiDiffusionEnabled: true }), diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts index 8a46747625e..0bb7f8098fb 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts @@ -50,6 +50,13 @@ import { type GenerateComponentFilter, } from './componentCompatibility'; import { DYNAMIC_PROMPTS_DEFAULT_MAX_PROMPTS } from './dynamicPrompts'; +import { + getExternalAspectRatioSizes, + getExternalResolutionPresets, + isSeedreamBatchModel, + resolveExternalImageSize, + SEEDREAM_BATCH_MAX_TOTAL_IMAGES, +} from './externalProvider'; import { DEFAULT_PID_STEPS, getIsPidActive, @@ -1432,6 +1439,40 @@ const getSettingsWithCompatibleModelSelections = ( addClearedLabel(clearedLabels, 'Dimensions'); } + if (settings.modelKey !== model.key) { + const sizes = getExternalResolutionPresets(model); + const supportedSizes = sizes.length > 0 ? sizes : getExternalAspectRatioSizes(model).map(([, size]) => size); + const matchesSize = supportedSizes.some((size) => size.width === width && size.height === height); + + if (!matchesSize && supportedSizes.length > 0) { + // Keep the frame's shape when possible, then its pixel budget within that shape. + const distance = (size: { width: number; height: number }) => ({ + area: Math.abs(Math.log((size.width * size.height) / (width * height))), + ratio: Math.abs(Math.log(size.width / size.height / (width / height))), + }); + // Rounded presets of one nominal ratio differ by ~0.02 in log ratio; distinct ratios by 0.1 or more. + const RATIO_TOLERANCE = 0.05; + const selected = supportedSizes.reduce((best, size) => { + const candidateDistance = distance(size); + const bestDistance = distance(best); + const sameRatio = Math.abs(candidateDistance.ratio - bestDistance.ratio) < RATIO_TOLERANCE; + + return (!sameRatio && candidateDistance.ratio < bestDistance.ratio) || + (sameRatio && candidateDistance.area < bestDistance.area) + ? size + : best; + }); + + nextSettings.width = selected.width; + nextSettings.height = selected.height; + nextSettings.aspectRatioId = deriveAspectRatioId(selected.width, selected.height); + nextSettings.aspectRatioValue = selected.width / selected.height; + addClearedLabel(clearedLabels, 'Dimensions'); + } + + nextSettings.externalImageSize = resolveExternalImageSize(nextSettings, model); + } + if (compatibleLoras.length !== settings.loras.length) { nextSettings.loras = compatibleLoras; addClearedLabel(clearedLabels, 'LoRAs'); @@ -1598,6 +1639,11 @@ const getDimensionValidationReasons = (model: GenerateModelConfig, settings: Gen reasons.push(`Generate height must be a multiple of ${dimensions.grid}.`); } + const maxSize = model.type === 'external_image_generator' ? model.capabilities?.max_image_size : null; + if (maxSize && (settings.width > maxSize.width || settings.height > maxSize.height)) { + reasons.push(`${model.name} supports a maximum size of ${maxSize.width}×${maxSize.height}.`); + } + return reasons; }; @@ -1774,6 +1820,21 @@ export const getGenerationValidationReasons = ( reasons.push(`No invocation node registered for external provider '${model.provider_id ?? ''}'.`); } + // Every request emits one output, and the Canvas edit modes add the composite as one init image. + if (isSeedreamBatchModel(model)) { + const referenceCount = settings.referenceImages.filter( + (reference) => + reference.isEnabled && reference.config.type === 'external_reference_image' && reference.config.image !== null + ).length; + const totalImages = referenceCount + (mode === 'txt2img' ? 0 : 1) + 1; + + if (totalImages > SEEDREAM_BATCH_MAX_TOTAL_IMAGES) { + reasons.push( + `${model.name} supports at most ${SEEDREAM_BATCH_MAX_TOTAL_IMAGES} images total (reference + init + output), got ${totalImages}.` + ); + } + } + return reasons; } diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts index b3976bc406b..9cb39a390f1 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts @@ -1,7 +1,7 @@ /** * None snaps the bbox to the model grid; auto grows undersized content toward optimal area at its aspect ratio; * manual uses the selected size. Restore final output to the exact bbox. Hosted providers are asked for the bbox - * itself: they bucket the request to their own size table and return one image, so there is nothing to scale. + * itself: they bucket the request to their own size table, then the compiler restores their output to the bbox. */ import type { GenerateModelConfig, PidMode } from '@features/generation/core/types'; diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index 245ea915733..3edee74a824 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -437,13 +437,36 @@ describe('compileCanvasGraph', () => { }; const smallBbox: Rect = { height: 512, width: 640, x: 10, y: 20 }; - it('generates txt2img from an empty bbox on one provider node sized to the bbox, with no encode or resize', () => { + it('writes hosted canvas graphs for backend invocation contract validation', async () => { + const clock = vi.spyOn(Date, 'now').mockReturnValue(0); + const random = vi.spyOn(Math, 'random').mockReturnValue(0.5); + const graphs: Record = {}; + try { + for (const mode of ['txt2img', 'img2img'] as const) { + graphs[mode] = compile({ ...editModel, hash: 'blake3:external' }, mode, { bbox: smallBbox }).backendGraph; + } + } finally { + clock.mockRestore(); + random.mockRestore(); + } + const fixture = { + _comment: + 'Generated by compileCanvasGraph.test.ts; regenerate with vitest -u. Validated by tests/app/invocations/test_frontend_external_canvas_graphs.py.', + ...graphs, + }; + await expect(`${JSON.stringify(fixture, null, 2)}\n`).toMatchFileSnapshot( + '../__snapshots__/externalCanvasGraphs.json' + ); + }); + + it.each(['canvas', 'gallery'] as const)('restores provider output to the exact bbox for %s', (destination) => { const { backendGraph, mode } = compile(externalModel, 'txt2img', { bbox: smallBbox, compositeImageName: null, + destination, scaling: { height: null, method: 'auto', width: null }, }); - const output = backendGraph.nodes.canvas_output; + const output = backendGraph.nodes.canvas_processing_output; expect(mode).toBe('txt2img'); expect(output).toMatchObject({ @@ -455,58 +478,81 @@ describe('compileCanvasGraph', () => { }); expect(output).not.toHaveProperty('init_image'); expect(output).not.toHaveProperty('mask_image'); - expect(getEdge(backendGraph, 'canvas_output', 'prompt')?.source.node_id).toBe('positive_prompt'); - expect(getEdge(backendGraph, 'canvas_output', 'seed')?.source.node_id).toBe('seed'); + expect(getEdge(backendGraph, 'canvas_processing_output', 'prompt')?.source.node_id).toBe('positive_prompt'); + expect(getEdge(backendGraph, 'canvas_processing_output', 'seed')?.source.node_id).toBe('seed'); expect(backendGraph.nodes.canvas_i2l).toBeUndefined(); - expect(backendGraph.nodes.canvas_processing_output).toBeUndefined(); - expect(Object.values(backendGraph.nodes).map((node) => node.type)).not.toContain('img_resize'); - // The provider returns one image at the bbox size, so metadata records the bbox too. + expect(backendGraph.nodes.canvas_output).toMatchObject({ + type: 'img_resize', + height: 512, + width: 640, + is_intermediate: destination === 'canvas', + }); + expect(backendGraph.nodes.canvas_provider_image.type).toBe('iterate'); + expect(getEdge(backendGraph, 'canvas_provider_image', 'collection')?.source).toEqual({ + node_id: 'canvas_processing_output', + field: 'collection', + }); + expect(getEdge(backendGraph, 'canvas_output', 'image')?.source).toEqual({ + node_id: 'canvas_provider_image', + field: 'item', + }); + expect(backendGraph.nodes.canvas_provider_metadata.type).toBe('metadata_from_image'); + expect(getEdge(backendGraph, 'canvas_provider_metadata', 'image')?.source).toEqual({ + node_id: 'canvas_provider_image', + field: 'item', + }); + expect(getEdge(backendGraph, 'canvas_output', 'metadata')?.source.node_id).toBe('canvas_provider_metadata'); + expect(getEdge(backendGraph, 'canvas_processing_output', 'metadata')).toBeDefined(); expect(getNodeByType(backendGraph, 'core_metadata')).toMatchObject({ height: 512, width: 640 }); }); it('sends the composited bbox as the init image for img2img when the model advertises it', () => { - const { backendGraph } = compile(editModel, 'img2img', { compositeImageName: 'bbox.png' }); + const { backendGraph } = compile({ ...editModel, capabilities: { modes: ['img2img'] } }, 'img2img', { + compositeImageName: 'bbox.png', + }); - expect(backendGraph.nodes.canvas_output).toMatchObject({ + expect(backendGraph.nodes.canvas_processing_output).toMatchObject({ + height: 1024, init_image: { image_name: 'bbox.png' }, + is_intermediate: true, mode: 'img2img', + width: 768, + }); + expect(backendGraph.nodes.canvas_processing_output).not.toHaveProperty('mask_image'); + // The edited whole image is restored to the bbox like a txt2img result. + expect(backendGraph.nodes.canvas_output).toMatchObject({ height: 1024, type: 'img_resize', width: 768 }); + expect(getEdge(backendGraph, 'canvas_output', 'image')?.source).toEqual({ + node_id: 'canvas_provider_image', + field: 'item', }); - expect(backendGraph.nodes.canvas_output).not.toHaveProperty('mask_image'); + expect(getNodeByType(backendGraph, 'core_metadata')).toMatchObject({ height: 1024, width: 768 }); // No local denoise path is grafted: the provider edits the whole image. expect(backendGraph.nodes.denoise_latents).toBeUndefined(); expect(backendGraph.nodes.create_gradient_mask).toBeUndefined(); }); - it('sends the composite and the grayscale mask for inpaint', () => { - const { backendGraph } = compile(editModel, 'inpaint', { - compositeImageName: 'bbox.png', - maskImageName: 'mask.png', - }); - - expect(backendGraph.nodes.canvas_output).toMatchObject({ - init_image: { image_name: 'bbox.png' }, - mask_image: { image_name: 'mask.png' }, - mode: 'inpaint', - }); - // The provider composites its own edit, so no blend/apply-mask tail follows it. - expect(backendGraph.nodes.invokeai_img_blend).toBeUndefined(); - expect(backendGraph.nodes.expand_mask).toBeUndefined(); + it('rejects provider inpaint until Canvas mask semantics and compositing are supported', () => { + expect(() => + compile(editModel, 'inpaint', { + compositeImageName: 'bbox.png', + maskImageName: 'mask.png', + }) + ).toThrow('OpenAI Image does not support inpainting on the canvas yet. Remove or disable the inpaint mask.'); }); - it('still requires the composite and the mask the mode needs', () => { + it('requires the composite for img2img', () => { expect(() => compile(editModel, 'img2img', { compositeImageName: null })).toThrow( 'Canvas generation requires a composited source image.' ); - expect(() => compile(editModel, 'inpaint', { maskImageName: null })).toThrow( - 'Canvas inpainting requires an inpaint mask.' - ); }); it('rejects the edit modes a model does not advertise, naming the mode', () => { expect(() => compile(externalModel, 'img2img')).toThrow( 'OpenAI Image does not support image-to-image generation.' ); - expect(() => compile(externalModel, 'inpaint')).toThrow('OpenAI Image does not support inpainting.'); + expect(() => compile(externalModel, 'inpaint')).toThrow( + 'OpenAI Image does not support inpainting on the canvas yet.' + ); expect(() => compile({ ...externalModel, capabilities: { modes: ['img2img'] } }, 'txt2img', { compositeImageName: null }) ).toThrow('OpenAI Image does not support text-to-image generation.'); @@ -567,7 +613,7 @@ describe('compileCanvasGraph', () => { }); expect(custom.backendGraph.nodes.canvas_output).toMatchObject({ height: 512, width: 640 }); - expect(Object.values(custom.backendGraph.nodes).map((node) => node.type)).not.toContain('img_resize'); + expect(custom.backendGraph.nodes.canvas_processing_output).toMatchObject({ height: 512, width: 640 }); }); }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts index 07da2406f01..c63383bdc4f 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts @@ -1,6 +1,6 @@ /** * Pure canvas compiler over uploaded inputs. Resize through the processing grid and restore the bbox; hosted - * providers take the bbox composite as-is in the modes they advertise; reject unsupported encode modes. + * providers take the bbox composite for edits and restore their result to the bbox; reject unsupported modes. */ import type { SupportedGenerateBase } from '@features/generation/core/baseGenerationPolicies'; @@ -98,9 +98,10 @@ const getCanvasValidationReasons = (input: CompileCanvasGraphInput, processingSi const reasons: string[] = []; if (model.type === 'external_image_generator') { - // A provider takes a whole image to edit and nothing else: a partially covered bbox has nothing to send for the - // empty area, and the request carries no conditioning beyond the prompt, the init image and the mask. The - // shared validator reports the other modes the model does not advertise. + // The Canvas keep-mask does not match provider edit-mask contracts, and there is no masked compositing tail. + if (mode === 'inpaint') { + reasons.push(`${model.name} does not support inpainting on the canvas yet. Remove or disable the inpaint mask.`); + } if (mode === 'outpaint') { reasons.push(`${model.name} does not support outpainting on the canvas.`); } @@ -262,6 +263,31 @@ const resizeCanvasOutputToBbox = ( } }; +/** Hosted nodes return an image collection; preserve each image's provider metadata through the resize. */ +const resizeExternalCanvasOutputToBbox = ( + graph: BackendGraphContract, + bbox: CompileCanvasGraphInput['bbox'], + destination: CompileCanvasGraphInput['destination'] +): void => { + const provider = renameNode(graph, 'canvas_output', 'canvas_processing_output'); + provider.is_intermediate = true; + const image = addNode(graph, { id: 'canvas_provider_image', type: 'iterate', use_cache: false }); + const metadata = addNode(graph, { id: 'canvas_provider_metadata', type: 'metadata_from_image' }); + const output = addNode(graph, { + height: bbox.height, + id: 'canvas_output', + is_intermediate: destination === 'canvas', + type: 'img_resize', + use_cache: false, + width: bbox.width, + }); + // Keep core metadata on the provider so its output adds request identity, defaults and actual seed to it. + addEdge(graph, provider, 'collection', image, 'collection'); + addEdge(graph, image, 'item', output, 'image'); + addEdge(graph, image, 'item', metadata, 'image'); + addEdge(graph, metadata, 'metadata', output, 'metadata'); +}; + /** Keeps saved-image metadata aligned to the bbox footprint, not the internal processing size. */ const setCanvasMetadataDimensions = (graph: BackendGraphContract, bbox: CompileCanvasGraphInput['bbox']): void => { const metadata = findMetadataNode(graph); @@ -777,25 +803,21 @@ export const compileCanvasGraph = (input: CompileCanvasGraphInput): CompiledCanv }; if (model.type === 'external_image_generator') { - if (!isExternalGenerationMode(mode)) { - throw new Error(`${model.name} does not support outpainting on the canvas.`); + // Validation above already named the mode; this narrows the provider inputs to the two modes Canvas compiles. + if (mode !== 'txt2img' && mode !== 'img2img') { + throw new Error(`${model.name} does not support ${mode}ing on the canvas.`); } - // Validation above guarantees the composite for the edit modes and the mask for inpaint. The provider returns - // the edited image whole, so no encode, mask tail, control, or regional grafts apply, and no denoising strength - // is sent or recorded. Metadata carries no `generation_mode`: the backend's mode list has no provider variants. - const initImageName = compositeImageName as string; + // Providers bucket the request to their own size table, so the returned whole image is restored to the exact + // bbox footprint before staging or saving. No local denoising strength applies. const backendGraph = buildExternalGraph( settings, model, outputIsIntermediate, runtimeProjectSettings, - mode === 'txt2img' - ? { mode } - : mode === 'img2img' - ? { initImageName, mode } - : { initImageName, maskImageName: input.maskImageName as string, mode } + mode === 'txt2img' ? { mode } : { initImageName: compositeImageName as string, mode } ); + resizeExternalCanvasOutputToBbox(backendGraph, bbox, destination); setCanvasMetadataDimensions(backendGraph, bbox); return { diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts index 5c1d627f2cd..e51abc6e7dc 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts @@ -60,7 +60,8 @@ describe('external provider options', () => { const gemini = external('gemini', { modes: ['txt2img'], resolution_presets: presets }); expect(resolveExternalImageSize({ externalImageSize: '2K', height: 1152, width: 2048 }, gemini)).toBe('2K'); - expect(resolveExternalImageSize({ externalImageSize: '4K', height: 1152, width: 2048 }, gemini)).toBeNull(); + expect(resolveExternalImageSize({ externalImageSize: '4K', height: 1152, width: 2048 }, gemini)).toBe('2K'); + expect(resolveExternalImageSize({ externalImageSize: null, height: 1152, width: 2048 }, gemini)).toBe('2K'); // A size typed by hand after picking a preset is custom: the provider buckets the dimensions instead. expect(resolveExternalImageSize({ externalImageSize: '2K', height: 1152, width: 2000 }, gemini)).toBeNull(); expect( @@ -71,14 +72,32 @@ describe('external provider options', () => { ).toBeNull(); }); - it('matches the preset select to the exact size and named size, so a hand-typed size reads as custom', () => { + it('recognizes typed preset dimensions while keeping unsupported dimensions custom', () => { const gemini = external('gemini', { modes: ['txt2img'], resolution_presets: presets }); expect(findExternalResolutionPreset({ externalImageSize: '2K', height: 1152, width: 2048 }, gemini)?.label).toBe( '16:9 (2K)' ); expect(findExternalResolutionPreset({ externalImageSize: '2K', height: 1152, width: 2000 }, gemini)).toBeNull(); - expect(findExternalResolutionPreset({ externalImageSize: null, height: 1024, width: 1024 }, gemini)).toBeNull(); + expect( + findExternalResolutionPreset({ externalImageSize: null, height: 1024, width: 1024 }, gemini)?.image_size + ).toBe('1K'); + }); + + it('breaks a tie between presets of one size by the stored name, else the first', () => { + const duplicate = external('gemini', { + modes: ['txt2img'], + resolution_presets: [ + ...presets, + { aspect_ratio: '1:1', height: 1024, image_size: 'HD', label: '1:1 (HD)', width: 1024 }, + ], + }); + + expect(findExternalResolutionPreset({ externalImageSize: 'HD', height: 1024, width: 1024 }, duplicate)?.label).toBe( + '1:1 (HD)' + ); + expect(resolveExternalImageSize({ externalImageSize: 'HD', height: 1024, width: 1024 }, duplicate)).toBe('HD'); + expect(resolveExternalImageSize({ externalImageSize: null, height: 1024, width: 1024 }, duplicate)).toBe('1K'); }); it('lists fixed ratio sizes in the provider’s order', () => { diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts index 6951bfc77a4..207c118e2d4 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts @@ -36,6 +36,26 @@ export const isGeminiThinkingLevel = (value: unknown): value is GeminiThinkingLe export const isExternalGenerationMode = (value: unknown): value is ExternalGenerationMode => value === 'txt2img' || value === 'img2img' || value === 'inpaint'; +/** + * Seedream's batch endpoint counts inputs and outputs against one total. Mirrors `_SEEDREAM_BATCH_PREFIXES` and + * `_SEEDREAM_BATCH_MAX_TOTAL_IMAGES` in `invokeai/app/services/external_generation/providers/seedream.py`. + */ +const SEEDREAM_BATCH_MODEL_PREFIXES = [ + 'seedream-5', + 'seedream-4.5', + 'seedream-4.0', + 'seedream-4-5', + 'seedream-4-0', + 'seedream-5-0', +] as const; +export const SEEDREAM_BATCH_MAX_TOTAL_IMAGES = 15; + +export const isSeedreamBatchModel = (model: GenerateModelConfig): boolean => + model.type === 'external_image_generator' && + model.provider_id === 'seedream' && + typeof model.provider_model_id === 'string' && + SEEDREAM_BATCH_MODEL_PREFIXES.some((prefix) => (model.provider_model_id as string).startsWith(prefix)); + export type ExternalProviderOptionKey = keyof Pick< GenerateSettings, | 'openaiQuality' @@ -97,20 +117,20 @@ export const getExternalAspectRatioSizes = ( return sizes ? Object.entries(sizes) : []; }; -/** The preset matching the current size and named size exactly, for the preset select's value. */ +/** Dimensions select the preset; the stored name breaks ties if the provider advertises duplicate sizes. */ export const findExternalResolutionPreset = ( settings: Pick, model: GenerateModelConfig | undefined -): ExternalResolutionPreset | null => - getExternalResolutionPresets(model).find( - (preset) => - preset.image_size === settings.externalImageSize && - preset.width === settings.width && - preset.height === settings.height - ) ?? null; +): ExternalResolutionPreset | null => { + const presets = getExternalResolutionPresets(model).filter( + (preset) => preset.width === settings.width && preset.height === settings.height + ); + + return presets.find((preset) => preset.image_size === settings.externalImageSize) ?? presets[0] ?? null; +}; /** - * The named size to send: the selected model's preset the current size matches exactly, so a hand-typed size or a + * The named size to send: the selected model's preset the current size matches exactly, including a hand-typed size. A * pick left over from another model never overrides the dimensions at the provider. */ export const resolveExternalImageSize = ( diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.ts index de4497a598c..9fcad602810 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.ts @@ -1515,15 +1515,12 @@ const buildWanGraph = ( return graph; }; -/** Canvas inputs per external mode: the composited bbox for the edit modes and its grayscale mask for inpaint. */ -export type ExternalCanvasInputs = - | { mode: 'txt2img' } - | { mode: 'img2img'; initImageName: string } - | { mode: 'inpaint'; initImageName: string; maskImageName: string }; +/** Canvas inputs per external mode: the composited bbox is the init image for img2img. */ +export type ExternalCanvasInputs = { mode: 'txt2img' } | { mode: 'img2img'; initImageName: string }; /** * Hosted providers take the whole request on one node: prompt, seed, size, provider options, and (on the canvas) the - * init image and mask. The provider composites its own edits, so the node's image is the canvas output as-is. + * init image. The Canvas keep-mask does not match any provider's edit-mask contract, so inpaint is not compiled. */ export const buildExternalGraph = ( settings: GenerateSettings, @@ -1564,8 +1561,7 @@ export const buildExternalGraph = ( use_cache: false, width: settings.width, ...getExternalProviderInvocationOptions(settings, model), - ...(canvas && canvas.mode !== 'txt2img' ? { init_image: { image_name: canvas.initImageName } } : {}), - ...(canvas?.mode === 'inpaint' ? { mask_image: { image_name: canvas.maskImageName } } : {}), + ...(canvas?.mode === 'img2img' ? { init_image: { image_name: canvas.initImageName } } : {}), }); addEdge(graph, positivePrompt, 'value', output, 'prompt'); diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts index 07d3bb94f0f..dce99cb1527 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts @@ -327,6 +327,16 @@ describe('normalizeGenerateSettings', () => { }); }); + it.each([0, 2])('preserves the Gemini temperature boundary %s', (geminiTemperature) => { + expect(normalizeGenerateSettings({ ...legacyStoredValues, geminiTemperature })?.geminiTemperature).toBe( + geminiTemperature + ); + }); + + it('drops a negative Gemini temperature back to the provider default', () => { + expect(normalizeGenerateSettings({ ...legacyStoredValues, geminiTemperature: -0.1 })?.geminiTemperature).toBeNull(); + }); + it('drops malformed persisted model identifiers missing a base', () => { const normalized = normalizeGenerateSettings({ ...legacyStoredValues, diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx index deb161a88a9..1782377a1e7 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.browser.test.tsx @@ -47,6 +47,7 @@ void i18n.use(initReactI18next).init({ height: 'Height', lockAspectRatio: 'Lock aspect ratio', megapixelsValue: '{{value}} MP', + providerSize: 'Provider size', size: 'Size', swapWidthAndHeight: 'Swap width and height', unlockAspectRatio: 'Unlock aspect ratio', @@ -139,6 +140,65 @@ describe('GenerateDimensionFields canvas render size', () => { }); }); +describe('GenerateDimensionFields provider size', () => { + const geminiImage: GenerateModelConfig = { + ...gptImage, + provider_id: 'gemini', + capabilities: { + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }, + { aspect_ratio: '16:9', height: 1152, image_size: '2K', label: '16:9 (2K)', width: 2048 }, + ], + }, + }; + const providerTrigger = () => { + const providerLabel = [...(host?.querySelectorAll('label') ?? [])].find( + (label) => label.textContent === 'Provider size' + ); + + return host?.querySelector(`[role="combobox"][aria-labelledby~="${providerLabel?.id}"]`) ?? null; + }; + + it('selects the preset from the panel dimensions, breaking a same-size tie by the stored name', async () => { + await render({ externalImageSize: null, height: 1152, width: 2048 }, geminiImage); + expect(providerTrigger()?.textContent).toContain('16:9 (2K)'); + + await settle(() => root?.unmount()); + host?.remove(); + + const duplicateSizes: GenerateModelConfig = { + ...geminiImage, + capabilities: { + modes: ['txt2img'], + resolution_presets: [ + ...(geminiImage.capabilities?.resolution_presets ?? []), + { aspect_ratio: '1:1', height: 1024, image_size: 'HD', label: '1:1 (HD)', width: 1024 }, + ], + }, + }; + await render({ externalImageSize: 'HD', height: 1024, width: 1024 }, duplicateSizes); + expect(providerTrigger()?.textContent).toContain('1:1 (HD)'); + }); + + it('commits the hosted preset through the dimension panel with its dimensions and name', async () => { + const onCommit = await render({ externalImageSize: null, height: 1024, width: 1024 }, geminiImage); + + expect(providerTrigger()?.textContent).toContain('1:1 (1K)'); + await settle(() => providerTrigger()?.click()); + await settle(() => document.querySelector('[role="option"][data-value="16:9|2K"]')?.click()); + + expect(onCommit).toHaveBeenCalledWith({ + aspectRatioId: '16:9', + aspectRatioIsLocked: true, + aspectRatioValue: 2048 / 1152, + externalImageSize: '2K', + height: 1152, + width: 2048, + }); + }); +}); + describe('GenerateDimensionFields aspect ratio', () => { it('reshapes the size to a chosen preset and locks it', async () => { const onCommit = await render({ aspectRatioId: 'Free', aspectRatioIsLocked: false, height: 1024, width: 1024 }); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx index dc428cad8e5..e2994d74e12 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx @@ -123,6 +123,12 @@ describe('ExternalSizeSelect', () => { }); }); + it('recognizes dimensions typed to a provider preset without a stored name', async () => { + await render(presetModel, { externalImageSize: null, height: 1152, width: 2048 }); + + expect(trigger()?.textContent).toContain('16:9 (2K)'); + }); + it('shows a hand-typed size as custom', async () => { await render(presetModel, { externalImageSize: '1K', height: 1000, width: 1024 }); diff --git a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts index 8cda91028db..1e0a6d71e08 100644 --- a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts +++ b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts @@ -1266,6 +1266,8 @@ describe('image recall', () => { expect(result?.values).toMatchObject({ externalImageSize: '2K', + height: 2048, + width: 2048, geminiTemperature: 0.4, geminiThinkingLevel: 'high', openaiQuality: 'auto', @@ -1273,6 +1275,100 @@ describe('image recall', () => { expect(result?.fields).toContain('externalProvider'); }); + it('skips a named size that is ambiguous without dimensions or an aspect ratio', () => { + const ambiguousModel = { + ...geminiModel, + capabilities: { + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '1:1', height: 2048, image_size: '2K', label: '1:1 (2K)', width: 2048 }, + { aspect_ratio: '16:9', height: 1152, image_size: '2K', label: '16:9 (2K)', width: 2048 }, + ], + }, + } as GenerateWidgetValues['model']; + const ambiguous = recall(ambiguousModel, { image_size: '2K' }, 'all'); + const resolved = recall(ambiguousModel, { image_size: '2K', aspect_ratio: '16:9' }, 'all'); + + expect(ambiguous?.values.externalImageSize).toBeNull(); + expect(ambiguous?.fields).not.toContain('externalProvider'); + expect(resolved?.values).toMatchObject({ externalImageSize: '2K', height: 1152, width: 2048 }); + }); + + it.each([0, 2])('recalls Gemini temperature boundary %s', (temperature) => { + expect(recall(geminiModel, { gemini_temperature: temperature }, 'all')?.values.geminiTemperature).toBe( + temperature + ); + }); + + it('rejects a negative recalled Gemini temperature while preserving the current choice', () => { + const result = buildImageRecallSettings({ + currentValues: createValues({ model: geminiModel, modelKey: geminiModel.key, geminiTemperature: 0.4 }), + image, + kind: 'all', + metadata: { external_provider: 'gemini', gemini_temperature: -0.1 }, + models: [], + supportedModels: [geminiModel], + vaeModels: [], + }); + + expect(result?.values.geminiTemperature).toBe(0.4); + }); + + it.each(['all', 'remix'] as const)('restores omitted nullable provider defaults during %s', (kind) => { + for (const model of [geminiModel, openaiModel]) { + const result = buildImageRecallSettings({ + currentValues: createValues({ + model, + modelKey: model.key, + geminiTemperature: 0.4, + geminiThinkingLevel: 'high', + openaiInputFidelity: 'high', + }), + image, + kind, + metadata: { external_provider: model.provider_id }, + models: [], + supportedModels: [model], + vaeModels: [], + }); + + if (model.provider_id === 'gemini') { + expect(result?.values).toMatchObject({ + geminiTemperature: null, + geminiThinkingLevel: null, + openaiInputFidelity: 'high', + }); + } else { + expect(result?.values).toMatchObject({ + geminiTemperature: 0.4, + geminiThinkingLevel: 'high', + openaiInputFidelity: null, + }); + } + expect(result?.fields).toContain('externalProvider'); + } + }); + + it('preserves optional provider choices when unrelated partial metadata omits them', () => { + const result = buildImageRecallSettings({ + currentValues: createValues({ + model: geminiModel, + modelKey: geminiModel.key, + geminiTemperature: 0.4, + geminiThinkingLevel: 'high', + }), + image, + kind: 'all', + metadata: { positive_prompt: 'a sunset', external_provider: 'openai' }, + models: [], + supportedModels: [geminiModel], + vaeModels: [], + }); + + expect(result?.values).toMatchObject({ geminiTemperature: 0.4, geminiThinkingLevel: 'high' }); + expect(result?.fields).not.toContain('externalProvider'); + }); + it('restores Seedream toggles and drops values the node would reject', () => { const result = recall( seedreamModel, @@ -1302,7 +1398,16 @@ describe('image recall', () => { vaeModels: [], }); - expect(result?.values.openaiQuality).toBe('auto'); + expect(result?.values).toMatchObject({ + externalImageSize: null, + geminiTemperature: null, + geminiThinkingLevel: null, + openaiBackground: 'auto', + openaiInputFidelity: null, + openaiQuality: 'auto', + seedreamOptimizePrompt: false, + seedreamWatermark: false, + }); expect(result?.fields).not.toContain('externalProvider'); }); diff --git a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts index ab97dc7c7b0..70de25b436b 100644 --- a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts +++ b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts @@ -438,17 +438,39 @@ const getMetadataExternalProvider = ( metadata: unknown, model: GenerateModelConfig | undefined ): Partial => { - if (model?.type !== 'external_image_generator') { - return {}; - } - + // Local models advertise no provider options or presets, so every lookup below answers nothing for them. const patch: Partial = {}; const imageSize = getString(metadata, 'image_size'); - if (imageSize !== null && getExternalResolutionPresets(model).some((preset) => preset.image_size === imageSize)) { - patch.externalImageSize = imageSize; + if (imageSize !== null) { + const width = getNumber(metadata, 'width'); + const height = getNumber(metadata, 'height'); + const ratio = getString(metadata, 'aspect_ratio'); + const presets = getExternalResolutionPresets(model).filter( + (preset) => + preset.image_size === imageSize && + (width === null || preset.width === width) && + (height === null || preset.height === height) && + (ratio === null || preset.aspect_ratio === ratio) + ); + const preset = presets.length === 1 ? presets[0] : undefined; + + // A name alone can describe many aspect ratios. Restore only an unambiguous size, with its dimensions. + if (preset) { + patch.externalImageSize = preset.image_size; + patch.width = preset.width; + patch.height = preset.height; + patch.aspectRatioId = deriveAspectRatioId(preset.width, preset.height); + patch.aspectRatioValue = preset.width / preset.height; + } } + // Provider outputs omit nullable fields when their default was used. The provider stamp distinguishes those + // omissions from partial or unrelated metadata, where the current choice should remain untouched. + const usesProviderDefaults = getString(metadata, 'external_provider') === model?.provider_id; + const isDefault = (key: string): boolean => + isRecord(metadata) && (metadata[key] === null || (usesProviderDefaults && !(key in metadata))); + for (const key of getExternalProviderOptionKeys(model)) { switch (key) { case 'openaiQuality': { @@ -472,6 +494,8 @@ const getMetadataExternalProvider = ( if (isOpenAIInputFidelity(value)) { patch.openaiInputFidelity = value; + } else if (isDefault('openai_input_fidelity')) { + patch.openaiInputFidelity = null; } break; } @@ -480,6 +504,8 @@ const getMetadataExternalProvider = ( if (value !== null && value >= GEMINI_TEMPERATURE_MIN && value <= GEMINI_TEMPERATURE_MAX) { patch.geminiTemperature = value; + } else if (isDefault('gemini_temperature')) { + patch.geminiTemperature = null; } break; } @@ -488,6 +514,8 @@ const getMetadataExternalProvider = ( if (isGeminiThinkingLevel(value)) { patch.geminiThinkingLevel = value; + } else if (isDefault('gemini_thinking_level')) { + patch.geminiThinkingLevel = null; } break; } diff --git a/invokeai/frontend/webv2/src/workbench/invocation.test.ts b/invokeai/frontend/webv2/src/workbench/invocation.test.ts index 94bf1d85d64..268321ef713 100644 --- a/invokeai/frontend/webv2/src/workbench/invocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/invocation.test.ts @@ -305,6 +305,51 @@ describe('resolveInvocationRoute — canvas source', () => { expect(route.validationReasons).toEqual([]); }); + it('allows Canvas preparation for an img2img-only provider while Generate still requires txt2img', () => { + const model: GenerateModelConfig = { + ...unknownExternalModel, + provider_id: 'openai', + capabilities: { modes: ['img2img'] }, + }; + const input = canvasInputFor({ generateValues: createGenerateValues(model) }); + expect(resolveInvocationRouteInput(input, 'global', canvasRoute).sourceValid).toBe(true); + expect( + resolveInvocationRouteInput(input, 'global', { ...canvasRoute, sourceId: 'generate' }).validationReasons + ).toContain('Unknown External Provider does not support text-to-image generation.'); + }); + + it('names unsupported provider control layers in readiness even without a loaded adapter catalog', () => { + const model: GenerateModelConfig = { ...unknownExternalModel, provider_id: 'openai' }; + const layer = modellessControlLayer(); + const input = canvasInputFor({ generateValues: createGenerateValues(model), layers: [layer] }); + expect(resolveInvocationRouteInput(input, 'global', canvasRoute).validationReasons).toEqual([ + 'Unknown External Provider does not support control layer "Control Layer 1". Disable it to generate.', + ]); + expect( + resolveInvocationRouteInput({ ...input, canvasLayers: [{ ...layer, isEnabled: false }] }, 'global', canvasRoute) + .sourceValid + ).toBe(true); + }); + + it('validates provider dimensions against the Canvas bbox rather than the Generate panel size', () => { + const model: GenerateModelConfig = { + ...unknownExternalModel, + provider_id: 'openai', + capabilities: { modes: ['txt2img'], max_image_size: { width: 1024, height: 1024 } }, + }; + const generateValues = createGenerateValues(model, { width: 2048, height: 2048 }); + expect(resolveInvocationRouteInput(canvasInputFor({ generateValues }), 'global', canvasRoute).sourceValid).toBe( + true + ); + expect( + resolveInvocationRouteInput( + canvasInputFor({ generateValues, bbox: { width: 1025, height: 512 } }), + 'global', + canvasRoute + ).validationReasons + ).toContain('Unknown External Provider supports a maximum size of 1024×1024.'); + }); + it('invalidates a zero-area generation frame', () => { const route = resolveInvocationRouteInput( canvasInputFor({ bbox: { height: 0, width: 512 } }), diff --git a/invokeai/frontend/webv2/src/workbench/invocation.ts b/invokeai/frontend/webv2/src/workbench/invocation.ts index cf562fb4756..caff5b2dc2a 100644 --- a/invokeai/frontend/webv2/src/workbench/invocation.ts +++ b/invokeai/frontend/webv2/src/workbench/invocation.ts @@ -26,6 +26,7 @@ import { getInvocationTemplatesSnapshot } from '@features/workflow/react'; import { localizeForLoopValidationReason } from '@features/workflow/utility'; import { areArraysEqual, createStableSelector } from '@platform/state/selectors'; import { compileContributingLayers } from '@workbench/canvas-engine/api'; +import { isCompositableControlLayer, isCompositableRegionalGuidanceLayer } from '@workbench/canvasLayerContent'; import { describeControlLayerIssue, getBlockingControlLayerIssues } from './controlLayerChecks'; import { getActiveProjectGraph } from './projectWorkflows'; @@ -163,7 +164,8 @@ const isWidgetMounted = (input: InvocationRouteInput, widgetId: WidgetId): boole const getGenerateSnapshotValidationReasons = ( generateValues: Record, - models?: readonly ModelConfig[] + models?: readonly ModelConfig[], + canvasBbox?: InvocationRouteInput['canvasBbox'] ): string[] => { const values = normalizeGenerateWidgetValues(generateValues); @@ -171,8 +173,21 @@ const getGenerateSnapshotValidationReasons = ( return ['Generate needs a supported model before it can be invoked.']; } + // Canvas discovers the mode from pixel coverage during preparation and asks a provider for the bbox itself, so + // readiness validates the bbox and permits an advertised whole-image mode; the compiler validates the captured + // one. Generate itself remains strictly text-to-image. + const providerModes = + canvasBbox !== undefined && values.model.type === 'external_image_generator' + ? (values.model.capabilities?.modes ?? ['txt2img']) + : null; + const mode = providerModes !== null && !providerModes.includes('txt2img') ? 'img2img' : 'txt2img'; + return [ - ...getGenerationValidationReasons(values.model, values), + ...getGenerationValidationReasons( + values.model, + providerModes !== null ? { ...values, ...canvasBbox } : values, + mode + ), ...(models ? getGenerationModelAvailabilityReasons(values.model, values, models) : []), ]; }; @@ -243,15 +258,27 @@ export const resolveInvocationRouteInput = ( if (sourceId === 'canvas') { // Canvas shares Generate readiness and additionally requires a nonzero generation frame. - validationReasons.push(...getGenerateSnapshotValidationReasons(input.generateValues, models)); + validationReasons.push(...getGenerateSnapshotValidationReasons(input.generateValues, models, input.canvasBbox)); if (input.canvasBbox.width <= 0 || input.canvasBbox.height <= 0) { validationReasons.push('Canvas generation frame must have a positive area.'); } - // Wait for models before validating control layers to avoid false missing-model errors. - if (models) { - const values = normalizeGenerateWidgetValues(input.generateValues); + const values = normalizeGenerateWidgetValues(input.generateValues); + if (values?.model.type === 'external_image_generator') { + for (const layer of input.canvasLayers) { + if (isCompositableControlLayer(layer)) { + validationReasons.push( + `${values.model.name} does not support control layer "${layer.name}". Disable it to generate.` + ); + } else if (isCompositableRegionalGuidanceLayer(layer)) { + validationReasons.push( + `${values.model.name} does not support regional guidance layer "${layer.name}". Disable it to generate.` + ); + } + } + } else if (models) { + // Wait for models before validating local adapters to avoid false missing-model errors. if (values && isSupportedGenerateModel(values.model)) { validationReasons.push( ...getBlockingControlLayerIssues({ diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts index 44d8b2cda8f..de82bd68792 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts @@ -795,6 +795,91 @@ describe('runCanvasInvocation', () => { expect(harness.releaseRasterSnapshot).toHaveBeenCalledTimes(1); }); + it.each(['control', 'regional'] as const)( + 'rejects contributing external %s layers by name before enqueue', + async (kind) => { + const layer = kind === 'control' ? controlLayer('Canny guide') : regionalLayer('Hat region'); + const harness = makeHarness({ model: externalModel, document: docWithLayers([layer]) }); + await runCanvasInvocation(harness.deps); + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices().map((notice) => notice.message)).toEqual([ + kind === 'control' + ? 'OpenAI Image does not support control layer "Canny guide". Disable it to generate.' + : 'OpenAI Image does not support regional guidance layer "Hat region". Disable it to generate.', + ]); + expect(harness.dedupe.byKey.size).toBe(0); + expect(harness.releaseRasterSnapshot).toHaveBeenCalledTimes(1); + } + ); + + it.each(['control', 'regional'] as const)('ignores disabled external %s layers', async (kind) => { + const layer = kind === 'control' ? controlLayer('Canny guide') : regionalLayer('Hat region'); + const harness = makeHarness({ model: externalModel, document: docWithLayers([{ ...layer, isEnabled: false }]) }); + await runCanvasInvocation(harness.deps); + expect(harness.notices()).toEqual([]); + expect(harness.submittedGraphs()).toHaveLength(1); + }); + + it('submits an img2img-only provider through the real preparation path', async () => { + const model: GenerateModelConfig = { ...externalModel, capabilities: { modes: ['img2img'] } }; + const harness = makeHarness({ model }); + await runCanvasInvocation(harness.deps); + expect(harness.notices()).toEqual([]); + expect(harness.submittedGraphs()).toHaveLength(1); + }); + + it('blocks advertised provider inpainting before enqueue', async () => { + const model: GenerateModelConfig = { ...externalModel, capabilities: { modes: ['inpaint', 'img2img'] } }; + const harness = makeHarness({ model, document: docWithLayers([rasterLayer('base'), inpaintMaskLayer('mask')]) }); + await runCanvasInvocation(harness.deps); + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices().map((notice) => notice.message)).toEqual([ + 'OpenAI Image does not support inpainting on the canvas yet. Remove or disable the inpaint mask.', + ]); + }); + + it('blocks oversized provider requests before enqueue', async () => { + const model: GenerateModelConfig = { + ...externalModel, + capabilities: { modes: ['img2img'], max_image_size: { width: 32, height: 64 } }, + }; + const harness = makeHarness({ model }); + await runCanvasInvocation(harness.deps); + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices().map((notice) => notice.message)).toEqual([ + 'OpenAI Image supports a maximum size of 32×64.', + ]); + }); + + it('counts the captured init image against the Seedream budget before enqueue', async () => { + const model: GenerateModelConfig = { + ...externalModel, + name: 'Seedream', + provider_id: 'seedream', + provider_model_id: 'seedream-4-5-251128', + capabilities: { modes: ['txt2img', 'img2img'], max_reference_images: 14, supports_reference_images: true }, + }; + const harness = makeHarness({ + model, + generateValues: { + ...generateValuesFor(model), + referenceImages: Array.from({ length: 14 }, (_, i) => ({ + id: `reference-${i}`, + isEnabled: true, + config: { + type: 'external_reference_image', + image: { imageName: `reference-${i}.png`, width: 64, height: 64 }, + }, + })), + }, + }); + await runCanvasInvocation(harness.deps); + expect(harness.submittedGraphs()).toHaveLength(0); + expect(harness.notices().map((notice) => notice.message)).toEqual([ + 'Seedream supports at most 15 images total (reference + init + output), got 16.', + ]); + }); + it('discards provisional dedupe entries when synchronous queue dispatch fails', async () => { const dispatch = vi.fn((action: WorkbenchAction) => { if (action.type === 'submitCanvasInvocationSnapshot') { diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts index 4215cfc6b1f..5093165046a 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts @@ -1,7 +1,7 @@ /** * Resolve the active engine, flush paint uploads, compose the fresh snapshot, compile, and enqueue. Invalid - * controls block; rejected regions skip. Publish dedupe only after dispatch succeeds and report failures as - * notices. A per-project in-flight guard prevents overlapping preparation. + * controls and unsupported hosted-provider regions block. Publish dedupe only after dispatch succeeds and report + * failures as notices. A per-project in-flight guard prevents overlapping preparation. */ import type { CanvasScalingSettings, GenerateModelConfig } from '@features/generation/contracts'; @@ -134,6 +134,9 @@ const createControlLayerCollector = ( return { shouldComposite: (layer) => { + if (model.type === 'external_image_generator') { + throw new Error(`${model.name} does not support control layer "${layer.name}". Disable it to generate.`); + } const { adapter } = layer; const resolved = adapter.model ? models?.find((candidate) => candidate.key === adapter.model) : undefined; const rejection = validate({ @@ -238,8 +241,8 @@ export const resolveRegionalReferenceImages = ( }; /** - * Resolve regional references and silently skip unsupported/noncontributing regions. Join uploaded masks with - * captured prompt/reference metadata for graph inputs. + * Reject hosted-provider regions; resolve local references and skip unsupported/noncontributing local regions. + * Join uploaded masks with captured prompt/reference metadata for graph inputs. */ const createRegionalGuidanceCollector = ( model: GenerateModelConfig @@ -251,6 +254,11 @@ const createRegionalGuidanceCollector = ( return { shouldComposite: (layer) => { + if (model.type === 'external_image_generator') { + throw new Error( + `${model.name} does not support regional guidance layer "${layer.name}". Disable it to generate.` + ); + } const referenceImages = resolveRegionalReferenceImages(layer, model.base); const rejection = getRegionalGuidanceRejectionReason({ hasContent: true, diff --git a/tests/app/invocations/test_frontend_external_canvas_graphs.py b/tests/app/invocations/test_frontend_external_canvas_graphs.py new file mode 100644 index 00000000000..ec6431ae241 --- /dev/null +++ b/tests/app/invocations/test_frontend_external_canvas_graphs.py @@ -0,0 +1,58 @@ +"""Validate the frontend's hosted Canvas graphs against the real invocation fields and edge types. + +The fixture is generated by compileCanvasGraph.test.ts. No provider request or model load is needed: +Graph.validate_self checks the collection-to-item conversion and the image/metadata resize inputs. +""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from invokeai.app.services.shared.graph import Graph + +FIXTURE_PATH = ( + Path(__file__).parents[3] + / "invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json" +) +GRAPHS: dict[str, Any] = { + name: graph for name, graph in json.loads(FIXTURE_PATH.read_text(encoding="utf-8")).items() if name != "_comment" +} + + +@pytest.mark.parametrize("mode", ["txt2img", "img2img"]) +def test_hosted_canvas_graph_is_valid_for_the_backend(mode: str) -> None: + graph = Graph.model_validate(GRAPHS[mode]) + graph.validate_self() + for node_id, raw in GRAPHS[mode]["nodes"].items(): + node = graph.nodes[node_id] + unknown = set(raw) - set(type(node).model_fields) + if type(node).model_config.get("extra") == "allow": + # Core metadata intentionally preserves provider-specific fields beyond its declared schema. + assert {key: node.model_dump()[key] for key in unknown} == {key: raw[key] for key in unknown} + else: + assert not unknown, f"{raw['type']} has no field(s) {sorted(unknown)}" + + provider = graph.nodes["canvas_processing_output"] + assert provider.mode == mode + assert (provider.init_image is not None) == (mode == "img2img") + assert provider.num_images == 1 + assert provider.is_intermediate + output = graph.nodes["canvas_output"] + assert output.get_type() == "img_resize" + assert (output.width, output.height) == (640, 512) + + +@pytest.mark.parametrize("mode", ["txt2img", "img2img"]) +def test_resized_image_retains_the_provider_output_metadata(mode: str) -> None: + graph = Graph.model_validate(GRAPHS[mode]) + sources = {(edge.destination.node_id, edge.destination.field): edge.source for edge in graph.edges} + output_metadata_source = sources[("canvas_output", "metadata")] + metadata_reader = graph.nodes[output_metadata_source.node_id] + assert metadata_reader.get_type() == "metadata_from_image" + assert sources[(metadata_reader.id, "image")] == sources[("canvas_output", "image")] + # Core dimensions enter the provider, which adds its authoritative request identity, seed and options. + provider_metadata = graph.nodes[sources[("canvas_processing_output", "metadata")].node_id] + assert provider_metadata.get_type() == "core_metadata" + assert (provider_metadata.width, provider_metadata.height) == (640, 512) From c54124d1be04c491905bff63c3872f010805c95d Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sat, 17 Oct 2026 02:36:37 -0600 Subject: [PATCH 06/10] fix(models): lock catalog-managed external records and restore timing baselines The External Provider editor keeps a starter-sourced record fully read-only, since the backend overlays the catalog entry matched by source on every response; edits wait for the catalog lookup, with Retry on failure, and a retained draft is shown but cannot be saved. The saved provider stays selectable after switching. Browser timing baselines return to the merge-base values; only bytes and source sets differ. --- .../Users Guide/03.Models/01.introduction.mdx | 11 +- .../03.Models/External Models/openai.mdx | 2 +- .../03.Models/External Models/seedream.mdx | 2 +- .../docs/contributing/external-providers.md | 2 + .../webv2/performance/browser-baseline.json | 110 +++++++++--------- .../frontend/webv2/public/locales/en.json | 9 +- .../models/core/externalModelFields.test.ts | 10 -- .../models/core/externalModelFields.ts | 8 -- .../ExternalModelSection.browser.test.tsx | 104 +++++++++++++++-- .../models/ui/detail/ExternalModelSection.tsx | 71 +++++++---- 10 files changed, 213 insertions(+), 116 deletions(-) diff --git a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx index 79f031fe038..a59840a4415 100644 --- a/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/01.introduction.mdx @@ -106,9 +106,14 @@ inputs (seed, reference images, negative prompt) it advertises, its request limits, and the size and image count the provider is asked for by default. The Generate panel and the Canvas only offer what the record advertises, so keep it in step with the provider's documentation. -A model installed from the starter catalog keeps the catalog's -capabilities and defaults, so those are shown read-only; its provider -and provider model ID remain editable. +The Canvas compiles only text to image and image to image for external +models and refuses inpaint, because its keep-mask does not match any +provider's edit-mask contract; an advertised inpaint mode serves the +workflow editor. A model whose source matches a starter catalog entry is +shown read-only, including its provider and provider model ID, because +Invoke applies that catalog entry's capabilities to it on every load, so +a changed route would keep the original entry's capabilities. Models +added outside the starter catalog get the full editor. :::note[Gated Downloads] The API Keys section is also used to provide keys for services that diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx index 9e20bb9c052..c90036d3832 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx @@ -56,7 +56,7 @@ The **Size** section lists the provider's fixed sizes per aspect ratio (**Provid ## Canvas -GPT Image models that advertise `img2img` can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider edits the composite whole at the bounding box's own size, so the render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available for OpenAI models. +GPT Image models that advertise `img2img` can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider is asked for the bounding box's size but buckets the request to its own size table, resizing the init image with it, and Invoke scales the returned image back to the exact bounding box before staging or saving it. The render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available for OpenAI models. ## How Requests Are Routed diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx index 2ce2095d2f7..7cdae42920b 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx @@ -57,7 +57,7 @@ Both are recorded in the image metadata and restored by **Recall All** and **Rem ## Canvas -Seedream models advertise `img2img`, so they can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider edits the composite whole at the bounding box's own size, so the render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available. +Seedream models advertise `img2img`, so they can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider is asked for the bounding box's size but buckets the request to its own size table, resizing the init image with it, and Invoke scales the returned image back to the exact bounding box before staging or saving it. The render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available. **Seed** and **guidance scale** are not accepted by the 4.x / 5.x family. diff --git a/docs/src/content/docs/contributing/external-providers.md b/docs/src/content/docs/contributing/external-providers.md index ab62fb886e4..05ecbdc86bf 100644 --- a/docs/src/content/docs/contributing/external-providers.md +++ b/docs/src/content/docs/contributing/external-providers.md @@ -81,12 +81,14 @@ Starter overrides are matched by `source` (`external://provider/model-id`). Keep - runtime capability/default overrides depend on it - installation detection in starter-model APIs depends on it +- the model manager's External Provider section treats a record whose `source` matches a starter as catalog-managed and shows its routing, capabilities and defaults read-only, because a changed route would keep the matched entry's capabilities `STARTER_MODELS` enforces unique `source` values with an assertion. ### Install behavior notes - External starter models are managed in **External Providers** setup (not the regular Starter Models tab). +- Canvas compiles `txt2img` and `img2img` for external models and restores the provider's bucketed output to the bounding box; `inpaint` is refused on Canvas because its keep-mask does not match any provider's edit-mask contract (the workflow editor nodes still accept `mask_image`). - External starter models auto-install when a provider is configured. - Removing a provider API key removes installed external models for that provider. diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index 5f09e048374..ebd3b8936e4 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -14,13 +14,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 159.79999999701977, + "domContentLoadedMedianMs": 130.60000000149012, "id": "launchpad", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 159.89999999850988, - "longestTaskMaxMs": 104, + "loadMedianMs": 130.79999999701977, + "longestTaskMaxMs": 82, "owner": "app", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:launchpad", @@ -36,7 +36,7 @@ "scriptRequestCount": 26, "totalRawBytes": 2078043 }, - "routeReadyMedianMs": 334.79999999701977, + "routeReadyMedianMs": 275.80000000447035, "scriptSourceOwnerSet": "launchpad-static", "stateProfile": "empty" }, @@ -52,13 +52,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 167.5, + "domContentLoadedMedianMs": 125.10000000149012, "id": "launchpad", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 167.70000000298023, - "longestTaskMaxMs": 105, + "loadMedianMs": 125.20000000298023, + "longestTaskMaxMs": 81, "owner": "app", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:launchpad", @@ -74,7 +74,7 @@ "scriptRequestCount": 26, "totalRawBytes": 2078043 }, - "routeReadyMedianMs": 353.1000000014901, + "routeReadyMedianMs": 277, "scriptSourceOwnerSet": "launchpad-static", "stateProfile": "representative" }, @@ -90,15 +90,15 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 156.20000000298023, + "domContentLoadedMedianMs": 125.5, "id": "editor-minimal", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 156.39999999850988, - "longestTaskMaxMs": 240, + "loadMedianMs": 125.70000000298023, + "longestTaskMaxMs": 173, "owner": "workbench", - "projectSwitchMedianMs": 55.19999999552965, + "projectSwitchMedianMs": 38.30000000447035, "readyMark": "invokeai:ready:widget:center:preview", "remediationTicket": "deepen-widget-registry-loading", "resourceBaseline": { @@ -112,7 +112,7 @@ "scriptRequestCount": 83, "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 549.5, + "routeReadyMedianMs": 430.19999999552965, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "empty" }, @@ -128,15 +128,15 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 152.60000000149012, + "domContentLoadedMedianMs": 129.60000000149012, "id": "editor-minimal", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 152.79999999701977, - "longestTaskMaxMs": 232, + "loadMedianMs": 129.69999999552965, + "longestTaskMaxMs": 180, "owner": "workbench", - "projectSwitchMedianMs": 58.899999998509884, + "projectSwitchMedianMs": 44.099999994039536, "readyMark": "invokeai:ready:widget:center:preview", "remediationTicket": "deepen-widget-registry-loading", "resourceBaseline": { @@ -150,7 +150,7 @@ "scriptRequestCount": 83, "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 569, + "routeReadyMedianMs": 466.3999999985099, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "representative" }, @@ -166,13 +166,13 @@ "scriptRequestCount": 9, "totalRawBytes": 515192 }, - "domContentLoadedMedianMs": 146.89999999850988, + "domContentLoadedMedianMs": 125.5, "id": "editor-canvas", - "layoutAckMedianMs": 3.399999998509884, - "layoutReturnSwitchMedianMs": 65.5, - "layoutSwitchMedianMs": 89.30000000447035, - "loadMedianMs": 147.09999999403954, - "longestTaskMaxMs": 211, + "layoutAckMedianMs": 2.6999999955296516, + "layoutReturnSwitchMedianMs": 57.899999998509884, + "layoutSwitchMedianMs": 76.60000000149012, + "loadMedianMs": 125.60000000149012, + "longestTaskMaxMs": 179, "owner": "canvas", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:canvas", @@ -188,7 +188,7 @@ "scriptRequestCount": 92, "totalRawBytes": 4889136 }, - "routeReadyMedianMs": 1295.3999999985099, + "routeReadyMedianMs": 1153.7999999970198, "scriptSourceOwnerSet": "editor-canvas-static", "stateProfile": "empty" }, @@ -204,13 +204,13 @@ "scriptRequestCount": 9, "totalRawBytes": 515192 }, - "domContentLoadedMedianMs": 126.19999999552965, + "domContentLoadedMedianMs": 126.59999999403954, "id": "editor-canvas", "layoutAckMedianMs": 3.100000001490116, - "layoutReturnSwitchMedianMs": 53.899999998509884, - "layoutSwitchMedianMs": 78.5, - "loadMedianMs": 126.29999999701977, - "longestTaskMaxMs": 205, + "layoutReturnSwitchMedianMs": 52.70000000298023, + "layoutSwitchMedianMs": 72.20000000298023, + "loadMedianMs": 126.69999999552965, + "longestTaskMaxMs": 196, "owner": "canvas", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:canvas", @@ -226,7 +226,7 @@ "scriptRequestCount": 92, "totalRawBytes": 4889136 }, - "routeReadyMedianMs": 1172, + "routeReadyMedianMs": 1147.1000000014901, "scriptSourceOwnerSet": "editor-canvas-static", "stateProfile": "representative" }, @@ -242,13 +242,13 @@ "scriptRequestCount": 6, "totalRawBytes": 436338 }, - "domContentLoadedMedianMs": 126.89999999850988, + "domContentLoadedMedianMs": 125, "id": "editor-workflow", - "layoutAckMedianMs": 3, - "layoutReturnSwitchMedianMs": 39.600000001490116, - "layoutSwitchMedianMs": 74.90000000596046, - "loadMedianMs": 127, - "longestTaskMaxMs": 172, + "layoutAckMedianMs": 2.7000000029802322, + "layoutReturnSwitchMedianMs": 38.600000001490116, + "layoutSwitchMedianMs": 82.60000000149012, + "loadMedianMs": 125.09999999403954, + "longestTaskMaxMs": 174, "owner": "workflow", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:workflow", @@ -264,7 +264,7 @@ "scriptRequestCount": 89, "totalRawBytes": 4810282 }, - "routeReadyMedianMs": 1176.7999999970198, + "routeReadyMedianMs": 1171.6000000014901, "scriptSourceOwnerSet": "editor-workflow-static", "stateProfile": "empty" }, @@ -280,13 +280,13 @@ "scriptRequestCount": 6, "totalRawBytes": 436338 }, - "domContentLoadedMedianMs": 125, + "domContentLoadedMedianMs": 123.70000000298023, "id": "editor-workflow", - "layoutAckMedianMs": 2.899999998509884, - "layoutReturnSwitchMedianMs": 43.100000001490116, - "layoutSwitchMedianMs": 132.20000000298023, - "loadMedianMs": 125, - "longestTaskMaxMs": 276, + "layoutAckMedianMs": 2.6999999955296516, + "layoutReturnSwitchMedianMs": 41.80000000447035, + "layoutSwitchMedianMs": 128.80000000447035, + "loadMedianMs": 123.80000000447035, + "longestTaskMaxMs": 259, "owner": "workflow", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:workflow", @@ -302,7 +302,7 @@ "scriptRequestCount": 89, "totalRawBytes": 4810282 }, - "routeReadyMedianMs": 1216.2000000029802, + "routeReadyMedianMs": 1192.8999999985099, "scriptSourceOwnerSet": "editor-workflow-static", "stateProfile": "representative" }, @@ -318,13 +318,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 126.59999999403954, + "domContentLoadedMedianMs": 125.79999999701977, "id": "editor-gallery", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, - "layoutSwitchMedianMs": 39.79999999701977, - "loadMedianMs": 126.59999999403954, - "longestTaskMaxMs": 178, + "layoutSwitchMedianMs": 41.100000001490116, + "loadMedianMs": 125.89999999850988, + "longestTaskMaxMs": 174, "owner": "gallery", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:gallery", @@ -340,7 +340,7 @@ "scriptRequestCount": 83, "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 1420.6000000014901, + "routeReadyMedianMs": 1431.3999999985099, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "empty" }, @@ -356,13 +356,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 126.20000000298023, + "domContentLoadedMedianMs": 126.89999999850988, "id": "editor-gallery", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, - "layoutSwitchMedianMs": 46.80000000447035, - "loadMedianMs": 126.39999999850988, - "longestTaskMaxMs": 171, + "layoutSwitchMedianMs": 45.399999998509884, + "loadMedianMs": 127.09999999403954, + "longestTaskMaxMs": 170, "owner": "gallery", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:gallery", @@ -378,7 +378,7 @@ "scriptRequestCount": 83, "totalRawBytes": 4373944 }, - "routeReadyMedianMs": 1411.6000000014901, + "routeReadyMedianMs": 1396.3999999985099, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "representative" } diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 7a181cab65e..d2477919144 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -1348,7 +1348,7 @@ "emptyCacheConfirmBody": "Unloads all models from RAM and VRAM. The next generation reloads them from disk.", "emptyCacheConfirmTitle": "Empty model cache?", "providerId": "Provider", - "providerIdHelp": "The provider whose API key and endpoint handle this model (e.g. openai, gemini, seedream).", + "providerIdHelp": "Every registered provider, configured or not, plus this record's saved provider. Configure the provider's API key before generating.", "providerModelId": "Provider Model ID", "providerModelIdHelp": "The model name the provider expects in requests.", "externalProvider": "External Provider", @@ -1363,7 +1363,7 @@ }, "externalFields": { "supportedModes": "Supported modes", - "supportedModesHelp": "Image to image and inpaint let the canvas send its composite to the provider.", + "supportedModesHelp": "The modes the provider accepts. Canvas supports text to image and image to image; provider masks for inpaint are not yet supported.", "supportsSeed": "Accepts a seed", "supportsReferenceImages": "Accepts reference images", "supportsNegativePrompt": "Accepts a negative prompt", @@ -1378,7 +1378,10 @@ "providerDefault": "Provider default", "supported": "Supported", "notSupported": "Not supported", - "starterManaged": "Managed by the starter catalog; the provider IDs can still be edited." + "starterManaged": "Managed by the starter catalog. Routing, capabilities, and defaults cannot be edited here.", + "lockedDraftBlocked": "These unsaved values are not the effective configuration and cannot be saved while this record is locked. Reset to discard them.", + "catalogUnavailable": "Could not check the starter catalog. Editing is unavailable until this check succeeds.", + "catalogLoading": "Checking the starter catalog before enabling edits…" }, "queueAttentionCount_one": "{{count}} needs attention", "queueAttentionCount_other": "{{count}} need attention", diff --git a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts index 09f09f8c083..613debb5809 100644 --- a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts @@ -6,7 +6,6 @@ import { isStarterManagedExternalModel, toExternalModelChanges, toExternalModelValues, - toExternalProviderChanges, validateExternalModel, } from './externalModelFields'; @@ -144,13 +143,4 @@ describe('external model record mapping', () => { ).toBe(false); expect(isStarterManagedExternalModel({ provider_id: 'custom', provider_model_id: 'mine' }, [])).toBe(false); }); - - it('sends only the routing ids for a starter-managed record', () => { - const values = toExternalModelValues(geminiModel); - - expect(toExternalProviderChanges({ ...values, providerId: ' gemini ', providerModelId: 'flash ' })).toEqual({ - provider_id: 'gemini', - provider_model_id: 'flash', - }); - }); }); diff --git a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts index f1a1760c02c..f24e0d6eac6 100644 --- a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts @@ -92,14 +92,6 @@ export const toExternalModelChanges = ( }; }; -/** The PATCH body for a starter-managed record: only the routing ids persist, so only they are sent. */ -export const toExternalProviderChanges = ( - values: ExternalModelFormValues -): Pick => ({ - provider_id: values.providerId.trim(), - provider_model_id: values.providerModelId.trim(), -}); - /** The first schema message, like the other record editors' messages, so the UI shows it as is. */ export const validateExternalModel = (values: ExternalModelFormValues): string | null => { const result = externalModelSchema.safeParse(values); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx index ba0f425bbd4..60b2fde501d 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx @@ -2,7 +2,8 @@ import type { ExternalModelSettingsModel } from '@features/models/core/externalM import type * as ModelsApi from '@features/models/data/api'; import { ChakraProvider } from '@chakra-ui/react'; -import { getModelDraft } from '@features/models/ui/modelDraftsStore'; +import { toExternalModelValues } from '@features/models/core/externalModelFields'; +import { recordModelDraftFields, getModelDraft } from '@features/models/ui/modelDraftsStore'; import { accountLifecycle } from '@platform/state/accountLifecycle'; import { system } from '@theme/system'; import { act } from 'react'; @@ -114,6 +115,7 @@ describe('ExternalModelSection', () => { beforeEach(() => { api.getExternalProviderConfigs.mockClear(); + api.getStarterModels.mockClear(); api.updateModel.mockReset(); onError.mockReset(); onSaved.mockReset(); @@ -198,7 +200,7 @@ describe('ExternalModelSection', () => { expect(checkbox('models.externalModes.txt2img').checked).toBe(true); }); - it('offers the registered providers, keeping an unregistered id of the record selectable', async () => { + it('keeps the saved unregistered provider selectable after switching without discarding other edits', async () => { await render({ ...gptImage, provider_id: 'custom' }); await settle(); @@ -213,7 +215,17 @@ describe('ExternalModelSection', () => { await settle(); expect(providerTrigger()?.textContent).toContain('gemini'); - expect(getModelDraft('gpt-image-1')?.external?.fields).toEqual({ providerId: 'gemini' }); + await type(input('models.externalFields.maxReferenceImages'), '4'); + await click(providerTrigger()!); + await settle(); + const savedProvider = [...document.querySelectorAll('[role="option"]')].find( + (option) => option.textContent === 'custom' + ); + expect(savedProvider).toBeDefined(); + await click(savedProvider!); + await settle(); + expect(providerTrigger()?.textContent).toContain('custom'); + expect(getModelDraft('gpt-image-1')?.external?.fields).toEqual({ maxReferenceImages: 4 }); }); it('keeps the provider a text field while the provider list is unavailable', async () => { @@ -227,7 +239,7 @@ describe('ExternalModelSection', () => { expect(getModelDraft('gpt-image-1')?.external?.fields).toEqual({ providerId: 'seedream' }); }); - it('shows a starter-managed record read-only and saves only its routing ids', async () => { + it('locks routing and capability edits for a starter-managed record', async () => { const geminiFlash: ExternalModelSettingsModel = { ...gptImage, capabilities: { max_images_per_request: 1, modes: ['txt2img'], supports_reference_images: true }, @@ -247,15 +259,87 @@ describe('ExternalModelSection', () => { expect(host.textContent).toContain('models.externalModes.txt2img'); expect(host.textContent).toContain('1024'); - await type(host.querySelector('input[value="gemini-2.5-flash-image"]')!, 'gemini-3-pro'); + expect(providerTrigger()?.disabled).toBe(true); + expect(host.querySelector('input[value="gemini-2.5-flash-image"]')?.disabled).toBe(true); + expect(button('models.saveExternalSettings').disabled).toBe(true); await click(button('models.saveExternalSettings')); + expect(api.updateModel).not.toHaveBeenCalled(); + }); - expect(api.updateModel).toHaveBeenCalledWith( - 'gemini-flash', - { provider_id: 'gemini', provider_model_id: 'gemini-3-pro' }, - expect.anything() + it('blocks edits while catalog discovery loads or fails and enables custom records after Retry', async () => { + let rejectCatalog!: (error: Error) => void; + api.getStarterModels.mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + rejectCatalog = reject; + }) ); - expect(onSaved).toHaveBeenCalledOnce(); + await render(gptImage); + expect(host.querySelector('[role="status"]')?.textContent).toContain('models.externalFields.catalogLoading'); + expect(providerTrigger()?.disabled).toBe(true); + expect(host.querySelector('input[value="gpt-image-1"]')?.disabled).toBe(true); + expect(host.querySelectorAll('input[type="checkbox"]')).toHaveLength(0); + + await act(() => rejectCatalog(new Error('offline'))); + expect(host.querySelector('[role="alert"]')?.textContent).toContain('models.externalFields.catalogUnavailable'); + expect(button('models.saveExternalSettings').disabled).toBe(true); + expect(host.querySelectorAll('input[type="checkbox"]')).toHaveLength(0); + expect(getModelDraft('gpt-image-1')?.external).toBeUndefined(); + + await click(button('common.retry')); + await settle(); + expect(host.querySelector('[role="alert"]')).toBeNull(); + expect(providerTrigger()?.disabled).toBe(false); + expect(checkbox('models.externalModes.txt2img').checked).toBe(true); + expect(getModelDraft('gpt-image-1')?.external).toBeUndefined(); + }); + + it('explains a retained draft while a failed catalog fetch keeps the record locked', async () => { + api.getStarterModels.mockRejectedValueOnce(new Error('offline')); + const saved = toExternalModelValues(gptImage); + recordModelDraftFields(gptImage.key, 'external', { ...saved, providerModelId: 'draft-model' }, saved); + await render(gptImage); + + expect(host.querySelector('[role="alert"]')?.textContent).toContain('models.externalFields.catalogUnavailable'); + expect(host.querySelector('[role="note"]')?.textContent).toBe('models.externalFields.lockedDraftBlocked'); + expect(host.querySelector('input[value="draft-model"]')?.disabled).toBe(true); + expect(button('models.saveExternalSettings').disabled).toBe(true); + expect(button('common.reset')).toBeDefined(); + }); + + it('keeps a retained draft visible but unsaveable when retry discovers a catalog-managed record', async () => { + api.getStarterModels.mockRejectedValueOnce(new Error('offline')); + api.getStarterModels.mockResolvedValueOnce({ + starter_bundles: {}, + starter_models: [{ source: gptImage.source! }], + }); + const saved = toExternalModelValues(gptImage); + recordModelDraftFields( + gptImage.key, + 'external', + { + ...saved, + providerModelId: 'draft-model', + maxImagesPerRequest: 7, + }, + saved + ); + await render(gptImage); + expect(button('models.saveExternalSettings').disabled).toBe(true); + await click(button('common.retry')); + await settle(); + + expect(host.textContent).toContain('models.externalFields.lockedDraftBlocked'); + expect(host.querySelector('input[value="draft-model"]')?.disabled).toBe(true); + expect(host.textContent).toContain('7'); + expect(button('models.saveExternalSettings').disabled).toBe(true); + await click(button('models.saveExternalSettings')); + expect(api.updateModel).not.toHaveBeenCalled(); + await click(button('common.reset')); + expect(getModelDraft(gptImage.key)?.external).toBeUndefined(); + expect(host.textContent).not.toContain('models.externalFields.lockedDraftBlocked'); + expect(host.querySelector('input[value="gpt-image-1"]')?.disabled).toBe(true); + expect(host.textContent).toContain('10'); }); it('surfaces a failed save and keeps the edits for another attempt', async () => { diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx index 0d500576c10..21005e1aeca 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx @@ -17,7 +17,6 @@ import { isStarterManagedExternalModel, toExternalModelChanges, toExternalModelValues, - toExternalProviderChanges, validateExternalModel, } from '@features/models/core/externalModelFields'; import { updateModel } from '@features/models/data/api'; @@ -26,7 +25,7 @@ import { useExternalProvidersSelector, } from '@features/models/data/externalProvidersStore'; import { replaceModelInStore } from '@features/models/data/modelsStore'; -import { ensureStartersLoaded, useStartersSelector } from '@features/models/data/startersStore'; +import { ensureStartersLoaded, refreshStarters, useStartersSelector } from '@features/models/data/startersStore'; import { applyModelDraftFields, beginModelDraftSave, @@ -54,9 +53,9 @@ import { DraftConflictNotice, UnsavedChangesLabel } from './ModelDraftNotices'; /** * Edits the hosted-provider record: which provider and model ID requests route to, the modes and inputs the model * advertises, its limits, and the size/count the provider is asked for by default. A record the starter catalog - * manages shows its capabilities and defaults read-only, since the backend reapplies the catalog's on every - * response; its routing ids still persist. Unsaved values live in the model's retained draft; `externalModelFields` - * owns the record mapping and validation. + * manages stays read-only: the backend overlays the catalog entry matched by `source` on every response, so a + * changed route would keep the original entry's capabilities. Unsaved values live in the model's retained draft; + * `externalModelFields` owns the record mapping and validation. */ const selectExternalDraft = (draft: ModelDraft | undefined) => draft?.external; @@ -156,10 +155,9 @@ export const ExternalModelSection = ({ const providerConfigs = useExternalProvidersSelector((snapshot) => snapshot.configs); const starters = useStartersSelector((snapshot) => snapshot.response?.starter_models ?? null); const startersStatus = useStartersSelector((snapshot) => snapshot.status); - // Read-only until the catalog says otherwise, so an edit never flips under the user; a catalog that cannot be - // loaded lets the editor through rather than locking custom records. - const isStarterManaged = - starters === null ? startersStatus !== 'error' : isStarterManagedExternalModel(model, starters); + // A failed lookup cannot establish that edits persist: the backend may still overlay catalog metadata. + const isStarterManaged = starters !== null && isStarterManagedExternalModel(model, starters); + const canEdit = starters !== null && !isStarterManaged; // The registered providers are what requests can route to; a record's own id stays offered even when it is not // among them, so an unknown provider reads as what it is rather than as nothing. const providerCollection = useMemo( @@ -167,11 +165,13 @@ export const ExternalModelSection = ({ providerConfigs === null ? null : createListCollection({ - items: [...new Set([...providerConfigs.map((config) => config.provider_id), values.providerId])] + items: [ + ...new Set([...providerConfigs.map((config) => config.provider_id), saved.providerId, values.providerId]), + ] .filter((id) => id !== '') .map((id) => ({ label: id, value: id })), }), - [providerConfigs, values.providerId] + [providerConfigs, saved.providerId, values.providerId] ); // From the draft, not this instance: a save started before the user navigated away may still be in flight. const isSaving = draft?.submitted !== undefined; @@ -194,6 +194,10 @@ export const ExternalModelSection = ({ }; const handleSave = async () => { + if (!canEdit) { + return; + } + const validationError = validateExternalModel(values); if (validationError) { @@ -209,11 +213,7 @@ export const ExternalModelSection = ({ return; } - const updated = await updateModel( - model.key, - isStarterManaged ? toExternalProviderChanges(submitted) : toExternalModelChanges(submitted, model), - owner.signal - ); + const updated = await updateModel(model.key, toExternalModelChanges(submitted, model), owner.signal); assertAccountScopeCurrent(owner); replaceModelInStore(updated); @@ -256,7 +256,7 @@ export const ExternalModelSection = ({ ) : null} - @@ -267,13 +267,34 @@ export const ExternalModelSection = ({ ) : null} {hasModelDraftConflict(saved, draft) ? : null} + {starters === null ? ( + + + {t( + startersStatus === 'error' + ? 'models.externalFields.catalogUnavailable' + : 'models.externalFields.catalogLoading' + )} + + {startersStatus === 'error' ? ( + + ) : null} + + ) : null} + {!canEdit && isDirty ? ( + + {t('models.externalFields.lockedDraftBlocked')} + + ) : null} {providerCollection ? ( setField('providerId', event.currentTarget.value)} @@ -294,14 +315,14 @@ export const ExternalModelSection = ({ setField('providerModelId', event.currentTarget.value)} /> - {isStarterManaged ? ( + {!canEdit ? ( {starters === null ? null : ( @@ -311,7 +332,7 @@ export const ExternalModelSection = ({ saved[key]) + value={MODE_FLAGS.filter(({ key }) => values[key]) .map(({ labelKey }) => t(labelKey)) .join(', ')} /> @@ -319,21 +340,21 @@ export const ExternalModelSection = ({ ))} {LIMIT_FIELDS.map(({ key, labelKey }) => ( ))} {DEFAULT_FIELDS.map(({ key, labelKey }) => ( ))} From 4f815014fc1e43867d435602458d7218ec7431c4 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Mon, 19 Oct 2026 14:53:12 -0600 Subject: [PATCH 07/10] fix(generation): keep hosted Canvas output undistorted and address review Canvas requests a provider size from the model's own table, then covers and center-crops text-to-image output to the frame; Gemini sends its covering preset. Size snaps on model switch are reported as adjustments, text-only hosted models block on frame content, and the record editor shows readable save errors, keeps Retry focus and marks invalid fields. Browser timing baselines return to main's values; graph and starter-size tests exercise the backend pass-through. --- .../03.Models/External Models/gemini.mdx | 4 + .../03.Models/External Models/openai.mdx | 8 +- .../03.Models/External Models/seedream.mdx | 2 +- .../docs/contributing/external-providers.md | 2 +- .../performance/architecture-baseline.json | 1 + .../webv2/performance/browser-baseline.json | 119 ++++++----- .../frontend/webv2/public/locales/en.json | 3 + .../webv2/scripts/mock-backend-fixtures.mjs | 33 ++- .../scripts/mock-backend-fixtures.test.mjs | 59 ++++++ .../frontend/webv2/scripts/mock-backend.mjs | 100 ++++++++- .../__snapshots__/externalCanvasGraphs.json | 162 +++++++++++--- .../core/baseGenerationPolicies.test.ts | 64 +++++- .../generation/core/baseGenerationPolicies.ts | 21 +- .../core/canvas/canvasProcessingSize.test.ts | 18 +- .../core/canvas/canvasProcessingSize.ts | 15 +- .../core/canvas/compileCanvasGraph.test.ts | 146 ++++++++++++- .../core/canvas/compileCanvasGraph.ts | 92 ++++++-- .../generation/core/dimensionLimits.ts | 3 + .../generation/core/externalProvider.test.ts | 145 ++++++++++++- .../generation/core/externalProvider.ts | 143 ++++++++++++- .../features/generation/core/graph.test.ts | 8 +- .../src/features/generation/core/graph.ts | 14 +- .../src/features/generation/core/settings.ts | 3 +- .../webv2/src/features/generation/react.ts | 2 +- .../webv2/src/features/generation/settings.ts | 4 + .../generation/ui/GenerateDimensionFields.tsx | 3 +- .../generation/ui/GenerateModelCard.tsx | 63 ++++-- .../GeneratePresetsPopover.browser.test.tsx | 30 +++ .../generation/ui/GeneratePresetsPopover.tsx | 3 +- ...ateProviderOptionsSection.browser.test.tsx | 12 +- .../ui/GenerateSettingsForm.browser.test.tsx | 25 ++- .../ui/modelSelectionNotice.test.ts | 38 +++- .../generation/ui/modelSelectionNotice.ts | 27 +++ .../models/core/externalModelFields.test.ts | 32 ++- .../models/core/externalModelFields.ts | 18 +- .../ExternalModelSection.browser.test.tsx | 134 +++++++++++- .../models/ui/detail/ExternalModelSection.tsx | 151 ++++++++++--- .../models/ui/modelDraftsStore.test.ts | 23 ++ .../image-actions/imageRecall.test.ts | 1 + .../webv2/src/workbench/invocation.test.ts | 84 ++++++++ .../webv2/src/workbench/invocation.ts | 86 ++++++-- .../palette/WorkbenchCommandPaletteDialog.tsx | 7 +- .../canvas/GenerateCanvasRenderSize.tsx | 25 +-- .../invoke/prepareCanvasInvocation.test.ts | 8 +- .../canvas/invoke/prepareCanvasInvocation.ts | 12 +- .../test_frontend_external_canvas_graphs.py | 199 ++++++++++++++++-- .../test_external_generation_service.py | 32 ++- 47 files changed, 1882 insertions(+), 302 deletions(-) create mode 100644 invokeai/frontend/webv2/src/features/generation/core/dimensionLimits.ts diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx index 1573a60a242..ee09997d3b0 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/gemini.mdx @@ -48,6 +48,10 @@ Selecting a Gemini model adds a **Provider options** section to the Generate pan Gemini 3 models list their resolution presets under **Provider size** in the Size section. Picking one sets the width and height to the preset and sends its named size (`1K`, `2K`, `4K`) to the API. The named size and the sampling options are recorded in the image metadata and restored by **Recall All** and **Remix**. +## Canvas + +Gemini models generate only from text, so on the Canvas the bounding box must be free of raster content: hide raster layers or move their content out of the box to generate. Invoke asks for the size whose aspect ratio is closest to the bounding box's; for Gemini 3 that is the smallest named size covering the box at that ratio, such as `2K` for a 2000×2000 box. The result is scaled to cover the bounding box and cropped to it from the center, so it is never stretched. + ## Tips diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx index c90036d3832..7081175ccc0 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/openai.mdx @@ -44,19 +44,23 @@ Inpainting (mask-based editing) is not currently supported for any OpenAI model ## Provider-Specific Options -Selecting an OpenAI model adds a **Provider options** section to the Generate panel: +Selecting a GPT Image model adds a **Provider options** section to the Generate panel: - **Quality** — `low`, `medium`, `high`, or `auto`. Higher quality costs more and takes longer. - **Background** — `auto`, `transparent`, or `opaque`. Use `transparent` for PNG output with an alpha channel. - **Input fidelity** — `low` or `high`, offered only for models that take image inputs (img2img or reference images). Leave it on the provider default to let OpenAI choose. +DALL·E 2 and DALL·E 3 do not expose these options. + These are recorded in the image metadata, so **Recall All** and **Remix** restore them when the image's model is selected. The **Size** section lists the provider's fixed sizes per aspect ratio (**Provider size**). Any other size is mapped to the closest one the provider accepts. ## Canvas -GPT Image models that advertise `img2img` can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider is asked for the bounding box's size but buckets the request to its own size table, resizing the init image with it, and Invoke scales the returned image back to the exact bounding box before staging or saving it. The render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available for OpenAI models. +GPT Image models that advertise `img2img` can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. Invoke asks for the provider size whose aspect ratio is closest to the bounding box's, such as 1536×1024 for a 640×512 box. A text-to-image result is scaled to cover the bounding box and cropped to it from the center, so it is never stretched; when the shapes differ, a strip at two opposite edges is cut off. For image-to-image, the init image is resized to the provider size and the result is resized back to the exact bounding box. The render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available for OpenAI models. + +DALL·E 3 generates only from text: on the Canvas, hide raster layers or move their content out of the bounding box to generate with it. ## How Requests Are Routed diff --git a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx index 7cdae42920b..800052bf362 100644 --- a/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/External Models/seedream.mdx @@ -57,7 +57,7 @@ Both are recorded in the image metadata and restored by **Recall All** and **Rem ## Canvas -Seedream models advertise `img2img`, so they can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. The provider is asked for the bounding box's size but buckets the request to its own size table, resizing the init image with it, and Invoke scales the returned image back to the exact bounding box before staging or saving it. The render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available. +Seedream models advertise `img2img`, so they can generate from the Canvas: an empty bounding box runs text-to-image, and a filled one sends the composited bounding box as the init image. Invoke asks for the provider size whose aspect ratio is closest to the bounding box's. A text-to-image result is scaled to cover the bounding box and cropped to it from the center, so it is never stretched; when the shapes differ, a strip at two opposite edges is cut off. For image-to-image, the init image is resized to the provider size and the result is resized back to the exact bounding box. The render size, denoising strength, control layers and regional guidance do not apply and are not offered. Inpainting and outpainting are not available. **Seed** and **guidance scale** are not accepted by the 4.x / 5.x family. diff --git a/docs/src/content/docs/contributing/external-providers.md b/docs/src/content/docs/contributing/external-providers.md index 05ecbdc86bf..0b132d69004 100644 --- a/docs/src/content/docs/contributing/external-providers.md +++ b/docs/src/content/docs/contributing/external-providers.md @@ -88,7 +88,7 @@ Starter overrides are matched by `source` (`external://provider/model-id`). Keep ### Install behavior notes - External starter models are managed in **External Providers** setup (not the regular Starter Models tab). -- Canvas compiles `txt2img` and `img2img` for external models and restores the provider's bucketed output to the bounding box; `inpaint` is refused on Canvas because its keep-mask does not match any provider's edit-mask contract (the workflow editor nodes still accept `mask_image`). +- Canvas compiles `txt2img` and `img2img` for external models; `inpaint` is refused on Canvas because its keep-mask does not match any provider's edit-mask contract (the workflow editor nodes still accept `mask_image`). For a model with `resolution_presets`, `aspect_ratio_sizes` or `allowed_aspect_ratios`, the Canvas requests one of the model's own sizes (the closest aspect ratio, then the smallest size covering the bounding box). `_bucket_request` passes a fixed size or an exact multiple of an allowed ratio through unchanged; a resolution preset whose rounded size is off its exact ratio may be nudged by a pixel, which is harmless because the provider sizes its output from the ratio and the preset's name. A `txt2img` result is scaled to cover the bounding box and center-cropped; an `img2img` init image is resized to the request size and the result resized back. Keep these capabilities accurate: the crop is planned from them. - External starter models auto-install when a provider is configured. - Removing a provider API key removes installed external models for that provider. diff --git a/invokeai/frontend/webv2/performance/architecture-baseline.json b/invokeai/frontend/webv2/performance/architecture-baseline.json index 5600d273360..6c3dc6fb8a6 100644 --- a/invokeai/frontend/webv2/performance/architecture-baseline.json +++ b/invokeai/frontend/webv2/performance/architecture-baseline.json @@ -553,6 +553,7 @@ "source:src/features/generation/core/catalogSearch.ts", "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", + "source:src/features/generation/core/dimensionLimits.ts", "source:src/features/generation/core/dynamicPrompts.ts", "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", diff --git a/invokeai/frontend/webv2/performance/browser-baseline.json b/invokeai/frontend/webv2/performance/browser-baseline.json index c9916480ae9..5e8972f97c3 100644 --- a/invokeai/frontend/webv2/performance/browser-baseline.json +++ b/invokeai/frontend/webv2/performance/browser-baseline.json @@ -1,6 +1,6 @@ { - "browserExecutable": "/home/toast/.cache/ms-playwright/chromium-1243/chrome-linux64/chrome", - "capturedAt": "2026-10-19", + "browserExecutable": "/home/lstein/.cache/ms-playwright/chromium-1243/chrome-linux64/chrome", + "capturedAt": "2026-10-18", "routes": [ { "activatedResourceBaseline": { @@ -14,13 +14,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 127.80000000074506, + "domContentLoadedMedianMs": 97.2000002861023, "id": "launchpad", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 128, - "longestTaskMaxMs": 85, + "loadMedianMs": 97.40000009536743, + "longestTaskMaxMs": 64, "owner": "app", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:launchpad", @@ -36,7 +36,7 @@ "scriptRequestCount": 25, "totalRawBytes": 2119018 }, - "routeReadyMedianMs": 275.5, + "routeReadyMedianMs": 215.30000019073486, "scriptSourceOwnerSet": "launchpad-static", "stateProfile": "empty" }, @@ -52,13 +52,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 128.5, + "domContentLoadedMedianMs": 101, "id": "launchpad", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 128.59999999776483, - "longestTaskMaxMs": 84, + "loadMedianMs": 101.09999990463257, + "longestTaskMaxMs": 67, "owner": "app", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:launchpad", @@ -74,7 +74,7 @@ "scriptRequestCount": 25, "totalRawBytes": 2119018 }, - "routeReadyMedianMs": 285.80000000074506, + "routeReadyMedianMs": 225.39999961853027, "scriptSourceOwnerSet": "launchpad-static", "stateProfile": "representative" }, @@ -90,15 +90,15 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 129, + "domContentLoadedMedianMs": 99.69999980926514, "id": "editor-minimal", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 129.10000000149012, - "longestTaskMaxMs": 183, + "loadMedianMs": 99.7999997138977, + "longestTaskMaxMs": 148, "owner": "workbench", - "projectSwitchMedianMs": 38.69999999925494, + "projectSwitchMedianMs": 34.90000009536743, "readyMark": "invokeai:ready:widget:center:preview", "remediationTicket": "deepen-widget-registry-loading", "resourceBaseline": { @@ -112,7 +112,7 @@ "scriptRequestCount": 82, "totalRawBytes": 4515078 }, - "routeReadyMedianMs": 448.9000000022352, + "routeReadyMedianMs": 360.90000009536743, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "empty" }, @@ -128,15 +128,15 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 126.69999999925494, + "domContentLoadedMedianMs": 100.5, "id": "editor-minimal", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, "layoutSwitchMedianMs": 0, - "loadMedianMs": 126.79999999701977, - "longestTaskMaxMs": 186, + "loadMedianMs": 100.60000038146973, + "longestTaskMaxMs": 146, "owner": "workbench", - "projectSwitchMedianMs": 47.70000000298023, + "projectSwitchMedianMs": 48.40000009536743, "readyMark": "invokeai:ready:widget:center:preview", "remediationTicket": "deepen-widget-registry-loading", "resourceBaseline": { @@ -150,7 +150,7 @@ "scriptRequestCount": 82, "totalRawBytes": 4515078 }, - "routeReadyMedianMs": 462.0999999977648, + "routeReadyMedianMs": 367.80000019073486, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "representative" }, @@ -166,13 +166,13 @@ "scriptRequestCount": 9, "totalRawBytes": 520084 }, - "domContentLoadedMedianMs": 129, + "domContentLoadedMedianMs": 100.40000009536743, "id": "editor-canvas", - "layoutAckMedianMs": 3, - "layoutReturnSwitchMedianMs": 58.5, - "layoutSwitchMedianMs": 75.59999999776483, - "loadMedianMs": 129.09999999776483, - "longestTaskMaxMs": 172, + "layoutAckMedianMs": 2.8000001907348633, + "layoutReturnSwitchMedianMs": 51.39999961853027, + "layoutSwitchMedianMs": 70.09999990463257, + "loadMedianMs": 100.5, + "longestTaskMaxMs": 149, "owner": "canvas", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:canvas", @@ -188,7 +188,7 @@ "scriptRequestCount": 91, "totalRawBytes": 5035162 }, - "routeReadyMedianMs": 1194.3999999985099, + "routeReadyMedianMs": 1095.3000001907349, "scriptSourceOwnerSet": "editor-canvas-static", "stateProfile": "empty" }, @@ -204,13 +204,13 @@ "scriptRequestCount": 9, "totalRawBytes": 520084 }, - "domContentLoadedMedianMs": 126.89999999850988, + "domContentLoadedMedianMs": 97.7000002861023, "id": "editor-canvas", - "layoutAckMedianMs": 3.2000000029802322, - "layoutReturnSwitchMedianMs": 50.30000000074506, - "layoutSwitchMedianMs": 75.90000000223517, - "loadMedianMs": 127, - "longestTaskMaxMs": 192, + "layoutAckMedianMs": 3, + "layoutReturnSwitchMedianMs": 45.30000019073486, + "layoutSwitchMedianMs": 68.90000009536743, + "loadMedianMs": 97.90000009536743, + "longestTaskMaxMs": 148, "owner": "canvas", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:canvas", @@ -226,7 +226,7 @@ "scriptRequestCount": 91, "totalRawBytes": 5035162 }, - "routeReadyMedianMs": 1175.699999999255, + "routeReadyMedianMs": 1042.5, "scriptSourceOwnerSet": "editor-canvas-static", "stateProfile": "representative" }, @@ -242,13 +242,13 @@ "scriptRequestCount": 6, "totalRawBytes": 460056 }, - "domContentLoadedMedianMs": 129.5, + "domContentLoadedMedianMs": 101.40000009536743, "id": "editor-workflow", - "layoutAckMedianMs": 3, - "layoutReturnSwitchMedianMs": 40.100000001490116, - "layoutSwitchMedianMs": 85.5, - "loadMedianMs": 129.69999999925494, - "longestTaskMaxMs": 182, + "layoutAckMedianMs": 2.9000000953674316, + "layoutReturnSwitchMedianMs": 32.700000286102295, + "layoutSwitchMedianMs": 73.09999990463257, + "loadMedianMs": 101.5, + "longestTaskMaxMs": 146, "owner": "workflow", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:workflow", @@ -264,7 +264,7 @@ "scriptRequestCount": 88, "totalRawBytes": 4975134 }, - "routeReadyMedianMs": 1206.3999999985099, + "routeReadyMedianMs": 1093.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", "stateProfile": "empty" }, @@ -280,13 +280,13 @@ "scriptRequestCount": 6, "totalRawBytes": 460056 }, - "domContentLoadedMedianMs": 127.10000000149012, + "domContentLoadedMedianMs": 99.89999961853027, "id": "editor-workflow", - "layoutAckMedianMs": 3, - "layoutReturnSwitchMedianMs": 46.70000000298023, - "layoutSwitchMedianMs": 147.40000000223517, - "loadMedianMs": 127.19999999925494, - "longestTaskMaxMs": 288, + "layoutAckMedianMs": 2.799999713897705, + "layoutReturnSwitchMedianMs": 41.09999990463257, + "layoutSwitchMedianMs": 125.60000038146973, + "loadMedianMs": 100, + "longestTaskMaxMs": 226, "owner": "workflow", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:workflow", @@ -302,7 +302,7 @@ "scriptRequestCount": 88, "totalRawBytes": 4975134 }, - "routeReadyMedianMs": 1246.699999999255, + "routeReadyMedianMs": 1105.0999999046326, "scriptSourceOwnerSet": "editor-workflow-static", "stateProfile": "representative" }, @@ -318,13 +318,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 127.80000000074506, + "domContentLoadedMedianMs": 98.30000019073486, "id": "editor-gallery", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, - "layoutSwitchMedianMs": 49.399999998509884, - "loadMedianMs": 127.90000000223517, - "longestTaskMaxMs": 172, + "layoutSwitchMedianMs": 43.59999990463257, + "loadMedianMs": 98.5, + "longestTaskMaxMs": 144, "owner": "gallery", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:gallery", @@ -340,7 +340,7 @@ "scriptRequestCount": 82, "totalRawBytes": 4515078 }, - "routeReadyMedianMs": 1431.0999999977648, + "routeReadyMedianMs": 1324.5, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "empty" }, @@ -356,13 +356,13 @@ "scriptRequestCount": 0, "totalRawBytes": 0 }, - "domContentLoadedMedianMs": 127.60000000149012, + "domContentLoadedMedianMs": 99.80000019073486, "id": "editor-gallery", "layoutAckMedianMs": 0, "layoutReturnSwitchMedianMs": 0, - "layoutSwitchMedianMs": 60.69999999925494, - "loadMedianMs": 127.80000000074506, - "longestTaskMaxMs": 178, + "layoutSwitchMedianMs": 50, + "loadMedianMs": 99.90000009536743, + "longestTaskMaxMs": 145, "owner": "gallery", "projectSwitchMedianMs": 0, "readyMark": "invokeai:ready:widget:center:gallery", @@ -378,7 +378,7 @@ "scriptRequestCount": 82, "totalRawBytes": 4515078 }, - "routeReadyMedianMs": 1446.300000000745, + "routeReadyMedianMs": 1301.5, "scriptSourceOwnerSet": "editor-minimal-static", "stateProfile": "representative" } @@ -979,6 +979,7 @@ "source:src/features/generation/core/catalogSearch.ts", "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", + "source:src/features/generation/core/dimensionLimits.ts", "source:src/features/generation/core/dynamicPrompts.ts", "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", @@ -2080,6 +2081,7 @@ "source:src/features/generation/core/catalogSearch.ts", "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", + "source:src/features/generation/core/dimensionLimits.ts", "source:src/features/generation/core/dynamicPrompts.ts", "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", @@ -3339,6 +3341,7 @@ "source:src/features/generation/core/catalogSearch.ts", "source:src/features/generation/core/componentCompatibility.ts", "source:src/features/generation/core/conditioningRebalance.ts", + "source:src/features/generation/core/dimensionLimits.ts", "source:src/features/generation/core/dynamicPrompts.ts", "source:src/features/generation/core/externalProvider.ts", "source:src/features/generation/core/graph.ts", diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index f3e47648ade..f1ceba831f6 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -2532,6 +2532,8 @@ "deletePresetNamed": "Delete preset {{name}}", "deletePromptHistoryItem": "Delete prompt history item", "denoisingStrength": "Denoising strength", + "dimensionsAdjusted": "Dimensions adjusted", + "dimensionsAdjustedDescription": "Set to {{width}}×{{height}} for {{name}}.", "disableConcept": "Disable {{name}}", "disableReferenceImage": "Disable reference image", "dropImageToPrompt": "Describe this image", @@ -2751,6 +2753,7 @@ "searchPresets": "Search presets", "secondsPerRun": "~{{value}}s/run", "sizePreviewHandle": "Drag to resize the generation size", + "switchModelAdjustsDimensions": "It also sets the dimensions to {{width}}×{{height}}.", "switchModelBody": "Switching to {{name}} turns off or removes: {{labels}}.", "switchModelConfirm": "Switch model", "switchModelTitle": "Switch model?", diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs index 7abcce6a56a..65b9fae6b54 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs @@ -398,10 +398,13 @@ const MODEL_TYPES = ['main', 'lora', 'controlnet', 'vae', 'embedding']; // Main models cycle through their own bases so Generate sees SDXL mains and refiners, not only SD 1.5. const MAIN_MODEL_BASES = ['sd-1', 'sdxl', 'sdxl-refiner']; +/** The source of the hosted record the mock starter catalog also lists, so the Model Manager shows it read-only. */ +export const CATALOG_MANAGED_EXTERNAL_SOURCE = 'external://gemini/gemini-2.5-flash-image'; + /** * Hosted-provider records with the capability shapes the Generate panel, Canvas, and Model Manager read: a named - * resolution table (Gemini), fixed sizes per ratio with an edit mode (OpenAI), and provider toggles (Seedream). - * No shipped provider accepts a mask, so none advertises inpaint. + * resolution table (Gemini), fixed sizes per ratio with an edit mode (OpenAI), provider toggles (Seedream), and a + * record the starter catalog manages. No shipped provider accepts a mask, so none advertises inpaint. */ const EXTERNAL_MODEL_FIXTURES = [ { @@ -507,6 +510,32 @@ const EXTERNAL_MODEL_FIXTURES = [ trigger_phrases: null, type: 'external_image_generator', }, + { + base: 'external', + capabilities: { + allowed_aspect_ratios: ['1:1', '16:9', '9:16'], + max_images_per_request: 1, + max_reference_images: 3, + modes: ['txt2img'], + supports_reference_images: true, + }, + cover_image: null, + default_settings: { height: 1024, num_images: 1, width: 1024 }, + description: 'Fixture hosted model whose capabilities the starter catalog manages.', + file_size: 0, + format: 'external_api', + hash: 'external:gemini:gemini-2.5-flash-image', + key: 'fixture-external-gemini-catalog', + name: 'Fixture Gemini Flash (Catalog)', + path: CATALOG_MANAGED_EXTERNAL_SOURCE, + provider_id: 'gemini', + provider_model_id: 'gemini-2.5-flash-image', + source: CATALOG_MANAGED_EXTERNAL_SOURCE, + source_type: 'external', + source_url: null, + trigger_phrases: null, + type: 'external_image_generator', + }, ]; /** The last slots of the model library hold the hosted-provider records so the profile counts stay as declared. */ diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs index 8f4d1fbcb50..139318c817a 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs @@ -1403,3 +1403,62 @@ test('runtime config PATCH keeps the other writable field and validates the queu assert.equal((await getJson(backend, '/api/v1/app/invocation_cache/status')).enabled, false); }); }); + +test('hosted-provider credentials and the starter catalog agree with the hosted fixture records', async () => { + await withRepresentativeBackend(async (backend) => { + const { models } = await getJson(backend, '/api/v2/models/'); + const hosted = models.filter((model) => model.type === 'external_image_generator'); + const configs = await getJson(backend, '/api/v1/app/external_providers/config'); + + // Every backend provider is listed; those the hosted records route to have a key. + assert.deepEqual( + configs.map((config) => config.provider_id), + ['alibabacloud', 'atlascloud', 'gemini', 'openai', 'seedream'] + ); + assert.deepEqual( + configs.filter((config) => config.api_key_configured).map((config) => config.provider_id), + [...new Set(hosted.map((model) => model.provider_id))].sort() + ); + assert.ok(configs.every((config) => config.base_url === null)); + + // Exactly one hosted record carries a starter's source, so the Model Manager shows it catalog-managed. + const { starter_models: starters } = await getJson(backend, '/api/v2/models/starter_models'); + const starterSources = new Set(starters.map((starter) => starter.source)); + const managed = hosted.filter((model) => starterSources.has(model.source)); + assert.deepEqual( + managed.map((model) => model.key), + ['fixture-external-gemini-catalog'] + ); + assert.equal(starters.find((starter) => starter.source === managed[0].source).is_installed, true); + assert.ok(starters.filter((starter) => starter.type !== 'external_image_generator').every((s) => !s.is_installed)); + + // Removing a provider's key removes its hosted records, as the backend does. + const reset = await getJson(backend, '/api/v1/app/external_providers/config/openai', { method: 'DELETE' }); + assert.deepEqual(reset, { api_key_configured: false, base_url: null, provider_id: 'openai' }); + const remaining = (await getJson(backend, '/api/v2/models/')).models; + assert.equal(remaining.length, models.length - hosted.filter((model) => model.provider_id === 'openai').length); + assert.equal( + (await fetch(`${backend.origin}/api/v1/app/external_providers/config/unknown`, { method: 'DELETE' })).status, + 404 + ); + + const empty = await startMockBackend(0, { profile: 'empty' }); + try { + const emptyConfigs = await getJson(empty, '/api/v1/app/external_providers/config'); + assert.equal(emptyConfigs.length, 5); + assert.ok(emptyConfigs.every((config) => !config.api_key_configured)); + } finally { + await empty.close(); + } + + const configured = await getJson(backend, '/api/v1/app/external_providers/config/openai', { + body: JSON.stringify({ api_key: 'sk-fixture', base_url: 'https://example.invalid/v1' }), + method: 'POST', + }); + assert.deepEqual(configured, { + api_key_configured: true, + base_url: 'https://example.invalid/v1', + provider_id: 'openai', + }); + }); +}); diff --git a/invokeai/frontend/webv2/scripts/mock-backend.mjs b/invokeai/frontend/webv2/scripts/mock-backend.mjs index 86a3d7f97da..2b9bc50c880 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend.mjs @@ -5,6 +5,7 @@ import { resolve } from 'node:path'; import { assertMockBackendFixture, assertMockBackendProfileName, + CATALOG_MANAGED_EXTERNAL_SOURCE, collectCanvasLeaves, createMockBackendFixture, getFixtureProjectWorkflowDocument, @@ -234,6 +235,17 @@ const STARTER_MODEL = { type: 'main', }; +/** The hosted starter whose source a fixture record carries, so that record reads as catalog-managed. */ +const EXTERNAL_STARTER_MODEL = { + base: 'external', + description: 'Fixture hosted starter model', + format: 'external_api', + is_installed: false, + name: 'Fixture Gemini Flash (Catalog)', + source: CATALOG_MANAGED_EXTERNAL_SOURCE, + type: 'external_image_generator', +}; + const MOCK_APP_VERSION = 'fixture'; const STARTER_MODELS_RESPONSE = { @@ -244,7 +256,52 @@ const STARTER_MODELS_RESPONSE = { name: 'SDXL', }, }, - starter_models: [STARTER_MODEL], + starter_models: [STARTER_MODEL, EXTERNAL_STARTER_MODEL], +}; + +/** Starters read as installed when a library record carries their source, as the backend reports them. */ +const getStarterModelsResponse = (state) => { + const installedSources = new Set([...state.models.values()].map((model) => model.source)); + const withInstalled = (starter) => ({ ...starter, is_installed: installedSources.has(starter.source) }); + + return { + starter_bundles: Object.fromEntries( + Object.entries(STARTER_MODELS_RESPONSE.starter_bundles).map(([base, bundle]) => [ + base, + { ...bundle, models: bundle.models.map(withInstalled) }, + ]) + ), + starter_models: STARTER_MODELS_RESPONSE.starter_models.map(withInstalled), + }; +}; + +/** The backend's `EXTERNAL_PROVIDER_FIELDS`, in its order: the config route lists each, configured or not. */ +const EXTERNAL_PROVIDER_IDS = ['alibabacloud', 'atlascloud', 'gemini', 'openai', 'seedream']; + +/** + * Hosted-provider credentials as the config route reports them. A provider the profile's hosted records route to has a + * key, since the backend removes a provider's records when its key is removed; the rest have none. + */ +const createExternalProviderConfigs = (models) => { + const routed = new Set( + models.filter((model) => model.type === 'external_image_generator').map((model) => model.provider_id) + ); + + return new Map( + EXTERNAL_PROVIDER_IDS.map((providerId) => [ + providerId, + { api_key_configured: routed.has(providerId), base_url: null, provider_id: providerId }, + ]) + ); +}; + +/** The backend drops a provider's hosted records when its key is removed or reset. */ +const removeExternalModelsForProvider = (state, providerId) => { + for (const [key, model] of state.models) { + if (model.type === 'external_image_generator' && model.provider_id === providerId) { + state.models.delete(key); + } + } }; const DELIVERY_CLAIM_TTL_MS = 120_000; @@ -262,6 +319,7 @@ const createState = (profile) => { JSON.stringify({ alphaNoticeAcknowledged: true, whatsNewSeenVersion: MOCK_APP_VERSION }), ], ]), + externalProviders: createExternalProviderConfigs(fixture.models), images: new Map(fixture.images.map((image) => [image.image_name, clone(image)])), models: new Map(fixture.models.map((model) => [model.key, clone(model)])), mutationClock: 0, @@ -1168,8 +1226,42 @@ export const startMockBackend = async (port, { profile = 'empty' } = {}) => { } } - if (path.startsWith('/api/v1/app/external_providers')) { - return json(200, []); + if (method === 'GET' && path === '/api/v1/app/external_providers/config') { + return json(200, [...state.externalProviders.values()].map(clone)); + } + + const externalProviderMatch = /^\/api\/v1\/app\/external_providers\/config\/([^/]+)$/.exec(path); + if (externalProviderMatch && (method === 'POST' || method === 'DELETE')) { + const providerId = decodeURIComponent(externalProviderMatch[1]); + const config = state.externalProviders.get(providerId); + + if (!config) { + return json(404, { detail: `Unknown external provider '${providerId}'` }); + } + if (method === 'DELETE') { + Object.assign(config, { api_key_configured: false, base_url: null }); + removeExternalModelsForProvider(state, providerId); + return json(200, clone(config)); + } + + const update = await readJsonBody(request); + const apiKey = typeof update.api_key === 'string' ? update.api_key.trim() : null; + const baseUrl = typeof update.base_url === 'string' ? update.base_url.trim() : null; + + if (apiKey === null && baseUrl === null) { + return json(400, { detail: 'No external provider config fields provided' }); + } + if (baseUrl !== null) { + config.base_url = baseUrl || null; + } + if (apiKey !== null) { + config.api_key_configured = apiKey !== ''; + if (apiKey === '') { + removeExternalModelsForProvider(state, providerId); + } + } + + return json(200, clone(config)); } if (path === '/api/v1/projects/' || path === '/api/v1/projects') { @@ -1752,7 +1844,7 @@ export const startMockBackend = async (port, { profile = 'empty' } = {}) => { return json(200, faults.capabilities === 'empty' ? [] : ARCHITECTURE_CAPABILITIES); } if (method === 'GET' && path === '/api/v2/models/starter_models') { - return json(200, STARTER_MODELS_RESPONSE); + return json(200, getStarterModelsResponse(state)); } if (method === 'GET' && path === '/api/v2/models/hf_login') { return json(200, 'unknown'); diff --git a/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json index 4a4253cf5f3..cae6c5dd4df 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json +++ b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/externalCanvasGraphs.json @@ -75,7 +75,7 @@ { "destination": { "field": "image", - "node_id": "canvas_output" + "node_id": "canvas_provider_metadata" }, "source": { "field": "item", @@ -85,13 +85,23 @@ { "destination": { "field": "image", - "node_id": "canvas_provider_metadata" + "node_id": "canvas_provider_cover" }, "source": { "field": "item", "node_id": "canvas_provider_image" } }, + { + "destination": { + "field": "image", + "node_id": "canvas_output" + }, + "source": { + "field": "image", + "node_id": "canvas_provider_cover" + } + }, { "destination": { "field": "metadata", @@ -133,10 +143,28 @@ "model": { "base": "external", "capabilities": { + "allowed_aspect_ratios": [ + "1:1", + "3:2", + "2:3" + ], + "aspect_ratio_sizes": { + "1:1": { + "height": 1024, + "width": 1024 + }, + "3:2": { + "height": 1024, + "width": 1536 + }, + "2:3": { + "height": 1536, + "width": 1024 + } + }, "modes": [ "txt2img", - "img2img", - "inpaint" + "img2img" ], "supports_seed": true }, @@ -145,6 +173,7 @@ "name": "OpenAI Image", "provider_id": "openai", "type": "external_image_generator", + "provider_model_id": "gpt-image-1", "hash": "blake3:external" }, "rand_device": "cpu", @@ -160,17 +189,35 @@ "canvas_processing_output": { "is_intermediate": true, "use_cache": false, - "height": 512, + "height": 1024, "id": "canvas_processing_output", "image_size": null, "mode": "txt2img", "model": { "base": "external", "capabilities": { + "allowed_aspect_ratios": [ + "1:1", + "3:2", + "2:3" + ], + "aspect_ratio_sizes": { + "1:1": { + "height": 1024, + "width": 1024 + }, + "3:2": { + "height": 1024, + "width": 1536 + }, + "2:3": { + "height": 1536, + "width": 1024 + } + }, "modes": [ "txt2img", - "img2img", - "inpaint" + "img2img" ], "supports_seed": true }, @@ -179,11 +226,12 @@ "name": "OpenAI Image", "provider_id": "openai", "type": "external_image_generator", + "provider_model_id": "gpt-image-1", "hash": "blake3:external" }, "num_images": 1, "type": "openai_image_generation", - "width": 640, + "width": 1536, "quality": "auto", "background": "auto" }, @@ -199,13 +247,23 @@ "id": "canvas_provider_metadata", "type": "metadata_from_image" }, - "canvas_output": { + "canvas_provider_cover": { "is_intermediate": true, "use_cache": false, "height": 512, - "id": "canvas_output", + "id": "canvas_provider_cover", "type": "img_resize", - "width": 640 + "width": 768 + }, + "canvas_output": { + "is_intermediate": true, + "use_cache": false, + "id": "canvas_output", + "height": 512, + "type": "img_crop", + "width": 640, + "x": 64, + "y": 0 } } }, @@ -271,6 +329,16 @@ "node_id": "core_metadata_0_i" } }, + { + "destination": { + "field": "init_image", + "node_id": "canvas_processing_output" + }, + "source": { + "field": "image", + "node_id": "canvas_resize_image_to_request" + } + }, { "destination": { "field": "collection", @@ -284,7 +352,7 @@ { "destination": { "field": "image", - "node_id": "canvas_output" + "node_id": "canvas_provider_metadata" }, "source": { "field": "item", @@ -294,7 +362,7 @@ { "destination": { "field": "image", - "node_id": "canvas_provider_metadata" + "node_id": "canvas_output" }, "source": { "field": "item", @@ -342,10 +410,28 @@ "model": { "base": "external", "capabilities": { + "allowed_aspect_ratios": [ + "1:1", + "3:2", + "2:3" + ], + "aspect_ratio_sizes": { + "1:1": { + "height": 1024, + "width": 1024 + }, + "3:2": { + "height": 1024, + "width": 1536 + }, + "2:3": { + "height": 1536, + "width": 1024 + } + }, "modes": [ "txt2img", - "img2img", - "inpaint" + "img2img" ], "supports_seed": true }, @@ -354,6 +440,7 @@ "name": "OpenAI Image", "provider_id": "openai", "type": "external_image_generator", + "provider_model_id": "gpt-image-1", "hash": "blake3:external" }, "rand_device": "cpu", @@ -366,20 +453,49 @@ "openai_background": "auto", "clip_skip": 0 }, + "canvas_resize_image_to_request": { + "is_intermediate": true, + "use_cache": true, + "height": 1024, + "id": "canvas_resize_image_to_request", + "image": { + "image_name": "canvas-composite.png" + }, + "type": "img_resize", + "width": 1536 + }, "canvas_processing_output": { "is_intermediate": true, "use_cache": false, - "height": 512, + "height": 1024, "id": "canvas_processing_output", "image_size": null, "mode": "img2img", "model": { "base": "external", "capabilities": { + "allowed_aspect_ratios": [ + "1:1", + "3:2", + "2:3" + ], + "aspect_ratio_sizes": { + "1:1": { + "height": 1024, + "width": 1024 + }, + "3:2": { + "height": 1024, + "width": 1536 + }, + "2:3": { + "height": 1536, + "width": 1024 + } + }, "modes": [ "txt2img", - "img2img", - "inpaint" + "img2img" ], "supports_seed": true }, @@ -388,16 +504,14 @@ "name": "OpenAI Image", "provider_id": "openai", "type": "external_image_generator", + "provider_model_id": "gpt-image-1", "hash": "blake3:external" }, "num_images": 1, "type": "openai_image_generation", - "width": 640, + "width": 1536, "quality": "auto", - "background": "auto", - "init_image": { - "image_name": "canvas-composite.png" - } + "background": "auto" }, "canvas_provider_image": { "is_intermediate": true, @@ -414,8 +528,8 @@ "canvas_output": { "is_intermediate": true, "use_cache": false, - "height": 512, "id": "canvas_output", + "height": 512, "type": "img_resize", "width": 640 } diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts index f6f6a39a027..a86c99f8750 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts @@ -702,6 +702,16 @@ describe('component policies', () => { ]); settings.referenceImages[0]!.isEnabled = false; expect(getGenerationValidationReasons(model, settings, 'img2img')).toEqual([]); + settings.referenceImages[0]!.isEnabled = true; + // A record that does not take references sends none, so the budget counts none; the references are refused + // on their own terms instead. + const reasons = getGenerationValidationReasons( + { ...model, capabilities: { modes: ['txt2img', 'img2img'], supports_reference_images: false } }, + settings, + 'img2img' + ); + expect(reasons).toContain('Reference Image #1 is not supported by external model.'); + expect(reasons.filter((reason) => reason.includes('images total'))).toEqual([]); }); it('renders no component slots for bases without extra requirements', () => { @@ -912,14 +922,64 @@ describe('component policies', () => { expect(result.settings.componentSourceModel).toBe(devSource); }); - it('reconciles dimensions to the new model grid when the selected model changes', () => { + it('reconciles dimensions to the new model grid as an adjustment, not a removed setting', () => { const model = createModel('cogview4'); const settings = createSettings(createModel('sdxl'), { height: 520, width: 520 }); const result = getGenerateModelSelectionResult({ currentValues: settings, model, models: [] }); expect(result.settings.width).toBe(512); expect(result.settings.height).toBe(512); - expect(result.clearedLabels).toContain('Dimensions'); + expect(result.clearedLabels).toEqual([]); + expect(result.dimensionsAdjusted).toBe(true); + }); + + it('reports a snap to a provider size as an adjustment, so the switch is not lossy', () => { + const seedream: GenerateModelConfig = { + ...externalModel, + key: 'seedream', + name: 'Seedream 4.5', + provider_id: 'seedream', + capabilities: { + modes: ['txt2img', 'img2img'], + aspect_ratio_sizes: { + '1:1': { width: 2048, height: 2048 }, + '16:9': { width: 2848, height: 1600 }, + }, + }, + }; + const result = getGenerateModelSelectionResult({ + currentValues: createSettings(createModel('sdxl'), { height: 1024, width: 1024 }), + model: seedream, + models: [], + }); + + expect(result.settings).toMatchObject({ height: 2048, width: 2048 }); + expect(result.clearedLabels).toEqual([]); + expect(result.dimensionsAdjusted).toBe(true); + }); + + it('reports the grid clamp from an off-grid provider size back to a local model as an adjustment', () => { + const provider: GenerateModelConfig = { ...externalModel, key: 'gemini', provider_id: 'gemini' }; + const result = getGenerateModelSelectionResult({ + currentValues: createSettings(provider, { height: 1024, width: 683 }), + model: createModel('sdxl'), + models: [], + }); + + expect(result.settings.width % 8).toBe(0); + expect(result.settings.height).toBe(1024); + expect(result.clearedLabels).toEqual([]); + expect(result.dimensionsAdjusted).toBe(true); + }); + + it('reports no adjustment when the size already fits the new model', () => { + const result = getGenerateModelSelectionResult({ + currentValues: createSettings(createModel('sd-1'), { height: 1024, width: 1024 }), + model: createModel('sdxl'), + models: [], + }); + + expect(result.dimensionsAdjusted).toBe(false); }); it('keeps selected VAE when applying model defaults without a VAE override', () => { diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts index 87fc1763e71..29f171ec218 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts @@ -49,9 +49,11 @@ import { isZImageQwen3Encoder, type GenerateComponentFilter, } from './componentCompatibility'; +import { MAX_DIMENSION, MIN_DIMENSION } from './dimensionLimits'; import { DYNAMIC_PROMPTS_DEFAULT_MAX_PROMPTS } from './dynamicPrompts'; import { getExternalAspectRatioSizes, + getExternalReferenceImages, getExternalResolutionPresets, isSeedreamBatchModel, resolveExternalImageSize, @@ -103,8 +105,6 @@ import { isValidKrea2RebalanceWeights, isWanLoraTargetingMain, KREA2_REBALANCE_WEIGHT_COUNT, - MAX_DIMENSION, - MIN_DIMENSION, normalizeGenerateSettings, } from './settings'; @@ -627,7 +627,10 @@ export interface ComponentSectionPolicy { export interface GenerateModelSelectionResult { settings: GenerateSettings; + /** Settings the switch turns off or removes, which callers confirm or report. */ clearedLabels: readonly string[]; + /** The size moved to the model's grid or provider sizes. Nothing is lost and it stays on screen, so it is reported. */ + dimensionsAdjusted: boolean; } const TYPE_CLIP_EMBED: ModelTaxonomyType[] = ['clip_embed']; @@ -1492,7 +1495,6 @@ const getSettingsWithCompatibleModelSelections = ( nextSettings.height = height; nextSettings.aspectRatioId = deriveAspectRatioId(width, height); nextSettings.aspectRatioValue = height > 0 ? width / height : 1; - addClearedLabel(clearedLabels, 'Dimensions'); } if (settings.modelKey !== model.key) { @@ -1523,7 +1525,6 @@ const getSettingsWithCompatibleModelSelections = ( nextSettings.height = selected.height; nextSettings.aspectRatioId = deriveAspectRatioId(selected.width, selected.height); nextSettings.aspectRatioValue = selected.width / selected.height; - addClearedLabel(clearedLabels, 'Dimensions'); } nextSettings.externalImageSize = resolveExternalImageSize(nextSettings, model); @@ -1627,7 +1628,11 @@ const getSettingsWithCompatibleModelSelections = ( } } - return { settings: nextSettings, clearedLabels }; + return { + clearedLabels, + dimensionsAdjusted: nextSettings.width !== settings.width || nextSettings.height !== settings.height, + settings: nextSettings, + }; }; /** Normalize, reconcile, and apply component policy before persisting model selection. */ @@ -1906,11 +1911,7 @@ export const getGenerationValidationReasons = ( // Every request emits one output, and the Canvas edit modes add the composite as one init image. if (isSeedreamBatchModel(model)) { - const referenceCount = settings.referenceImages.filter( - (reference) => - reference.isEnabled && reference.config.type === 'external_reference_image' && reference.config.image !== null - ).length; - const totalImages = referenceCount + (mode === 'txt2img' ? 0 : 1) + 1; + const totalImages = getExternalReferenceImages(settings, model).length + (mode === 'txt2img' ? 0 : 1) + 1; if (totalImages > SEEDREAM_BATCH_MAX_TOTAL_IMAGES) { reasons.push( diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts index 37326ca3cee..d8b850239f9 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.test.ts @@ -12,7 +12,7 @@ const sdxl = { base: 'sdxl', type: 'main' } as GenerateModelConfig; const external = { base: 'external', type: 'external_image_generator' } as GenerateModelConfig; describe('resolveCanvasProcessingSize', () => { - it('asks a hosted provider for the bbox itself, whatever the scaling method', () => { + it('asks a free-form hosted provider for the bbox itself, whatever the scaling method', () => { const bbox = { height: 300, width: 500 }; expect(resolveCanvasProcessingSize(external, 'off', bbox, undefined)).toEqual(bbox); @@ -24,6 +24,22 @@ describe('resolveCanvasProcessingSize', () => { ); }); + it('asks a fixed-size hosted provider for one of its sizes, whatever the scaling method', () => { + const fixed = { + ...external, + capabilities: { aspect_ratio_sizes: { '1:1': { height: 1024, width: 1024 } }, modes: ['txt2img'] }, + } as GenerateModelConfig; + + expect( + resolveCanvasProcessingSize( + fixed, + 'off', + { height: 300, width: 500 }, + { height: 64, method: 'manual', width: 64 } + ) + ).toEqual({ height: 1024, width: 1024 }); + }); + it('keeps the bbox on the model grid when scaling is off or unset', () => { expect(resolveCanvasProcessingSize(sd1, 'off', { height: 300, width: 500 }, undefined)).toEqual({ height: 304, diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts index 9cb39a390f1..8897fe77680 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasProcessingSize.ts @@ -1,13 +1,16 @@ /** * None snaps the bbox to the model grid; auto grows undersized content toward optimal area at its aspect ratio; - * manual uses the selected size. Restore final output to the exact bbox. Hosted providers are asked for the bbox - * itself: they bucket the request to their own size table, then the compiler restores their output to the bbox. + * manual uses the selected size. Restore final output to the exact bbox. Hosted providers ignore the scaling method: + * they are asked for one of their own sizes nearest the bbox (or the bbox itself when free-form), and the compiler + * restores their output to the bbox. */ import type { GenerateModelConfig, PidMode } from '@features/generation/core/types'; import { getGenerationDimensions } from '@features/generation/core/baseGenerationPolicies'; -import { clampDimension, MAX_DIMENSION } from '@features/generation/core/settings'; +import { MAX_DIMENSION } from '@features/generation/core/dimensionLimits'; +import { resolveExternalCanvasRequestSize } from '@features/generation/core/externalProvider'; +import { clampDimension } from '@features/generation/core/settings'; import type { CanvasScalingSettings } from './types'; @@ -75,14 +78,14 @@ const growToOptimalArea = (size: CanvasSize, optimal: number, grid: number): Can }; export const resolveCanvasProcessingSize = ( - // Keep the variant: Wan 5B and A14B use different grids. - model: Pick & { variant?: unknown }, + // Keep the variant (Wan 5B and A14B use different grids) and a provider's capabilities (its sizes). + model: GenerateModelConfig, pidMode: PidMode, bbox: CanvasSize, scaling: CanvasScalingSettings | undefined ): CanvasSize => { if (model.type === 'external_image_generator') { - return { height: bbox.height, width: bbox.width }; + return resolveExternalCanvasRequestSize(model, bbox); } const { grid, optimal } = getGenerationDimensions(model, pidMode); switch (scaling?.method) { diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index e3ddb2de33d..7cc04533a12 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -437,6 +437,21 @@ describe('compileCanvasGraph', () => { capabilities: { modes: ['txt2img', 'img2img', 'inpaint'], supports_seed: true }, }; const smallBbox: Rect = { height: 512, width: 640, x: 10, y: 20 }; + // The starter catalog's GPT Image size table. + const gptImage: GenerateModelConfig = { + ...editModel, + capabilities: { + allowed_aspect_ratios: ['1:1', '3:2', '2:3'], + aspect_ratio_sizes: { + '1:1': { height: 1024, width: 1024 }, + '3:2': { height: 1024, width: 1536 }, + '2:3': { height: 1536, width: 1024 }, + }, + modes: ['txt2img', 'img2img'], + supports_seed: true, + }, + provider_model_id: 'gpt-image-1', + }; it('writes hosted canvas graphs for backend invocation contract validation', async () => { const clock = vi.spyOn(Date, 'now').mockReturnValue(0); @@ -444,7 +459,7 @@ describe('compileCanvasGraph', () => { const graphs: Record = {}; try { for (const mode of ['txt2img', 'img2img'] as const) { - graphs[mode] = compile({ ...editModel, hash: 'blake3:external' }, mode, { bbox: smallBbox }).backendGraph; + graphs[mode] = compile({ ...gptImage, hash: 'blake3:external' }, mode, { bbox: smallBbox }).backendGraph; } } finally { clock.mockRestore(); @@ -507,6 +522,117 @@ describe('compileCanvasGraph', () => { expect(getNodeByType(backendGraph, 'core_metadata')).toMatchObject({ height: 512, width: 640 }); }); + it('asks a fixed-size provider for its nearest-ratio size and center-crops a text-to-image result to the bbox', () => { + const { backendGraph } = compile(gptImage, 'txt2img', { bbox: smallBbox, compositeImageName: null }); + + // 640×512 is nearest 3:2 among the table's ratios; the provider receives that exact entry. + expect(backendGraph.nodes.canvas_processing_output).toMatchObject({ height: 1024, width: 1536 }); + // 1536×1024 scaled to cover 640×512 is 768×512, then 64 px come off each side. + expect(backendGraph.nodes.canvas_provider_cover).toMatchObject({ + height: 512, + is_intermediate: true, + type: 'img_resize', + width: 768, + }); + expect(backendGraph.nodes.canvas_output).toMatchObject({ + height: 512, + type: 'img_crop', + width: 640, + x: 64, + y: 0, + }); + expect(getEdge(backendGraph, 'canvas_provider_cover', 'image')?.source).toEqual({ + node_id: 'canvas_provider_image', + field: 'item', + }); + expect(getEdge(backendGraph, 'canvas_output', 'image')?.source.node_id).toBe('canvas_provider_cover'); + // Each image keeps the metadata of the provider image it was cropped from. + expect(getEdge(backendGraph, 'canvas_provider_metadata', 'image')?.source.node_id).toBe('canvas_provider_image'); + expect(getEdge(backendGraph, 'canvas_output', 'metadata')?.source.node_id).toBe('canvas_provider_metadata'); + expect(getNodeByType(backendGraph, 'core_metadata')).toMatchObject({ height: 512, width: 640 }); + }); + + it('crops a portrait frame vertically', () => { + const { backendGraph } = compile(gptImage, 'txt2img', { + bbox: { height: 1000, width: 700, x: 0, y: 0 }, + compositeImageName: null, + }); + + // 1024×1536 scaled to cover 700×1000 is 700×1050, then 25 px come off the top and bottom. + expect(backendGraph.nodes.canvas_processing_output).toMatchObject({ height: 1536, width: 1024 }); + expect(backendGraph.nodes.canvas_provider_cover).toMatchObject({ height: 1050, width: 700 }); + expect(backendGraph.nodes.canvas_output).toMatchObject({ + height: 1000, + type: 'img_crop', + width: 700, + x: 0, + y: 25, + }); + }); + + it('resizes without cropping when the provider size has the bbox’s shape', () => { + const { backendGraph } = compile(gptImage, 'txt2img', { + bbox: { height: 512, width: 512, x: 0, y: 0 }, + compositeImageName: null, + }); + + expect(backendGraph.nodes.canvas_processing_output).toMatchObject({ height: 1024, width: 1024 }); + expect(backendGraph.nodes.canvas_provider_cover).toBeUndefined(); + expect(backendGraph.nodes.canvas_output).toMatchObject({ height: 512, type: 'img_resize', width: 512 }); + expect(getEdge(backendGraph, 'canvas_output', 'image')?.source.node_id).toBe('canvas_provider_image'); + }); + + it('resizes an image-to-image init to the provider size and its result plainly back, without cropping', () => { + const { backendGraph } = compile(gptImage, 'img2img', { bbox: smallBbox, compositeImageName: 'bbox.png' }); + + expect(backendGraph.nodes.canvas_processing_output).toMatchObject({ height: 1024, width: 1536 }); + expect(backendGraph.nodes.canvas_processing_output).not.toHaveProperty('init_image'); + expect(backendGraph.nodes.canvas_resize_image_to_request).toMatchObject({ + height: 1024, + image: { image_name: 'bbox.png' }, + type: 'img_resize', + width: 1536, + }); + expect(getEdge(backendGraph, 'canvas_processing_output', 'init_image')?.source.node_id).toBe( + 'canvas_resize_image_to_request' + ); + expect(backendGraph.nodes.canvas_provider_cover).toBeUndefined(); + expect(backendGraph.nodes.canvas_output).toMatchObject({ height: 512, type: 'img_resize', width: 640 }); + }); + + it('sends the smallest named size covering the frame, so a 2000 px square gets 2K', () => { + const gemini: GenerateModelConfig = { + ...externalModel, + capabilities: { + allowed_aspect_ratios: ['1:1', '16:9'], + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }, + { aspect_ratio: '1:1', height: 2048, image_size: '2K', label: '1:1 (2K)', width: 2048 }, + { aspect_ratio: '1:1', height: 4096, image_size: '4K', label: '1:1 (4K)', width: 4096 }, + ], + }, + provider_id: 'gemini', + }; + const { backendGraph } = compile(gemini, 'txt2img', { + bbox: { height: 2000, width: 2000, x: 0, y: 0 }, + compositeImageName: null, + }); + + expect(backendGraph.nodes.canvas_processing_output).toMatchObject({ + height: 2048, + image_size: '2K', + type: 'gemini_image_generation', + width: 2048, + }); + expect(backendGraph.nodes.canvas_output).toMatchObject({ height: 2000, type: 'img_resize', width: 2000 }); + expect(getNodeByType(backendGraph, 'core_metadata')).toMatchObject({ + height: 2000, + image_size: '2K', + width: 2000, + }); + }); + it('sends the composited bbox as the init image for img2img when the model advertises it', () => { const { backendGraph } = compile({ ...editModel, capabilities: { modes: ['img2img'] } }, 'img2img', { compositeImageName: 'bbox.png', @@ -547,13 +673,15 @@ describe('compileCanvasGraph', () => { ); }); - it('rejects the edit modes a model does not advertise, naming the mode', () => { - expect(() => compile(externalModel, 'img2img')).toThrow( - 'OpenAI Image does not support image-to-image generation.' - ); - expect(() => compile(externalModel, 'inpaint')).toThrow( - 'OpenAI Image does not support inpainting on the canvas yet.' - ); + it('names raster content in the frame as the cause for a text-only model, in every edit mode', () => { + for (const mode of ['img2img', 'outpaint', 'inpaint'] as const) { + expect(() => compile(externalModel, mode, { maskImageName: mode === 'inpaint' ? 'mask.png' : null })).toThrow( + 'OpenAI Image cannot use image content in the generation frame. Hide or move raster content out of the frame to generate.' + ); + } + }); + + it('rejects text-to-image for a model that does not advertise it, naming the mode', () => { expect(() => compile({ ...externalModel, capabilities: { modes: ['img2img'] } }, 'txt2img', { compositeImageName: null }) ).toThrow('OpenAI Image does not support text-to-image generation.'); @@ -606,7 +734,7 @@ describe('compileCanvasGraph', () => { expect(metadata).not.toHaveProperty('generation_mode'); }); - it('asks for the bbox itself even when the render size is set to auto or a custom size', () => { + it('asks a free-form provider for the bbox itself even when the render size is set to auto or a custom size', () => { const custom = compile(editModel, 'img2img', { bbox: smallBbox, compositeImageName: 'bbox.png', diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts index 21e617bd51a..5bfa1258f0f 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts @@ -1,6 +1,7 @@ /** * Pure canvas compiler over uploaded inputs. Resize through the processing grid and restore the bbox; hosted - * providers take the bbox composite for edits and restore their result to the bbox; reject unsupported modes. + * providers are asked for one of their own sizes, take the bbox composite for edits, and have their result restored + * to the bbox; reject unsupported modes. */ import type { SupportedGenerateBase } from '@features/generation/core/baseGenerationPolicies'; @@ -8,7 +9,11 @@ import type { BackendGraphContract, BackendInvocationContract } from '@features/ import type { ComponentModelConfig, GenerateModelConfig, GenerateSettings } from '@features/generation/core/types'; import { getGenerationValidationReasons } from '@features/generation/core/baseGenerationPolicies'; -import { isExternalGenerationMode } from '@features/generation/core/externalProvider'; +import { + describeExternalCanvasRefusal, + isExternalGenerationMode, + isExternalTextOnlyCanvasModel, +} from '@features/generation/core/externalProvider'; import { buildExternalGraph, findVaeSource, GRAPH_BUILDERS } from '@features/generation/core/graph'; import { addEdge, @@ -116,11 +121,13 @@ const getCanvasValidationReasons = (input: CompileCanvasGraphInput, processingSi const reasons: string[] = []; if (model.type === 'external_image_generator') { - // The Canvas keep-mask does not match provider edit-mask contracts, and there is no masked compositing tail. - if (mode === 'inpaint') { + if (mode !== 'txt2img' && isExternalTextOnlyCanvasModel(model)) { + // Every edit mode comes from raster content in the frame, which names the cause better than the mode does. + reasons.push(describeExternalCanvasRefusal(model, { kind: 'raster_content' })); + } else if (mode === 'inpaint') { + // The Canvas keep-mask does not match provider edit-mask contracts, and there is no masked compositing tail. reasons.push(`${model.name} does not support inpainting on the canvas yet. Remove or disable the inpaint mask.`); - } - if (mode === 'outpaint') { + } else if (mode === 'outpaint') { reasons.push(`${model.name} does not support outpainting on the canvas.`); } if (input.controlLayers && input.controlLayers.length > 0) { @@ -250,29 +257,67 @@ const resizeCanvasOutputToBbox = ( addEdge(graph, retired, 'image', output, 'image'); }; -/** Hosted nodes return an image collection; preserve each image's provider metadata through the resize. */ -const resizeExternalCanvasOutputToBbox = ( +/** + * Hosted nodes return an image collection; restore each image to the bbox with its own provider metadata. A + * text-to-image result at another aspect ratio is scaled to cover the bbox and center-cropped, never stretched. + * Scaling before cropping keeps the crop centered when a provider answers at another scale of the requested shape + * (named sizes such as Gemini's `2K`). An edit result is resized plainly, undoing its init image's resize. + */ +const restoreExternalCanvasOutputToBbox = ( graph: BackendGraphContract, - bbox: CompileCanvasGraphInput['bbox'], - destination: CompileCanvasGraphInput['destination'] + input: Pick & { mode: 'txt2img' | 'img2img' }, + requestSize: CanvasSize ): void => { + const { bbox, destination, mode } = input; const provider = renameNode(graph, 'canvas_output', 'canvas_processing_output'); provider.is_intermediate = true; const image = addNode(graph, { id: 'canvas_provider_image', type: 'iterate', use_cache: false }); const metadata = addNode(graph, { id: 'canvas_provider_metadata', type: 'metadata_from_image' }); - const output = addNode(graph, { - height: bbox.height, - id: 'canvas_output', - is_intermediate: destination === 'canvas', + const output = { id: 'canvas_output', is_intermediate: destination === 'canvas', use_cache: false }; + // Keep core metadata on the provider so its output adds request identity, defaults and actual seed to it. + addEdge(graph, provider, 'collection', image, 'collection'); + addEdge(graph, image, 'item', metadata, 'image'); + + if (mode === 'img2img' || requestSize.width * bbox.height === requestSize.height * bbox.width) { + const resize = addNode(graph, { ...output, height: bbox.height, type: 'img_resize', width: bbox.width }); + addEdge(graph, image, 'item', resize, 'image'); + addEdge(graph, metadata, 'metadata', resize, 'metadata'); + return; + } + + const scale = Math.max(bbox.width / requestSize.width, bbox.height / requestSize.height); + const cover = addNode(graph, { + height: Math.max(bbox.height, Math.round(requestSize.height * scale)), + id: 'canvas_provider_cover', + is_intermediate: true, type: 'img_resize', use_cache: false, + width: Math.max(bbox.width, Math.round(requestSize.width * scale)), + }); + const crop = addNode(graph, { + ...output, + height: bbox.height, + type: 'img_crop', width: bbox.width, + x: Math.floor((cover.width - bbox.width) / 2), + y: Math.floor((cover.height - bbox.height) / 2), }); - // Keep core metadata on the provider so its output adds request identity, defaults and actual seed to it. - addEdge(graph, provider, 'collection', image, 'collection'); - addEdge(graph, image, 'item', output, 'image'); - addEdge(graph, image, 'item', metadata, 'image'); - addEdge(graph, metadata, 'metadata', output, 'metadata'); + addEdge(graph, image, 'item', cover, 'image'); + addEdge(graph, cover, 'image', crop, 'image'); + addEdge(graph, metadata, 'metadata', crop, 'metadata'); +}; + +/** Feeds the provider the composite at the request size, as the backend would have resized it to its own bucket. */ +const resizeExternalInitImage = (graph: BackendGraphContract, imageName: string, requestSize: CanvasSize): void => { + const provider = graph.nodes.canvas_output as BackendInvocationContract; + delete provider.init_image; + addEdge( + graph, + addInputResizeNode(graph, 'canvas_resize_image_to_request', imageName, requestSize), + 'image', + provider, + 'init_image' + ); }; /** Keeps saved-image metadata aligned to the bbox footprint, not the internal processing size. */ @@ -788,8 +833,8 @@ export const compileCanvasGraph = (input: CompileCanvasGraphInput): CompiledCanv throw new Error(`${model.name} does not support ${mode}ing on the canvas.`); } - // Providers bucket the request to their own size table, so the returned whole image is restored to the exact - // bbox footprint before staging or saving. No local denoising strength applies. + // The provider is asked for the processing size, one of its own sizes, and the returned whole image is restored + // to the exact bbox footprint before staging or saving. No local denoising strength applies. const backendGraph = buildExternalGraph( settings, model, @@ -797,7 +842,10 @@ export const compileCanvasGraph = (input: CompileCanvasGraphInput): CompiledCanv runtimeProjectSettings, mode === 'txt2img' ? { mode } : { initImageName: compositeImageName as string, mode } ); - resizeExternalCanvasOutputToBbox(backendGraph, bbox, destination); + if (mode === 'img2img' && !sizesMatch(processingSize, bbox)) { + resizeExternalInitImage(backendGraph, compositeImageName as string, processingSize); + } + restoreExternalCanvasOutputToBbox(backendGraph, { bbox, destination, mode }, processingSize); setCanvasMetadataDimensions(backendGraph, bbox); return { diff --git a/invokeai/frontend/webv2/src/features/generation/core/dimensionLimits.ts b/invokeai/frontend/webv2/src/features/generation/core/dimensionLimits.ts new file mode 100644 index 00000000000..ada0b2f0394 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/core/dimensionLimits.ts @@ -0,0 +1,3 @@ +/** The generation size limits every model and the Canvas share; a leaf so provider policy can read them too. */ +export const MIN_DIMENSION = 64; +export const MAX_DIMENSION = 4096; diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts index e51abc6e7dc..38deea345dc 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.test.ts @@ -6,12 +6,15 @@ import { findExternalResolutionPreset, getExternalAspectRatioSizes, getExternalProviderOptionKeys, + getExternalReferenceImages, + resolveExternalCanvasRequestSize, resolveExternalImageSize, } from './externalProvider'; const external = ( providerId: string, - capabilities: ExternalImageGeneratorModelConfig['capabilities'] + capabilities: ExternalImageGeneratorModelConfig['capabilities'], + providerModelId = providerId === 'openai' ? 'gpt-image-1' : `${providerId}-model` ): GenerateModelConfig => ({ base: 'external', capabilities, @@ -19,6 +22,7 @@ const external = ( key: `${providerId}-model`, name: providerId, provider_id: providerId, + provider_model_id: providerModelId, type: 'external_image_generator', }); @@ -56,6 +60,40 @@ describe('external provider options', () => { ).toContain('openaiInputFidelity'); }); + it('offers no OpenAI options to DALL·E, which the provider sends none of', () => { + for (const modelId of ['dall-e-2', 'dall-e-3']) { + expect(getExternalProviderOptionKeys(external('openai', { modes: ['txt2img', 'img2img'] }, modelId))).toEqual([]); + } + expect(getExternalProviderOptionKeys(external('openai', { modes: ['txt2img'] }, 'gpt-image-2'))).toEqual([ + 'openaiQuality', + 'openaiBackground', + ]); + }); + + it('sends enabled reference images with an image, and none to a model that takes no references', () => { + const image = { + height: 64, + imageName: 'reference.png', + original: { image: { height: 64, image_name: 'reference.png', width: 64 } }, + width: 64, + }; + const settings = { + referenceImages: [ + { config: { image, type: 'external_reference_image' as const }, id: 'on', isEnabled: true }, + { config: { image, type: 'external_reference_image' as const }, id: 'off', isEnabled: false }, + { config: { image: null, type: 'external_reference_image' as const }, id: 'empty', isEnabled: true }, + ], + }; + + expect( + getExternalReferenceImages( + settings, + external('seedream', { modes: ['txt2img'], supports_reference_images: true }) + ) + ).toEqual([image]); + expect(getExternalReferenceImages(settings, external('seedream', { modes: ['txt2img'] }))).toEqual([]); + }); + it('resolves a named size only from the selected model’s own presets, at the preset’s exact size', () => { const gemini = external('gemini', { modes: ['txt2img'], resolution_presets: presets }); @@ -113,3 +151,108 @@ describe('external provider options', () => { expect(getExternalAspectRatioSizes(external('gemini', { modes: ['txt2img'] }))).toEqual([]); }); }); + +describe('Canvas request size', () => { + // The starter catalog's GPT Image and Seedream 2K size tables. + const gptImage = external('openai', { + aspect_ratio_sizes: { + '1:1': { height: 1024, width: 1024 }, + '3:2': { height: 1024, width: 1536 }, + '2:3': { height: 1536, width: 1024 }, + }, + modes: ['txt2img', 'img2img'], + }); + const seedream = external('seedream', { + aspect_ratio_sizes: { + '1:1': { height: 2048, width: 2048 }, + '4:3': { height: 1728, width: 2304 }, + '16:9': { height: 1600, width: 2848 }, + '21:9': { height: 1344, width: 3136 }, + }, + modes: ['txt2img', 'img2img'], + }); + + it('asks a fixed-size provider for its size at the ratio nearest the frame, whatever the frame’s size', () => { + expect(resolveExternalCanvasRequestSize(gptImage, { height: 512, width: 640 })).toEqual({ + height: 1024, + width: 1536, + }); + expect(resolveExternalCanvasRequestSize(gptImage, { height: 500, width: 520 })).toEqual({ + height: 1024, + width: 1024, + }); + expect(resolveExternalCanvasRequestSize(gptImage, { height: 3000, width: 1800 })).toEqual({ + height: 1536, + width: 1024, + }); + expect(resolveExternalCanvasRequestSize(seedream, { height: 1080, width: 1920 })).toEqual({ + height: 1600, + width: 2848, + }); + }); + + it('picks the smallest named size covering the frame at the nearest ratio, else the largest', () => { + const gemini = external('gemini', { + allowed_aspect_ratios: ['1:1', '16:9'], + modes: ['txt2img'], + resolution_presets: [ + { aspect_ratio: '1:1', height: 1024, image_size: '1K', label: '1:1 (1K)', width: 1024 }, + { aspect_ratio: '16:9', height: 576, image_size: '1K', label: '16:9 (1K)', width: 1024 }, + { aspect_ratio: '1:1', height: 2048, image_size: '2K', label: '1:1 (2K)', width: 2048 }, + { aspect_ratio: '1:1', height: 4096, image_size: '4K', label: '1:1 (4K)', width: 4096 }, + ], + }); + + expect(resolveExternalCanvasRequestSize(gemini, { height: 2000, width: 2000 })).toEqual({ + height: 2048, + width: 2048, + }); + expect(resolveExternalCanvasRequestSize(gemini, { height: 512, width: 512 })).toEqual({ + height: 1024, + width: 1024, + }); + expect(resolveExternalCanvasRequestSize(gemini, { height: 5000, width: 5000 })).toEqual({ + height: 4096, + width: 4096, + }); + // The only 16:9 size is too small for this frame, yet the shape wins over a larger size of another shape. + expect(resolveExternalCanvasRequestSize(gemini, { height: 1080, width: 1920 })).toEqual({ + height: 576, + width: 1024, + }); + }); + + it('asks a ratio-limited provider for an exact multiple of the nearest ratio covering the frame, within its maximum', () => { + const ratios = external('custom', { + allowed_aspect_ratios: ['1:1', '21:9', 'wide'], + max_image_size: { height: 1024, width: 2048 }, + modes: ['txt2img'], + }); + + expect(resolveExternalCanvasRequestSize(ratios, { height: 270, width: 630 })).toEqual({ height: 270, width: 630 }); + expect(resolveExternalCanvasRequestSize(ratios, { height: 500, width: 1000 })).toEqual({ + height: 504, + width: 1176, + }); + expect(resolveExternalCanvasRequestSize(ratios, { height: 1500, width: 1500 })).toEqual({ + height: 1024, + width: 1024, + }); + }); + + it('keeps an exact ratio multiple within the generation limit when the provider declares no maximum', () => { + const ratios = external('custom', { allowed_aspect_ratios: ['21:9'], modes: ['txt2img'] }); + + // The covering multiple would be 4116×1764; the largest one within 4096 px is 195 × (21, 9). + expect(resolveExternalCanvasRequestSize(ratios, { height: 1755, width: 4096 })).toEqual({ + height: 1755, + width: 4095, + }); + }); + + it('asks a free-form provider for the frame itself', () => { + expect( + resolveExternalCanvasRequestSize(external('atlascloud', { modes: ['txt2img'] }), { height: 333, width: 777 }) + ).toEqual({ height: 333, width: 777 }); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts index 207c118e2d4..a0d95c1eeda 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/externalProvider.ts @@ -10,12 +10,15 @@ import type { ExternalResolutionPreset, GeminiThinkingLevel, GenerateModelConfig, + GenerateReferenceImageAsset, GenerateSettings, OpenAIImageBackground, OpenAIImageQuality, OpenAIInputFidelity, } from './types'; +import { MAX_DIMENSION } from './dimensionLimits'; + export const OPENAI_IMAGE_QUALITIES: readonly OpenAIImageQuality[] = ['auto', 'high', 'medium', 'low']; export const OPENAI_IMAGE_BACKGROUNDS: readonly OpenAIImageBackground[] = ['auto', 'transparent', 'opaque']; export const OPENAI_INPUT_FIDELITIES: readonly OpenAIInputFidelity[] = ['low', 'high']; @@ -67,6 +70,14 @@ export type ExternalProviderOptionKey = keyof Pick< | 'seedreamOptimizePrompt' >; +/** OpenAI applies its options only to these models; mirrors `_GPT_IMAGE_MODELS` in `providers/openai.py`. */ +const OPENAI_GPT_IMAGE_MODEL_IDS: readonly unknown[] = [ + 'gpt-image-1', + 'gpt-image-1.5', + 'gpt-image-1-mini', + 'gpt-image-2', +]; + const getCapabilities = (model: GenerateModelConfig | undefined): ExternalModelCapabilities | null => model?.type === 'external_image_generator' ? (model.capabilities ?? null) : null; @@ -80,14 +91,17 @@ const acceptsImageInputs = (capabilities: ExternalModelCapabilities | null): boo /** * The provider options the selected model's invocation accepts, in display order. Each provider's node declares its - * own fields, so the provider decides; OpenAI's input fidelity is only consulted for image inputs, so a model that - * takes none does not offer it. + * own fields, so the provider decides. OpenAI sends them only for GPT Image models (DALL·E takes none), and input + * fidelity only with image inputs, so a model that takes none does not offer it. */ export const getExternalProviderOptionKeys = (model: GenerateModelConfig | undefined): ExternalProviderOptionKey[] => { const capabilities = getCapabilities(model); switch (getProviderId(model)) { case 'openai': + if (!OPENAI_GPT_IMAGE_MODEL_IDS.includes(model?.provider_model_id)) { + return []; + } return [ 'openaiQuality', 'openaiBackground', @@ -138,6 +152,131 @@ export const resolveExternalImageSize = ( model: GenerateModelConfig | undefined ): string | null => findExternalResolutionPreset(settings, model)?.image_size ?? null; +/** A "W:H" ratio of whole numbers, the only form the backend sizes from (`_size_for_ratio`). */ +const parseWholeRatio = (ratio: string): ExternalImageSize | null => { + const match = /^(\d+):(\d+)$/.exec(ratio); + const width = Number(match?.[1]); + const height = Number(match?.[2]); + + return width > 0 && height > 0 ? { height, width } : null; +}; + +/** + * The sizes a Canvas request may name, keyed by nominal ratio: the resolution presets, else the fixed size per ratio, + * else per whole-number allowed ratio its smallest exact multiple covering the frame within the maximum size and the + * generation limit. The backend's bucketing (`_bucket_request`) passes a fixed size (its own table entry) and an exact + * multiple (it reduces to a declared ratio) through unchanged. A preset whose rounded size is off its exact ratio may + * be nudged by a pixel or so, which is harmless: the provider sizes its output from the ratio and the preset's name, + * and the compiler restores whatever comes back to the bbox. + */ +const getCanvasRequestSizes = ( + model: GenerateModelConfig, + frame: ExternalImageSize +): Array<{ ratio: string; size: ExternalImageSize }> => { + const presets = getExternalResolutionPresets(model); + + if (presets.length > 0) { + return presets.map((preset) => ({ ratio: preset.aspect_ratio, size: preset })); + } + + const sizes = getExternalAspectRatioSizes(model); + + if (sizes.length > 0) { + return sizes.map(([ratio, size]) => ({ ratio, size })); + } + + const capabilities = getCapabilities(model); + const maxSize = capabilities?.max_image_size; + + return (capabilities?.allowed_aspect_ratios ?? []).flatMap((ratio) => { + const unit = parseWholeRatio(ratio); + + if (!unit) { + return []; + } + + const scale = Math.min( + Math.ceil(Math.max(frame.width / unit.width, frame.height / unit.height)), + Math.floor(Math.min(maxSize?.width ?? MAX_DIMENSION, MAX_DIMENSION) / unit.width), + Math.floor(Math.min(maxSize?.height ?? MAX_DIMENSION, MAX_DIMENSION) / unit.height) + ); + + return scale >= 1 ? [{ ratio, size: { height: unit.height * scale, width: unit.width * scale } }] : []; + }); +}; + +/** + * The size a Canvas frame asks a hosted provider for. A provider limited to certain shapes gets one of its own sizes, + * so the compiler knows the shape it gets back: the ratio closest to the frame's, then the smallest size of that ratio + * covering the frame, else the largest. A free-form provider gets the frame. + */ +export const resolveExternalCanvasRequestSize = ( + model: GenerateModelConfig, + frame: ExternalImageSize +): ExternalImageSize => { + const candidates = getCanvasRequestSizes(model, frame); + + if (candidates.length === 0) { + return { height: frame.height, width: frame.width }; + } + + const frameRatio = frame.width / frame.height; + const ratioDistance = ({ size }: (typeof candidates)[number]) => + Math.abs(Math.log(size.width / size.height / frameRatio)); + const closest = candidates.reduce((best, candidate) => + ratioDistance(candidate) < ratioDistance(best) ? candidate : best + ); + const area = ({ size }: (typeof candidates)[number]) => size.width * size.height; + const sameRatio = candidates.filter((candidate) => candidate.ratio === closest.ratio); + const covering = sameRatio.filter(({ size }) => size.width >= frame.width && size.height >= frame.height); + const { size } = + covering.length > 0 + ? covering.reduce((best, candidate) => (area(candidate) < area(best) ? candidate : best)) + : sameRatio.reduce((best, candidate) => (area(candidate) > area(best) ? candidate : best)); + + return { height: size.height, width: size.width }; +}; + +/** The reference images a request sends: none unless the model takes them. */ +export const getExternalReferenceImages = ( + settings: Pick, + model: GenerateModelConfig +): GenerateReferenceImageAsset[] => + getCapabilities(model)?.supports_reference_images === true + ? settings.referenceImages.flatMap((referenceImage) => + referenceImage.isEnabled && + referenceImage.config.type === 'external_reference_image' && + referenceImage.config.image !== null + ? [referenceImage.config.image] + : [] + ) + : []; + +/** + * Canvas content a hosted model cannot take, worded once for readiness and for the invoke-time check that measures + * pixel coverage. Raster content in the frame selects an edit mode, which a model without image-to-image cannot run. + */ +export const describeExternalCanvasRefusal = ( + model: Pick, + refusal: { kind: 'control_layer' | 'regional_guidance'; layerName: string } | { kind: 'raster_content' } +): string => { + switch (refusal.kind) { + case 'control_layer': + return `${model.name} does not support control layer "${refusal.layerName}". Disable it to generate.`; + case 'regional_guidance': + return `${model.name} does not support regional guidance layer "${refusal.layerName}". Disable it to generate.`; + case 'raster_content': + return `${model.name} cannot use image content in the generation frame. Hide or move raster content out of the frame to generate.`; + } +}; + +/** + * Whether a hosted model only generates from text on the Canvas. Image-to-image is the only provider edit mode the + * Canvas compiles, so a model without it (even one advertising inpaint) cannot use content in the frame. + */ +export const isExternalTextOnlyCanvasModel = (model: GenerateModelConfig): boolean => + model.type === 'external_image_generator' && !(model.capabilities?.modes ?? ['txt2img']).includes('img2img'); + /** * Provider-specific invocation fields for the selected model, keyed as the node declares them. Optional fields are * omitted when unset so the node applies its own default. diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts index 7acd352f442..f62a66117f2 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts @@ -642,7 +642,13 @@ describe('compileGenerateGraph', () => { const providerModel = ( providerId: string, capabilities: ExternalImageGeneratorModelConfig['capabilities'] - ): GenerateModelConfig => ({ ...externalModel, capabilities, provider_id: providerId }) as GenerateModelConfig; + ): GenerateModelConfig => + ({ + ...externalModel, + capabilities, + provider_id: providerId, + provider_model_id: providerId === 'openai' ? 'gpt-image-1' : `${providerId}-model`, + }) as GenerateModelConfig; it('compiles OpenAI quality, background and input fidelity, recording them in metadata', () => { const model = providerModel('openai', { modes: ['txt2img', 'img2img'] }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.ts index 557dda83ba0..1ed4d7138aa 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.ts @@ -46,6 +46,7 @@ import { import { getExternalProviderInvocationOptions, getExternalProviderMetadata, + getExternalReferenceImages, resolveExternalImageSize, } from './externalProvider'; import { @@ -1652,15 +1653,10 @@ export const buildExternalGraph = ( if (model.capabilities?.supports_seed === true) { addEdge(graph, seed, 'value', output, 'seed'); } - if (model.capabilities?.supports_reference_images === true) { - const referenceImages = getEnabledReferenceImages(settings, 'external_reference_image') - .map((referenceImage) => referenceImage.config.image) - .filter((image): image is GenerateReferenceImageAsset => image !== null) - .map(toImageField); - - if (referenceImages.length > 0) { - (output as Record).reference_images = referenceImages; - } + const referenceImages = getExternalReferenceImages(settings, model).map(toImageField); + + if (referenceImages.length > 0) { + (output as Record).reference_images = referenceImages; } addMetadata(graph, output, settings, model, null, projectSettings, { scheduler: undefined, diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.ts index 02da1f21d04..bdcf28de9d9 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.ts @@ -29,6 +29,7 @@ import { isValidKrea2RebalanceWeights, KREA2_REBALANCE_WEIGHT_COUNT, } from './conditioningRebalance'; +import { MAX_DIMENSION, MIN_DIMENSION } from './dimensionLimits'; import { isDynamicPromptsSeedBehaviour, sanitizeMaxPrompts, sanitizeSampleSeed } from './dynamicPrompts'; import { GEMINI_TEMPERATURE_MAX, @@ -93,8 +94,6 @@ export const isAspectRatioId = (value: unknown): value is AspectRatioId => value === 'Free' || (typeof value === 'string' && value in ASPECT_RATIO_MAP); export const DIMENSION_GRID = 8; -export const MIN_DIMENSION = 64; -export const MAX_DIMENSION = 4096; /** The expansion config the settings describe, under the names the expansion route and preview use. */ export const getDynamicPromptsConfig = ( diff --git a/invokeai/frontend/webv2/src/features/generation/react.ts b/invokeai/frontend/webv2/src/features/generation/react.ts index ff7232243b9..abb17daaeed 100644 --- a/invokeai/frontend/webv2/src/features/generation/react.ts +++ b/invokeai/frontend/webv2/src/features/generation/react.ts @@ -10,7 +10,7 @@ export { flushGenerateDrafts, useRegisterGenerateDraftFlusher } from './ui/gener export { useDebouncedDraftValue } from './ui/useDebouncedDraftValue'; export { useDynamicPrompts, type DynamicPromptsExpansion } from './ui/useDynamicPrompts'; export { createGenerateFormValuesSelector } from './ui/generateFormViewModel'; -export { notifyGenerateModelSelectionCleared } from './ui/modelSelectionNotice'; +export { notifyGenerateDimensionsAdjusted, notifyGenerateModelSelectionCleared } from './ui/modelSelectionNotice'; export { adjustFocusedPromptAttention } from './ui/promptFields/promptAttentionHotkeys'; export { focusPositivePrompt } from './ui/promptFields/promptFocus'; export { promptHistoryNavigation } from './ui/promptFields/promptHistoryNavigation'; diff --git a/invokeai/frontend/webv2/src/features/generation/settings.ts b/invokeai/frontend/webv2/src/features/generation/settings.ts index 78cc6a3eed8..d7e17c305ea 100644 --- a/invokeai/frontend/webv2/src/features/generation/settings.ts +++ b/invokeai/frontend/webv2/src/features/generation/settings.ts @@ -15,6 +15,7 @@ export { hasModelDefaultVae, isAspectRatioLocked, isFluxDypePreset, + isGenerateModelConfig, isLoraCompatibleWithModel, isLoraModelConfig, isMainModelConfig, @@ -67,14 +68,17 @@ export { } from './core/baseGenerationPolicies'; export { hasArchitectureCapabilities } from './core/architectureCapabilities'; export { + describeExternalCanvasRefusal, GEMINI_TEMPERATURE_MAX, GEMINI_TEMPERATURE_MIN, getExternalProviderOptionKeys, getExternalResolutionPresets, + isExternalTextOnlyCanvasModel, isGeminiThinkingLevel, isOpenAIImageBackground, isOpenAIImageQuality, isOpenAIInputFidelity, + resolveExternalCanvasRequestSize, } from './core/externalProvider'; export { resolveGenerateWidgetValues, diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx index 7dda83397ff..e61dce60519 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx @@ -3,14 +3,13 @@ import type { AspectRatioId, GenerateModelConfig, GenerateSettings } from '@feat import { Badge, Box, HStack, Icon, Stack, Text } from '@chakra-ui/react'; import { getDefaultGenerateSettings, getGenerationDimensions } from '@features/generation/core/baseGenerationPolicies'; +import { MAX_DIMENSION, MIN_DIMENSION } from '@features/generation/core/dimensionLimits'; import { ASPECT_RATIO_MAP, calculateNewSize, clampDimension, deriveAspectRatioId, isAspectRatioLocked, - MAX_DIMENSION, - MIN_DIMENSION, } from '@features/generation/core/settings'; import { useExternalStoreSelector } from '@platform/state/selectors'; import { Button, IconButton, Tooltip } from '@platform/ui'; diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateModelCard.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateModelCard.tsx index dcfb5e10cf9..888c9e33fcf 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateModelCard.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateModelCard.tsx @@ -18,9 +18,22 @@ import { useTranslation } from 'react-i18next'; import type { GenerateDraft } from './generateDebounce'; import { GenerationModelSelect as ModelSelect, useGenerationUi } from './GenerationUiContext'; +import { notifyGenerateDimensionsAdjusted } from './modelSelectionNotice'; const MAIN_MODEL_TYPES = ['main', 'external_image_generator']; -const NO_CLEARED_LABELS: readonly string[] = []; + +/** What a pending switch would change in the live draft: settings it removes, and the size it moves to, if any. */ +interface PendingSwitchPreview { + clearedLabels: readonly string[]; + adjustedSize: { height: number; width: number } | null; +} + +const NO_PENDING_SWITCH: PendingSwitchPreview = { adjustedSize: null, clearedLabels: [] }; + +const arePreviewsEqual = (left: PendingSwitchPreview, right: PendingSwitchPreview) => + areArraysEqual(left.clearedLabels, right.clearedLabels) && + left.adjustedSize?.width === right.adjustedSize?.width && + left.adjustedSize?.height === right.adjustedSize?.height; interface GenerateModelCardProps { draft: GenerateDraft; @@ -50,14 +63,24 @@ export const GenerateModelCard = ({ const [pendingSwitchModel, setPendingSwitchModel] = useState(null); // Constant until a switch awaits confirmation; then the open dialog lists what the latest draft would lose. - const pendingSwitchClearedLabels = useExternalStoreSelector( + const pendingSwitch = useExternalStoreSelector( draft.subscribe, draft.getSnapshot, - (settings: GenerateSettings) => - pendingSwitchModel - ? getGenerateModelSelectionResult({ currentValues: settings, model: pendingSwitchModel, models }).clearedLabels - : NO_CLEARED_LABELS, - areArraysEqual + (settings: GenerateSettings): PendingSwitchPreview => { + if (!pendingSwitchModel) { + return NO_PENDING_SWITCH; + } + + const result = getGenerateModelSelectionResult({ currentValues: settings, model: pendingSwitchModel, models }); + + return { + adjustedSize: result.dimensionsAdjusted + ? { height: result.settings.height, width: result.settings.width } + : null, + clearedLabels: result.clearedLabels, + }; + }, + arePreviewsEqual ); const commitModelSelection = (model: GenerateModelConfig) => { @@ -67,13 +90,14 @@ export const GenerateModelCard = ({ const selectModel = (model: GenerateModelConfig) => { const result = getGenerateModelSelectionResult({ currentValues: draft.getSnapshot(), model, models }); - // Lossy switches confirm before committing; lossless ones stay instant. + // Lossy switches confirm before committing; lossless ones stay instant and report a moved size. if (result.clearedLabels.length > 0) { setPendingSwitchModel(model); return; } onCommitSettings(result.settings); + notifyGenerateDimensionsAdjusted({ modelName: model.name, notifications: ui.notifications, result, t }); }; const hasNoSupportedModels = !isLoadingModels && !loadError && supportedModels.length === 0; @@ -116,14 +140,21 @@ export const GenerateModelCard = ({ - {t('widgets.generate.switchModelBody', { - labels: new Intl.ListFormat(i18n.resolvedLanguage, { style: 'long', type: 'conjunction' }).format( - pendingSwitchClearedLabels - ), - name: pendingSwitchModel?.name ?? '', - })} - + <> + + {t('widgets.generate.switchModelBody', { + labels: new Intl.ListFormat(i18n.resolvedLanguage, { style: 'long', type: 'conjunction' }).format( + pendingSwitch.clearedLabels + ), + name: pendingSwitchModel?.name ?? '', + })} + + {pendingSwitch.adjustedSize ? ( + + {t('widgets.generate.switchModelAdjustsDimensions', pendingSwitch.adjustedSize)} + + ) : null} + } confirmLabel={t('widgets.generate.switchModelConfirm')} isOpen={pendingSwitchModel !== null} diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.browser.test.tsx index a1b4075f1d0..d06e1db9b30 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.browser.test.tsx @@ -130,6 +130,36 @@ describe('GeneratePresetsPopover', () => { } ); + it('names the size a preset moved to when its stored size does not fit the model', async () => { + const model: MainModelConfig = { base: 'sdxl', key: 'sdxl', name: 'SDXL', type: 'main' }; + // 1028 is off SDXL's 8 px grid, as a preset saved from a hosted model's size can be. + const preset = { ...getDefaultGenerateSettings(model), height: 1024, width: 1028 }; + // The current size differs from the preset's, so the preset is not shown as active. + const currentValues = { ...preset, model, width: 1024 }; + const info = vi.fn(); + const patchGenerateSettings = vi.fn(); + const adapter = { + ...ADAPTER, + generateValues: { getSnapshot: () => currentValues, subscribe: () => () => {} }, + models: { ...ADAPTER.models, catalog: [model] }, + presets: { ...ADAPTER.presets, presets: [{ id: 'off-grid', label: 'Off grid', values: preset }] }, + notifications: { ...ADAPTER.notifications, info }, + settings: { ...ADAPTER.settings, patchGenerateSettings }, + } as unknown as GenerationUiAdapter; + await renderAdapter(adapter); + await act(() => userEvent.click(document.querySelector('[aria-label="widgets.generate.presets"]')!)); + const presetButton = [...document.querySelectorAll('button')].find( + (button) => button.textContent === 'Off grid' + ); + await act(() => presetButton!.click()); + + expect(patchGenerateSettings).toHaveBeenCalledWith(expect.objectContaining({ width: 1032 }), 'project-1'); + expect(info).toHaveBeenCalledWith( + 'widgets.generate.dimensionsAdjusted', + 'widgets.generate.dimensionsAdjustedDescription' + ); + }); + it('animates the rename dialog out instead of unmounting it on close', async () => { await openRename(0); expect(renameInput()?.value).toBe('Portrait'); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.tsx index 9c92a0c5fd4..70fe6b0d383 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GeneratePresetsPopover.tsx @@ -28,7 +28,7 @@ import type { GeneratePresetRecord } from './GenerationUiContext'; import { flushGenerateDrafts } from './generateDraftRegistry'; import { getGenerateFormCommitPatch } from './generateFormViewModel'; import { useGenerateValues, useGenerationUi } from './GenerationUiContext'; -import { notifyGenerateModelSelectionCleared } from './modelSelectionNotice'; +import { notifyGenerateDimensionsAdjusted, notifyGenerateModelSelectionCleared } from './modelSelectionNotice'; import { PANEL_HEADER_CONTROL_HEIGHT, PromptPanelHeader } from './promptFields/PromptPanelHeader'; const POPOVER_POSITIONING = { placement: 'bottom-end' } as const; @@ -223,6 +223,7 @@ export const GeneratePresetsPopover = () => { reason: 'preset', t, }); + notifyGenerateDimensionsAdjusted({ modelName: model.name, notifications: ui.notifications, result, t }); setIsOpen(false); }, [i18n.resolvedLanguage, models, projectId, settings, supportedModels, t, ui.notifications, ui.settings] diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx index 89da813f69a..c839470cab1 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateProviderOptionsSection.browser.test.tsx @@ -56,7 +56,11 @@ void i18n.use(initReactI18next).init({ }, }); -const external = (providerId: string, capabilities: Record): GenerateModelConfig => +const external = ( + providerId: string, + capabilities: Record, + providerModelId = providerId === 'openai' ? 'gpt-image-1' : `${providerId}-model` +): GenerateModelConfig => ({ base: 'external', capabilities, @@ -64,6 +68,7 @@ const external = (providerId: string, capabilities: Record): Ge key: `${providerId}-model`, name: providerId, provider_id: providerId, + provider_model_id: providerModelId, type: 'external_image_generator', }) as GenerateModelConfig; const sd1Model: MainModelConfig = { base: 'sd-1', key: 'sd1', name: 'SD 1.5', type: 'main' }; @@ -131,6 +136,11 @@ describe('GenerateProviderOptionsSection', () => { await settle(() => root?.unmount()); await render(external('alibabacloud', { modes: ['txt2img'] })); expect(host?.childElementCount).toBe(0); + + // OpenAI sends its options to GPT Image models only. + await settle(() => root?.unmount()); + await render(external('openai', { modes: ['txt2img'] }, 'dall-e-3')); + expect(host?.childElementCount).toBe(0); }); it('offers OpenAI quality and background, adding input fidelity only for image inputs', async () => { diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.browser.test.tsx index 4c83f094b9f..35ff9a6bd3e 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.browser.test.tsx @@ -639,6 +639,25 @@ describe('GenerateSettingsForm render isolation', () => { expect(stepsValue()).toBe(31); }); + it('switches at once when only the size moves, reporting the new size', async () => { + const info = vi.fn(); + stableGroups.notifications.info = info; + stableGroups.models.catalog = [MODEL, FLUX]; + await renderAdapter(buildAdapter()); + // 1032 sits on SDXL's 8 px grid but not FLUX's 16 px one. + await pressRight(sliderNamed('widgets.generate.width')); + + await settle(() => host!.querySelector('[data-model-pick="flux"]')?.click()); + + expect(document.querySelector('[role="alertdialog"]')).toBeNull(); + expect(patches.at(-1)?.patch).toMatchObject({ modelKey: 'flux' }); + expect((patches.at(-1)?.patch.width as number) % 16).toBe(0); + expect(info).toHaveBeenCalledWith( + 'widgets.generate.dimensionsAdjusted', + 'widgets.generate.dimensionsAdjustedDescription' + ); + }); + it('confirms a lossy switch against the draft, pending edits included', async () => { stableGroups.models.catalog = [MODEL, FLUX, ink]; storedValues().patchSnapshot({ loras: [{ isEnabled: true, model: ink, weight: 0.75 }] }); @@ -650,7 +669,9 @@ describe('GenerateSettingsForm render isolation', () => { await settle(() => host!.querySelector('[data-model-pick="flux"]')?.click()); const dialog = document.querySelector('[role="alertdialog"]'); - expect(dialog?.textContent).toContain('widgets.generate.switchModelBody: Dimensions and LoRAs'); + // The size moves to the FLUX grid, which the dialog reports beside the loss rather than as one. + expect(dialog?.textContent).toContain('widgets.generate.switchModelBody: LoRAs'); + expect(dialog?.textContent).toContain('widgets.generate.switchModelAdjustsDimensions'); expect(patches).toEqual([]); const confirm = [...dialog!.querySelectorAll('button')].find( @@ -661,7 +682,7 @@ describe('GenerateSettingsForm render isolation', () => { // The switch lands as the dialog closes, and the dialog animates out still listing what was confirmed. expect(frames).not.toHaveLength(0); for (const frame of frames) { - expect(frame.text).toContain('widgets.generate.switchModelBody: Dimensions and LoRAs'); + expect(frame.text).toContain('widgets.generate.switchModelBody: LoRAs'); } await settle(noop, 400); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.test.ts b/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.test.ts index 4a5083ca27b..d545a13933c 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.test.ts @@ -1,8 +1,9 @@ +import type { GenerateSettings } from '@features/generation/core/types'; import type { TFunction } from 'i18next'; import { describe, expect, it, vi } from 'vitest'; -import { notifyGenerateModelSelectionCleared } from './modelSelectionNotice'; +import { notifyGenerateDimensionsAdjusted, notifyGenerateModelSelectionCleared } from './modelSelectionNotice'; const createTranslation = () => vi.fn((key: string, options?: Record) => { @@ -71,3 +72,38 @@ describe('notifyGenerateModelSelectionCleared', () => { ); }); }); + +describe('notifyGenerateDimensionsAdjusted', () => { + const t = vi.fn((key: string, options?: Record) => + options ? `${key} ${JSON.stringify(options)}` : key + ) as unknown as TFunction; + + it('names the size the switch moved to', () => { + const info = vi.fn(); + + notifyGenerateDimensionsAdjusted({ + modelName: 'Seedream 4.5', + notifications: { info }, + result: { dimensionsAdjusted: true, settings: { height: 1600, width: 2848 } as GenerateSettings }, + t, + }); + + expect(info).toHaveBeenCalledWith( + 'widgets.generate.dimensionsAdjusted', + 'widgets.generate.dimensionsAdjustedDescription {"height":1600,"name":"Seedream 4.5","width":2848}' + ); + }); + + it('stays quiet when the size already fit', () => { + const info = vi.fn(); + + notifyGenerateDimensionsAdjusted({ + modelName: 'SDXL', + notifications: { info }, + result: { dimensionsAdjusted: false, settings: { height: 1024, width: 1024 } as GenerateSettings }, + t, + }); + + expect(info).not.toHaveBeenCalled(); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.ts b/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.ts index b64052a88df..396b41d1e85 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.ts +++ b/invokeai/frontend/webv2/src/features/generation/ui/modelSelectionNotice.ts @@ -1,3 +1,4 @@ +import type { GenerateModelSelectionResult } from '@features/generation/core/baseGenerationPolicies'; import type { TFunction } from 'i18next'; interface GenerateModelSelectionNotifier { @@ -39,3 +40,29 @@ export const notifyGenerateModelSelectionCleared = ({ ) ); }; + +/** A switch that only moves the size is not confirmed, so this names the size it moved to. */ +export const notifyGenerateDimensionsAdjusted = ({ + modelName, + notifications, + result, + t, +}: { + modelName: string; + notifications: GenerateModelSelectionNotifier; + result: Pick; + t: TFunction; +}): void => { + if (!result.dimensionsAdjusted) { + return; + } + + notifications.info( + t('widgets.generate.dimensionsAdjusted'), + t('widgets.generate.dimensionsAdjustedDescription', { + height: result.settings.height, + name: modelName, + width: result.settings.width, + }) + ); +}; diff --git a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts index 613debb5809..12d6f0ce1a6 100644 --- a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.test.ts @@ -105,19 +105,31 @@ describe('external model record mapping', () => { expect(changes.default_settings).toBeNull(); }); - it('rejects an empty route, no modes, half a max size, and non-positive limits', () => { + it('rejects an empty route, no modes, half a max size, and non-positive limits, naming the field to mark', () => { const values = toExternalModelValues(geminiModel); expect(validateExternalModel(values)).toBeNull(); - expect(validateExternalModel({ ...values, providerModelId: ' ' })).toBe('Provider model ID is required.'); - expect(validateExternalModel({ ...values, modeTxt2img: false })).toBe('Choose at least one supported mode.'); - expect(validateExternalModel({ ...values, maxImageHeight: null })).toBe( - 'Set both a max width and a max height, or neither.' - ); - expect(validateExternalModel({ ...values, maxImagesPerRequest: 0 })).toBe( - 'Images per request must be greater than 0.' - ); - expect(validateExternalModel({ ...values, defaultWidth: 1024.5 })).toBe('Default width must be a whole number.'); + expect(validateExternalModel({ ...values, providerModelId: ' ' })).toEqual({ + field: 'providerModelId', + message: 'Provider model ID is required.', + }); + expect(validateExternalModel({ ...values, modeTxt2img: false })).toEqual({ + field: 'modeTxt2img', + message: 'Choose at least one supported mode.', + }); + expect(validateExternalModel({ ...values, maxImageHeight: null })).toEqual({ + field: 'maxImageHeight', + message: 'Set both a max width and a max height, or neither.', + }); + expect(validateExternalModel({ ...values, maxImageWidth: null })?.field).toBe('maxImageWidth'); + expect(validateExternalModel({ ...values, maxImagesPerRequest: 0 })).toEqual({ + field: 'maxImagesPerRequest', + message: 'Images per request must be greater than 0.', + }); + expect(validateExternalModel({ ...values, defaultWidth: 1024.5 })).toEqual({ + field: 'defaultWidth', + message: 'Default width must be a whole number.', + }); }); it('recognises a record the starter catalog manages by its source, or by the source an install would record', () => { diff --git a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts index f24e0d6eac6..2f25eb8353a 100644 --- a/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts +++ b/invokeai/frontend/webv2/src/features/models/core/externalModelFields.ts @@ -92,9 +92,21 @@ export const toExternalModelChanges = ( }; }; -/** The first schema message, like the other record editors' messages, so the UI shows it as is. */ -export const validateExternalModel = (values: ExternalModelFormValues): string | null => { +export interface ExternalModelValidationIssue { + /** The field to mark invalid; a missing mode names the first mode flag, which stands for the whole group. */ + field: keyof ExternalModelFormValues; + message: string; +} + +/** The first schema issue, with its message shown as is like the other record editors' messages. */ +export const validateExternalModel = (values: ExternalModelFormValues): ExternalModelValidationIssue | null => { const result = externalModelSchema.safeParse(values); + const issue = result.success ? undefined : result.error.issues[0]; + + if (issue === undefined) { + return null; + } - return result.success ? null : (result.error.issues[0]?.message ?? 'Invalid provider settings.'); + // Every schema issue names one of the form's own keys. + return { field: issue.path[0] as keyof ExternalModelFormValues, message: issue.message }; }; diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx index 60b2fde501d..04c57c91a93 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.browser.test.tsx @@ -1,10 +1,13 @@ import type { ExternalModelSettingsModel } from '@features/models/core/externalModelFields'; +import type { ModelConfig } from '@features/models/core/types'; import type * as ModelsApi from '@features/models/data/api'; import { ChakraProvider } from '@chakra-ui/react'; import { toExternalModelValues } from '@features/models/core/externalModelFields'; +import { setModelsSnapshotForTests, useModelsSelector } from '@features/models/data/modelsStore'; import { recordModelDraftFields, getModelDraft } from '@features/models/ui/modelDraftsStore'; import { accountLifecycle } from '@platform/state/accountLifecycle'; +import { ApiError } from '@platform/transport/http'; import { system } from '@theme/system'; import { act } from 'react'; import { createRoot, type Root } from 'react-dom/client'; @@ -55,6 +58,13 @@ const gptImage: ExternalModelSettingsModel = { type: 'external_image_generator', }; +// Reads the record from the library store, as the model detail does, so a save's reply reaches the form. +const LibraryBackedSection = ({ onError, onSaved }: { onError: (message: string) => void; onSaved: () => void }) => { + const model = useModelsSelector((snapshot) => snapshot.models.find((candidate) => candidate.key === gptImage.key)); + + return model ? : null; +}; + const setInputValue = (input: HTMLInputElement, value: string): void => { const setter = Object.getOwnPropertyDescriptor(window.HTMLInputElement.prototype, 'value')!.set!; @@ -185,21 +195,56 @@ describe('ExternalModelSection', () => { expect(getModelDraft('gpt-image-1')?.external).toBeUndefined(); }); - it('refuses to save an invalid record and reports why without calling the backend', async () => { + it('refuses to save an invalid record, marking the mode group and moving focus to it', async () => { await render(gptImage); await click(checkbox('models.externalModes.txt2img')); await click(checkbox('models.externalModes.img2img')); await click(button('models.saveExternalSettings')); - expect(host.querySelector('[role="alert"]')?.textContent).toBe('Choose at least one supported mode.'); + const group = host.querySelector('[role="group"][aria-labelledby]')!; + const alert = group.querySelector('[role="alert"]')!; + expect(document.getElementById(group.getAttribute('aria-labelledby')!)?.textContent).toBe( + 'models.externalFields.supportedModes' + ); + expect(alert.textContent).toBe('Choose at least one supported mode.'); + expect(group.getAttribute('aria-describedby')).toBe(alert.id); + expect(host.querySelectorAll('[role="alert"]')).toHaveLength(1); + expect(checkbox('models.externalModes.inpaint').getAttribute('aria-invalid')).toBe('true'); + expect(document.activeElement).toBe(checkbox('models.externalModes.txt2img')); expect(api.updateModel).not.toHaveBeenCalled(); // Reset discards the draft and the error together. await click(button('common.reset')); expect(host.querySelector('[role="alert"]')).toBeNull(); + expect(checkbox('models.externalModes.inpaint').getAttribute('aria-invalid')).not.toBe('true'); expect(checkbox('models.externalModes.txt2img').checked).toBe(true); }); + it('marks the limit field a failed validation names and describes it with the message', async () => { + await render(gptImage); + + await type(input('models.externalFields.maxImageWidth'), '2048'); + // Focus and blur in one task leave the number input a refocus for the next frame; a user's click comes later. + await settle(); + await click(button('models.saveExternalSettings')); + + const height = input('models.externalFields.maxImageHeight'); + const message = [...host.querySelectorAll('[role="alert"]')]; + expect(message.map((element) => element.textContent)).toEqual([ + 'Set both a max width and a max height, or neither.', + ]); + expect(height.getAttribute('aria-invalid')).toBe('true'); + expect(height.getAttribute('aria-describedby')?.split(' ')).toContain(message[0]!.id); + expect(input('models.externalFields.maxImageWidth').getAttribute('aria-invalid')).toBeNull(); + expect(document.activeElement).toBe(height); + expect(api.updateModel).not.toHaveBeenCalled(); + + // Editing clears the marking along with the message. + await type(height, '2048'); + expect(height.getAttribute('aria-invalid')).toBeNull(); + expect(host.querySelector('[role="alert"]')).toBeNull(); + }); + it('keeps the saved unregistered provider selectable after switching without discarding other edits', async () => { await render({ ...gptImage, provider_id: 'custom' }); await settle(); @@ -286,14 +331,49 @@ describe('ExternalModelSection', () => { expect(host.querySelectorAll('input[type="checkbox"]')).toHaveLength(0); expect(getModelDraft('gpt-image-1')?.external).toBeUndefined(); - await click(button('common.retry')); + let resolveCatalog!: (response: Awaited>) => void; + api.getStarterModels.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveCatalog = resolve; + }) + ); + const retry = button('common.retry'); + retry.focus(); + await click(retry); + // The button stays mounted and focused while the catalog reloads, and refuses a second request. + expect(document.activeElement).toBe(retry); + expect(retry.getAttribute('aria-busy')).toBe('true'); + expect(retry.getAttribute('aria-disabled')).toBe('true'); + await click(retry); + expect(api.getStarterModels).toHaveBeenCalledTimes(2); + + await act(async () => { + resolveCatalog({ starter_bundles: {}, starter_models: [] }); + await Promise.resolve(); + }); await settle(); expect(host.querySelector('[role="alert"]')).toBeNull(); expect(providerTrigger()?.disabled).toBe(false); + // Focus moves to the editor the catalog unlocked rather than falling to the page. + expect(document.activeElement).toBe(providerTrigger()); expect(checkbox('models.externalModes.txt2img').checked).toBe(true); expect(getModelDraft('gpt-image-1')?.external).toBeUndefined(); }); + it('moves focus to the catalog note when a retry finds the record catalog-managed', async () => { + api.getStarterModels.mockRejectedValueOnce(new Error('offline')); + api.getStarterModels.mockResolvedValueOnce({ starter_bundles: {}, starter_models: [{ source: gptImage.source! }] }); + await render(gptImage); + + const retry = button('common.retry'); + retry.focus(); + await click(retry); + await settle(); + + expect(document.activeElement?.textContent).toBe('models.externalFields.starterManaged'); + }); + it('explains a retained draft while a failed catalog fetch keeps the record locked', async () => { api.getStarterModels.mockRejectedValueOnce(new Error('offline')); const saved = toExternalModelValues(gptImage); @@ -342,15 +422,57 @@ describe('ExternalModelSection', () => { expect(host.textContent).toContain('10'); }); - it('surfaces a failed save and keeps the edits for another attempt', async () => { - api.updateModel.mockRejectedValue(new Error('Model not found')); + it('surfaces a refused save as the readable validation message and keeps the edits for another attempt', async () => { + api.updateModel.mockRejectedValue( + new ApiError( + JSON.stringify({ + detail: [ + { + ctx: { gt: 0 }, + input: 0, + loc: ['body', 'changes', 'capabilities', 'max_images_per_request'], + msg: 'Input should be greater than 0', + type: 'greater_than', + }, + ], + }), + 422 + ) + ); await render(gptImage); await click(checkbox('models.externalFields.supportsSeed')); await click(button('models.saveExternalSettings')); - expect(onError).toHaveBeenCalledWith('Model not found'); + expect(onError).toHaveBeenCalledOnce(); + const [message] = onError.mock.calls[0]!; + expect(message).toContain('max images per request'); + expect(message).not.toContain('{'); expect(checkbox('models.externalFields.supportsSeed').checked).toBe(true); expect(button('models.saveExternalSettings').disabled).toBe(false); }); + + it('shows what the backend saved, not what was sent, once the save lands', async () => { + setModelsSnapshotForTests({ models: [gptImage as ModelConfig], status: 'loaded' }); + // The backend normalizes the route it stores. + api.updateModel.mockImplementation((_key: string, changes: Record) => + Promise.resolve({ ...gptImage, ...changes, provider_model_id: 'gpt-image-1.5-2026' }) + ); + await act(() => { + root.render( + + + + ); + }); + await settle(); + + await type(host.querySelector('input[value="gpt-image-1"]')!, 'gpt-image-1.5'); + await click(button('models.saveExternalSettings')); + + expect(onSaved).toHaveBeenCalledOnce(); + expect(host.querySelector('input[value="gpt-image-1.5-2026"]')).not.toBeNull(); + expect(host.textContent).not.toContain('models.unsavedChanges'); + expect(button('models.saveExternalSettings').disabled).toBe(true); + }); }); diff --git a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx index 21005e1aeca..43008dc0aa6 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx +++ b/invokeai/frontend/webv2/src/features/models/ui/detail/ExternalModelSection.tsx @@ -1,5 +1,8 @@ /* eslint-disable react-perf/jsx-no-jsx-as-prop, react-perf/jsx-no-new-array-as-prop, react-perf/jsx-no-new-function-as-prop, react-perf/jsx-no-new-object-as-prop */ -import type { ExternalModelSettingsModel } from '@features/models/core/externalModelFields'; +import type { + ExternalModelSettingsModel, + ExternalModelValidationIssue, +} from '@features/models/core/externalModelFields'; import type { ExternalModelFormValues } from '@features/models/core/schemas'; import { @@ -9,9 +12,11 @@ import { HStack, Input, NumberInput, + Spinner, Stack, Switch, Text, + useFieldContext, } from '@chakra-ui/react'; import { isStarterManagedExternalModel, @@ -38,14 +43,17 @@ import { useModelDraft, type ModelDraft, } from '@features/models/ui/modelDraftsStore'; +import { focusFirstOperable, focusIfUnclaimed } from '@platform/react/focusIfUnclaimed'; import { useMountEffect } from '@platform/react/useMountEffect'; import { useScopedAction } from '@platform/react/useScopedAction'; import { assertAccountScopeCurrent } from '@platform/state/accountLifecycle'; +import { getApiErrorMessage } from '@platform/transport/http'; import { Button } from '@platform/ui/Button'; import { Field, FieldLabel } from '@platform/ui/Field'; import { Panel } from '@platform/ui/Panel'; import { Select } from '@platform/ui/Select'; -import { useMemo, useState } from 'react'; +import { useCallback, useId, useMemo, useRef, useState } from 'react'; +import { flushSync } from 'react-dom'; import { useTranslation } from 'react-i18next'; import { DraftConflictNotice, UnsavedChangesLabel } from './ModelDraftNotices'; @@ -101,20 +109,31 @@ const ReadOnlyValue = ({ label, value }: { label: string; value: string }) => ( ); +/** NumberInput takes only the Field's state; its description and error message are referenced here. */ +const FieldNumberInputControl = ({ label, placeholder }: { label: string; placeholder: string }) => { + const field = useFieldContext(); + const describedBy = + [field?.ariaDescribedby, field?.getInputProps()['aria-errormessage']].filter(Boolean).join(' ') || undefined; + + return ; +}; + const OptionalNumberField = ({ disabled, + error, label, placeholder, setValue, value, }: { disabled: boolean; + error: string | null; label: string; placeholder: string; setValue: (value: number | null) => void; value: number | null; }) => ( - + - + ); @@ -150,11 +169,22 @@ export const ExternalModelSection = ({ const saved = useMemo(() => toExternalModelValues(model), [model]); const draft = useModelDraft(model.key, selectExternalDraft); const values = useMemo(() => applyModelDraftFields(saved, draft), [draft, saved]); - const [error, setError] = useState(null); + const [error, setError] = useState(null); + const errorFor = (field: keyof ExternalModelFormValues) => (error?.field === field ? error.message : null); + // The modes are one group; the schema reports a missing mode against the first flag. + const modesError = errorFor('modeTxt2img'); + const modesHeadingId = useId(); + const modesMessageId = useId(); + const sectionRef = useRef(null); + const starterManagedNoteRef = useRef(null); const { run } = useScopedAction(); const providerConfigs = useExternalProvidersSelector((snapshot) => snapshot.configs); const starters = useStartersSelector((snapshot) => snapshot.response?.starter_models ?? null); const startersStatus = useStartersSelector((snapshot) => snapshot.status); + const [hasRequestedRetry, setHasRequestedRetry] = useState(false); + // The failure stays on screen through a retry, so the Retry button keeps the focus it holds. + const isRetryingCatalog = hasRequestedRetry && startersStatus === 'loading'; + const showsCatalogFailure = startersStatus === 'error' || isRetryingCatalog; // A failed lookup cannot establish that edits persist: the backend may still overlay catalog metadata. const isStarterManaged = starters !== null && isStarterManagedExternalModel(model, starters); const canEdit = starters !== null && !isStarterManaged; @@ -198,10 +228,12 @@ export const ExternalModelSection = ({ return; } - const validationError = validateExternalModel(values); + const validationIssue = validateExternalModel(values); - if (validationError) { - setError(validationError); + if (validationIssue) { + // Commit the marking first so focus can move to the field it names, scrolling it into view. + flushSync(() => setError(validationIssue)); + focusFirstOperable(sectionRef.current?.querySelector('[data-invalid]') ?? null); return; } @@ -222,15 +254,47 @@ export const ExternalModelSection = ({ }, (_message, saveError) => { failModelDraftSave(model.key, 'external', saved, null); - onError(saveError instanceof Error ? saveError.message : t('models.failedToSaveExternalSettings')); + onError(getApiErrorMessage(saveError, t('models.failedToSaveExternalSettings'))); } ); }; + const handleRetryCatalog = () => { + if (isRetryingCatalog) { + return; + } + + setHasRequestedRetry(true); + void refreshStarters().then(() => setHasRequestedRetry(false)); + }; + + // A successful retry removes the button that held focus: hand it to the editor the catalog unlocked, or to the + // note explaining why a catalog-managed record stays locked. A user who already moved focus elsewhere keeps it. + const retryButtonRef = useRef(null); + const attachRetryButton = useCallback((node: HTMLButtonElement | null) => { + const previous = retryButtonRef.current; + + retryButtonRef.current = node; + + if (node || !previous?.contains(document.activeElement)) { + return; + } + + // Detaching runs before the node leaves the document; hand focus on once the replacement has committed. + queueMicrotask(() => { + focusIfUnclaimed(sectionRef.current); + + if (document.activeElement === null || document.activeElement === document.body) { + starterManagedNoteRef.current?.focus(); + } + }); + }, []); + const numberField = ({ key, labelKey }: { key: OptionalNumberKey; labelKey: string }, placeholderKey: string) => ( setField(key, value)} @@ -239,7 +303,7 @@ export const ExternalModelSection = ({ ); return ( - + {t('models.externalProvider')} @@ -261,23 +325,31 @@ export const ExternalModelSection = ({ - {error ? ( - - {error} - - ) : null} {hasModelDraftConflict(saved, draft) ? : null} {starters === null ? ( - - + + {t( - startersStatus === 'error' - ? 'models.externalFields.catalogUnavailable' - : 'models.externalFields.catalogLoading' + showsCatalogFailure ? 'models.externalFields.catalogUnavailable' : 'models.externalFields.catalogLoading' )} - {startersStatus === 'error' ? ( - ) : null} @@ -289,7 +361,7 @@ export const ExternalModelSection = ({ ) : null} - + {providerCollection ? ( {starters === null ? null : ( - + {t('models.externalFields.starterManaged')} )} @@ -361,8 +437,16 @@ export const ExternalModelSection = ({ ) : ( <> - - + + {t('models.externalFields.supportedModes')} @@ -371,6 +455,7 @@ export const ExternalModelSection = ({ key={key} checked={values[key]} disabled={isSaving} + invalid={modesError !== null} size="sm" onCheckedChange={(event) => setField(key, event.checked === true)} > @@ -380,9 +465,15 @@ export const ExternalModelSection = ({ ))} - - {t('models.externalFields.supportedModesHelp')} - + {modesError ? ( + + {modesError} + + ) : ( + + {t('models.externalFields.supportedModesHelp')} + + )} {SUPPORT_FLAGS.map(({ key, labelKey }) => ( diff --git a/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts b/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts index 6997cd4c513..0078aacbe4b 100644 --- a/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts +++ b/invokeai/frontend/webv2/src/features/models/ui/modelDraftsStore.test.ts @@ -1,5 +1,6 @@ import type { ModelConfig } from '@features/models/core/types'; +import { toExternalModelValues } from '@features/models/core/externalModelFields'; import { setModelsSnapshotForTests } from '@features/models/data/modelsStore'; import { accountLifecycle } from '@platform/state/accountLifecycle'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -212,6 +213,28 @@ describe('model drafts store', () => { expect(getModelDraft('other')).toBeUndefined(); }); + it('drops hosted-provider edits a refreshed record comes to agree with', () => { + const hosted = { + capabilities: { modes: ['txt2img'] }, + key: 'hosted', + name: 'hosted', + provider_id: 'openai', + provider_model_id: 'gpt-image-1', + type: 'external_image_generator', + } as ModelConfig; + const saved = toExternalModelValues(hosted); + + setModelsSnapshotForTests({ models: [hosted], status: 'loaded' }); + recordModelDraftFields('hosted', 'external', { ...saved, maxReferenceImages: 4, modeImg2img: true }, saved); + + setModelsSnapshotForTests({ + models: [{ ...hosted, capabilities: { modes: ['txt2img', 'img2img'] } }], + status: 'loaded', + }); + + expect(getModelDraft('hosted')?.external?.fields).toEqual({ maxReferenceImages: 4 }); + }); + it('drops drafts of models that leave the loaded library and keeps the rest', () => { setModelsSnapshotForTests({ models: [model('kept'), model('gone')], status: 'loaded' }); recordModelDraftFields('kept', 'defaults', { steps: 20 }, { steps: 30 }); diff --git a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts index bc68e6b4283..7fec9e4b4e2 100644 --- a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts +++ b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.test.ts @@ -1223,6 +1223,7 @@ describe('image recall', () => { key: 'gpt-image', name: 'GPT Image', provider_id: 'openai', + provider_model_id: 'gpt-image-1', type: 'external_image_generator', } as GenerateWidgetValues['model']; const geminiModel = { diff --git a/invokeai/frontend/webv2/src/workbench/invocation.test.ts b/invokeai/frontend/webv2/src/workbench/invocation.test.ts index 78953c44db3..7cc5bcbbb69 100644 --- a/invokeai/frontend/webv2/src/workbench/invocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/invocation.test.ts @@ -270,6 +270,7 @@ describe('resolveInvocationRoute — canvas source', () => { mountedWidgetIds?: WidgetId[]; bbox?: { width: number; height: number }; layers?: CanvasLayerContract[]; + rasterInFrame?: boolean; } = {} ) => { const project = getActiveProject(overrides.generateValues ?? validGenerateValues); @@ -278,6 +279,7 @@ describe('resolveInvocationRoute — canvas source', () => { ...getInvocationRouteInput(project), canvasBbox: overrides.bbox ?? { height: 512, width: 512 }, canvasLayers: overrides.layers ?? [], + canvasRasterInFrame: overrides.rasterInFrame ?? false, mountedWidgetIds: overrides.mountedWidgetIds ?? (['canvas', 'generate'] as WidgetId[]), }; }; @@ -351,6 +353,88 @@ describe('resolveInvocationRoute — canvas source', () => { ).toContain('Unknown External Provider supports a maximum size of 1024×1024.'); }); + it('validates a fixed-size provider against the size it is asked for, not the frame', () => { + const model: GenerateModelConfig = { + ...unknownExternalModel, + provider_id: 'openai', + capabilities: { + aspect_ratio_sizes: { '1:1': { height: 1024, width: 1024 } }, + max_image_size: { height: 1024, width: 1024 }, + modes: ['txt2img'], + }, + }; + const input = canvasInputFor({ bbox: { height: 3000, width: 3000 }, generateValues: createGenerateValues(model) }); + + expect(resolveInvocationRouteInput(input, 'global', canvasRoute).validationReasons).toEqual([]); + }); + + it('blocks a text-only provider while raster content overlaps the frame, naming the content', () => { + const textOnly: GenerateModelConfig = { ...unknownExternalModel, provider_id: 'openai' }; + const editing: GenerateModelConfig = { ...textOnly, capabilities: { modes: ['txt2img', 'img2img'] } }; + + expect( + resolveInvocationRouteInput( + canvasInputFor({ generateValues: createGenerateValues(textOnly), rasterInFrame: true }), + 'global', + canvasRoute + ).validationReasons + ).toEqual([ + 'Unknown External Provider cannot use image content in the generation frame. Hide or move raster content out of the frame to generate.', + ]); + expect( + resolveInvocationRouteInput( + canvasInputFor({ generateValues: createGenerateValues(textOnly) }), + 'global', + canvasRoute + ).sourceValid + ).toBe(true); + expect( + resolveInvocationRouteInput( + canvasInputFor({ generateValues: createGenerateValues(editing), rasterInFrame: true }), + 'global', + canvasRoute + ).sourceValid + ).toBe(true); + }); + + it('derives raster overlap from enabled raster content bounds against the frame, for a text-only provider', () => { + const textOnly: GenerateModelConfig = { ...unknownExternalModel, provider_id: 'openai' }; + let project = getActiveProject(createGenerateValues(textOnly)); + const raster = (x: number, isEnabled = true): CanvasLayerContract => ({ + blendMode: 'normal', + id: `raster-${x}`, + isEnabled, + isLocked: false, + name: `Raster ${x}`, + opacity: 1, + source: { image: { height: 64, imageName: 'raster.png', width: 64 }, type: 'image' }, + transform: { rotation: 0, scaleX: 1, scaleY: 1, x, y: 0 }, + type: 'raster', + }); + const withRaster = (layers: CanvasLayerContract[]) => + getInvocationRouteInput({ + ...project, + canvas: { + ...project.canvas, + document: { + ...project.canvas.document, + bbox: { height: 100, width: 100, x: 0, y: 0 }, + stacks: { ...project.canvas.document.stacks, raster: layers }, + }, + }, + }).canvasRasterInFrame; + + expect(withRaster([])).toBe(false); + expect(withRaster([raster(50)])).toBe(true); + // Touching the frame's edge is not overlap, and a hidden layer contributes nothing. + expect(withRaster([raster(100)])).toBe(false); + expect(withRaster([raster(50, false)])).toBe(false); + + // Only a text-only provider is gated by it, so no other model pays for measuring it. + project = getActiveProject(validGenerateValues); + expect(withRaster([raster(50)])).toBe(false); + }); + it('invalidates a zero-area generation frame', () => { const route = resolveInvocationRouteInput( canvasInputFor({ bbox: { height: 0, width: 512 } }), diff --git a/invokeai/frontend/webv2/src/workbench/invocation.ts b/invokeai/frontend/webv2/src/workbench/invocation.ts index caff5b2dc2a..162f8c5b3e0 100644 --- a/invokeai/frontend/webv2/src/workbench/invocation.ts +++ b/invokeai/frontend/webv2/src/workbench/invocation.ts @@ -13,10 +13,14 @@ import type { Project } from '@workbench/projectContracts'; import type { WidgetId } from '@workbench/widgetContracts'; import { + describeExternalCanvasRefusal, getGenerationModelAvailabilityReasons, getGenerationValidationReasons, + isExternalTextOnlyCanvasModel, + isGenerateModelConfig, isSupportedGenerateModel, normalizeGenerateWidgetValues, + resolveExternalCanvasRequestSize, sanitizeBatchCount, } from '@features/generation/settings'; import { getUpscaleValidationReasons, normalizeUpscaleWidgetValues } from '@features/upscale'; @@ -25,7 +29,7 @@ import { getProjectGraphReadiness } from '@features/workflow/graph'; import { getInvocationTemplatesSnapshot } from '@features/workflow/react'; import { localizeForLoopValidationReason } from '@features/workflow/utility'; import { areArraysEqual, createStableSelector } from '@platform/state/selectors'; -import { compileContributingLayers } from '@workbench/canvas-engine/api'; +import { compileContributingLayers, getBaseRasterContentBounds } from '@workbench/canvas-engine/api'; import { isCompositableControlLayer, isCompositableRegionalGuidanceLayer } from '@workbench/canvasLayerContent'; import { describeControlLayerIssue, getBlockingControlLayerIssues } from './controlLayerChecks'; @@ -100,6 +104,12 @@ export interface InvocationRouteInput { canvasBbox: { width: number; height: number }; /** The canvas layer stack (identity-stable) — control layers gate a canvas invoke. */ canvasLayers: readonly CanvasLayerContract[]; + /** + * Whether enabled raster content's bounds overlap the frame: the geometry Canvas mode detection starts from, so a + * text-only hosted model is blocked before invoke without reading pixels. Measured only for such a model; false + * otherwise. + */ + canvasRasterInFrame: boolean; } const getMountedWidgetIds = (project: Project): WidgetId[] => { @@ -127,21 +137,45 @@ const getMountedWidgetIds = (project: Project): WidgetId[] => { return Array.from(mountedWidgetIds).sort(); }; -export const getInvocationRouteInput = (project: Project): InvocationRouteInput => ({ - canvasBbox: { - height: project.canvas.document.bbox.height, - width: project.canvas.document.bbox.width, - }, - canvasLayers: compileContributingLayers(project.canvas.document), - generateValues: getProjectWidgetValues(project, 'generate'), - upscaleValues: getProjectWidgetValues(project, 'upscale'), - videoValues: getProjectWidgetValues(project, 'video'), - workflowValues: getProjectWidgetValues(project, 'workflow'), - invocation: project.invocation, - mountedWidgetIds: getMountedWidgetIds(project), - projectGraph: getActiveProjectGraph(project), - projectId: project.id, -}); +/** Interior overlap, as the composite planner requires: touching edges and empty boxes do not count. */ +const hasRasterContentInFrame = (document: Project['canvas']['document']): boolean => { + const content = getBaseRasterContentBounds(document); + const { bbox } = document; + + return ( + content !== null && + bbox.width > 0 && + bbox.height > 0 && + content.x < bbox.x + bbox.width && + bbox.x < content.x + content.width && + content.y < bbox.y + bbox.height && + bbox.y < content.y + content.height + ); +}; + +export const getInvocationRouteInput = (project: Project): InvocationRouteInput => { + const generateValues = getProjectWidgetValues(project, 'generate'); + // Planning the raster composite is not free and runs on every project change, so only the model it gates pays. + const isTextOnlyModel = + isGenerateModelConfig(generateValues.model) && isExternalTextOnlyCanvasModel(generateValues.model); + + return { + canvasBbox: { + height: project.canvas.document.bbox.height, + width: project.canvas.document.bbox.width, + }, + canvasLayers: compileContributingLayers(project.canvas.document), + canvasRasterInFrame: isTextOnlyModel && hasRasterContentInFrame(project.canvas.document), + generateValues, + upscaleValues: getProjectWidgetValues(project, 'upscale'), + videoValues: getProjectWidgetValues(project, 'video'), + workflowValues: getProjectWidgetValues(project, 'workflow'), + invocation: project.invocation, + mountedWidgetIds: getMountedWidgetIds(project), + projectGraph: getActiveProjectGraph(project), + projectId: project.id, + }; +}; export const areInvocationRouteInputsEqual = (left: InvocationRouteInput, right: InvocationRouteInput): boolean => left.projectId === right.projectId && @@ -154,6 +188,7 @@ export const areInvocationRouteInputsEqual = (left: InvocationRouteInput, right: left.canvasBbox.width === right.canvasBbox.width && left.canvasBbox.height === right.canvasBbox.height && left.canvasLayers === right.canvasLayers && + left.canvasRasterInFrame === right.canvasRasterInFrame && areArraysEqual(left.mountedWidgetIds, right.mountedWidgetIds); export const createInvocationRouteInputSelector = () => @@ -173,9 +208,9 @@ const getGenerateSnapshotValidationReasons = ( return ['Generate needs a supported model before it can be invoked.']; } - // Canvas discovers the mode from pixel coverage during preparation and asks a provider for the bbox itself, so - // readiness validates the bbox and permits an advertised whole-image mode; the compiler validates the captured - // one. Generate itself remains strictly text-to-image. + // Canvas discovers the mode from pixel coverage during preparation and asks a provider for its size nearest the + // bbox, so readiness validates that size and permits an advertised whole-image mode; the compiler validates the + // captured one. Generate itself remains strictly text-to-image. const providerModes = canvasBbox !== undefined && values.model.type === 'external_image_generator' ? (values.model.capabilities?.modes ?? ['txt2img']) @@ -185,7 +220,9 @@ const getGenerateSnapshotValidationReasons = ( return [ ...getGenerationValidationReasons( values.model, - providerModes !== null ? { ...values, ...canvasBbox } : values, + providerModes !== null && canvasBbox + ? { ...values, ...resolveExternalCanvasRequestSize(values.model, canvasBbox) } + : values, mode ), ...(models ? getGenerationModelAvailabilityReasons(values.model, values, models) : []), @@ -266,14 +303,19 @@ export const resolveInvocationRouteInput = ( const values = normalizeGenerateWidgetValues(input.generateValues); if (values?.model.type === 'external_image_generator') { + const { model } = values; + + if (input.canvasRasterInFrame && isExternalTextOnlyCanvasModel(model)) { + validationReasons.push(describeExternalCanvasRefusal(model, { kind: 'raster_content' })); + } for (const layer of input.canvasLayers) { if (isCompositableControlLayer(layer)) { validationReasons.push( - `${values.model.name} does not support control layer "${layer.name}". Disable it to generate.` + describeExternalCanvasRefusal(model, { kind: 'control_layer', layerName: layer.name }) ); } else if (isCompositableRegionalGuidanceLayer(layer)) { validationReasons.push( - `${values.model.name} does not support regional guidance layer "${layer.name}". Disable it to generate.` + describeExternalCanvasRefusal(model, { kind: 'regional_guidance', layerName: layer.name }) ); } } diff --git a/invokeai/frontend/webv2/src/workbench/palette/WorkbenchCommandPaletteDialog.tsx b/invokeai/frontend/webv2/src/workbench/palette/WorkbenchCommandPaletteDialog.tsx index f59956c49d2..2a04d24dfe0 100644 --- a/invokeai/frontend/webv2/src/workbench/palette/WorkbenchCommandPaletteDialog.tsx +++ b/invokeai/frontend/webv2/src/workbench/palette/WorkbenchCommandPaletteDialog.tsx @@ -6,7 +6,11 @@ import type { getWidgetsForRegion as GetWidgetsForRegion } from '@workbench/widg import type { TFunction } from 'i18next'; import { imageIndexAvailabilityOptions } from '@features/gallery/queries'; -import { flushGenerateDrafts, notifyGenerateModelSelectionCleared } from '@features/generation/react'; +import { + flushGenerateDrafts, + notifyGenerateDimensionsAdjusted, + notifyGenerateModelSelectionCleared, +} from '@features/generation/react'; import { getModelsSnapshot } from '@features/models'; import { getQueueQueryScope, getQueueReadModelOptions } from '@features/queue/queries'; import { queryClient } from '@platform/query/client'; @@ -218,6 +222,7 @@ const WorkbenchCommandPaletteDialog = ({ notifications: notify, t, }); + notifyGenerateDimensionsAdjusted({ modelName: model.name, notifications: notify, result, t }); }, openGenerateWidget: () => openWidget('generate'), openModelManager: () => void executeCommand('app.selectModelsTab'), diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/GenerateCanvasRenderSize.tsx b/invokeai/frontend/webv2/src/workbench/widgets/canvas/GenerateCanvasRenderSize.tsx index 472f761fe54..38b92e174a7 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/GenerateCanvasRenderSize.tsx +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/GenerateCanvasRenderSize.tsx @@ -12,7 +12,7 @@ import type { Project } from '@workbench/projectContracts'; import { Box, chakra, createListCollection, Grid, HStack, NumberInput, Text } from '@chakra-ui/react'; import { resolveCanvasProcessingSize } from '@features/generation/canvasProcessingSize'; import { getArchitectureCapabilitiesSnapshot, subscribeArchitectureCapabilities } from '@features/generation/runtime'; -import { clampDimension, getGenerationDimensions } from '@features/generation/settings'; +import { clampDimension, getGenerationDimensions, isGenerateModelConfig } from '@features/generation/settings'; import { useExternalStoreSelector } from '@platform/state/selectors'; import { FieldLabel, Select, Tooltip } from '@platform/ui'; import { @@ -40,30 +40,19 @@ const scalingEqual = (a: CanvasScalingSettings, b: CanvasScalingSettings): boole const readSize = (value: unknown): number | null => typeof value === 'number' && Number.isFinite(value) ? value : null; -/** - * Use model base/type/variant and PiD mode matching compileCanvasGraph so displayed size follows identical - * grid/area policy. - */ +/** Use the stored model and PiD mode as compileCanvasGraph does, so the displayed size follows the same policy. */ const selectProcessingContext = (project: Project) => { const generate = getProjectWidgetValues(project, 'generate') as { height?: unknown; - model?: { base?: string; type?: string; variant?: unknown } | null; + model?: unknown; pidMode?: PidMode; width?: unknown; }; - const model = generate.model; return { bbox: project.canvas.document.bbox, committedFrame: { height: readSize(generate.height), width: readSize(generate.width) }, lockedRatio: readLockedAspectRatio(generate), - model: - model && typeof model.base === 'string' - ? { - base: model.base, - type: model.type ?? 'main', - variant: typeof model.variant === 'string' ? model.variant : null, - } - : null, + model: isGenerateModelConfig(generate.model) ? generate.model : null, pidMode: generate.pidMode ?? 'off', }; }; @@ -76,9 +65,7 @@ const processingContextEqual = (a: ProcessingContext, b: ProcessingContext): boo a.committedFrame.width === b.committedFrame.width && a.committedFrame.height === b.committedFrame.height && a.lockedRatio === b.lockedRatio && - a.model?.base === b.model?.base && - a.model?.type === b.model?.type && - a.model?.variant === b.model?.variant && + a.model === b.model && a.pidMode === b.pidMode; export const GenerateCanvasRenderSize = ({ children, frame }: CanvasRenderSizeProps) => { @@ -99,7 +86,7 @@ export const GenerateCanvasRenderSize = ({ children, frame }: CanvasRenderSizePr subscribeArchitectureCapabilities, getArchitectureCapabilitiesSnapshot, useCallback(() => { - const model = context.model as Parameters[0] | null; + const { model } = context; return { dimensions: getGenerationDimensions(model ?? undefined, context.pidMode), processingSize: model diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts index a51ac1c7a03..36dd0bdac58 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.test.ts @@ -792,8 +792,8 @@ describe('runCanvasInvocation', () => { ); it('records a notice and dispatches no snapshot when the graph fails validation', async () => { - // The compiler rejects a hosted provider asked for a canvas mode its record does not advertise: the raster - // snapshot makes this an image-to-image run, and the record lists text-to-image only. + // The raster snapshot makes this an image-to-image run, and the record lists text-to-image only, so the notice + // names the frame's content as the cause. const harness = makeHarness({ model: externalModel }); await runCanvasInvocation(harness.deps); @@ -801,7 +801,9 @@ describe('runCanvasInvocation', () => { expect(harness.submittedGraphs()).toHaveLength(0); const notices = harness.notices(); expect(notices).toHaveLength(1); - expect(notices[0]?.message).toBe('OpenAI Image does not support image-to-image generation.'); + expect(notices[0]?.message).toBe( + 'OpenAI Image cannot use image content in the generation frame. Hide or move raster content out of the frame to generate.' + ); expect(harness.dedupe.byKey).toHaveLength(0); expect(harness.dedupe.byHash).toHaveLength(0); expect(harness.releaseRasterSnapshot).toHaveBeenCalledTimes(1); diff --git a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts index 099c8de3df5..d432a0f1704 100644 --- a/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts +++ b/invokeai/frontend/webv2/src/workbench/widgets/canvas/invoke/prepareCanvasInvocation.ts @@ -34,7 +34,11 @@ import { type RegionalReferenceImageInput, resolveGenerateSeed, } from '@features/generation/graph'; -import { normalizeGenerateWidgetValues, syncGenerateWidgetValuesWithModels } from '@features/generation/settings'; +import { + describeExternalCanvasRefusal, + normalizeGenerateWidgetValues, + syncGenerateWidgetValuesWithModels, +} from '@features/generation/settings'; import { getGenerationDevicesSnapshot, resolveRandDeviceMetadata } from '@features/queue/devices'; import { captureAccountScope, @@ -140,7 +144,7 @@ const createControlLayerCollector = ( return { shouldComposite: (layer) => { if (model.type === 'external_image_generator') { - throw new Error(`${model.name} does not support control layer "${layer.name}". Disable it to generate.`); + throw new Error(describeExternalCanvasRefusal(model, { kind: 'control_layer', layerName: layer.name })); } const { adapter } = layer; const resolved = adapter.model ? models?.find((candidate) => candidate.key === adapter.model) : undefined; @@ -260,9 +264,7 @@ const createRegionalGuidanceCollector = ( return { shouldComposite: (layer) => { if (model.type === 'external_image_generator') { - throw new Error( - `${model.name} does not support regional guidance layer "${layer.name}". Disable it to generate.` - ); + throw new Error(describeExternalCanvasRefusal(model, { kind: 'regional_guidance', layerName: layer.name })); } const referenceImages = resolveRegionalReferenceImages(layer, model.base); const rejection = getRegionalGuidanceRejectionReason({ diff --git a/tests/app/invocations/test_frontend_external_canvas_graphs.py b/tests/app/invocations/test_frontend_external_canvas_graphs.py index ec6431ae241..1e513fd2b52 100644 --- a/tests/app/invocations/test_frontend_external_canvas_graphs.py +++ b/tests/app/invocations/test_frontend_external_canvas_graphs.py @@ -1,16 +1,38 @@ -"""Validate the frontend's hosted Canvas graphs against the real invocation fields and edge types. +"""Validate the frontend's hosted Canvas graphs against the real invocation fields and edge types, and run them. -The fixture is generated by compileCanvasGraph.test.ts. No provider request or model load is needed: -Graph.validate_self checks the collection-to-item conversion and the image/metadata resize inputs. +The fixture is generated by compileCanvasGraph.test.ts. Graph.validate_self checks the collection-to-item conversion +and the image/metadata inputs of the output nodes. The execution test runs a graph through GraphExecutionState with +real image storage and a fake provider in place of the network call. """ import json +import logging from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest +from PIL import Image -from invokeai.app.services.shared.graph import Graph +from invokeai.app.invocations.primitives import ImageOutput +from invokeai.app.services.config.config_default import InvokeAIAppConfig +from invokeai.app.services.external_generation.external_generation_base import ExternalProvider +from invokeai.app.services.external_generation.external_generation_common import ( + ExternalGeneratedImage, + ExternalGenerationRequest, + ExternalGenerationResult, +) +from invokeai.app.services.external_generation.external_generation_default import ExternalGenerationService +from invokeai.app.services.image_files.image_files_disk import DiskImageFileStorage +from invokeai.app.services.invoker import Invoker +from invokeai.app.services.model_records.model_records_sql import ModelRecordServiceSQL +from invokeai.app.services.names.names_default import SimpleNameService +from invokeai.app.services.session_queue.session_queue_common import SessionQueueItem +from invokeai.app.services.shared.database.database import Database +from invokeai.app.services.shared.graph import EdgeConnection, Graph, GraphExecutionState +from invokeai.app.services.shared.invocation_context import InvocationContextData, build_invocation_context +from invokeai.app.services.urls.urls_default import LocalUrlService +from invokeai.backend.model_manager.configs.external_api import ExternalApiModelConfig, ExternalModelCapabilities FIXTURE_PATH = ( Path(__file__).parents[3] @@ -21,6 +43,15 @@ } +# The Canvas bbox is 640x512; GPT Image 1's size table answers its 5:4 frame with the 3:2 size 1536x1024. +BBOX = (640, 512) +PROVIDER_SIZE = (1536, 1024) + + +def _sources(graph: Graph) -> dict[tuple[str, str], EdgeConnection]: + return {(edge.destination.node_id, edge.destination.field): edge.source for edge in graph.edges} + + @pytest.mark.parametrize("mode", ["txt2img", "img2img"]) def test_hosted_canvas_graph_is_valid_for_the_backend(mode: str) -> None: graph = Graph.model_validate(GRAPHS[mode]) @@ -36,23 +67,167 @@ def test_hosted_canvas_graph_is_valid_for_the_backend(mode: str) -> None: provider = graph.nodes["canvas_processing_output"] assert provider.mode == mode - assert (provider.init_image is not None) == (mode == "img2img") assert provider.num_images == 1 assert provider.is_intermediate + # The provider is asked for a size from its own table, so the backend has nothing to re-bucket. + assert (provider.width, provider.height) == PROVIDER_SIZE + + +def test_text_to_image_covers_then_center_crops_the_provider_image_to_the_bbox() -> None: + graph = Graph.model_validate(GRAPHS["txt2img"]) + sources = _sources(graph) + assert graph.nodes["canvas_processing_output"].init_image is None + assert ("canvas_processing_output", "init_image") not in sources + + output = graph.nodes["canvas_output"] + assert output.get_type() == "img_crop" + assert (output.width, output.height) == BBOX + cover = graph.nodes[sources[("canvas_output", "image")].node_id] + assert cover.get_type() == "img_resize" + # 1536x1024 scaled by max(640/1536, 512/1024) = 0.5 covers the bbox; the 128 px overflow is split evenly. + assert (cover.width, cover.height) == (768, 512) + assert (output.x, output.y) == (64, 0) + assert sources[(cover.id, "image")] == EdgeConnection(node_id="canvas_provider_image", field="item") + + +def test_image_to_image_sends_the_composite_at_the_request_size_and_resizes_the_result_to_the_bbox() -> None: + graph = Graph.model_validate(GRAPHS["img2img"]) + sources = _sources(graph) + assert graph.nodes["canvas_processing_output"].init_image is None + request_image = graph.nodes[sources[("canvas_processing_output", "init_image")].node_id] + assert request_image.get_type() == "img_resize" + assert (request_image.width, request_image.height) == PROVIDER_SIZE + assert request_image.image is not None and request_image.image.image_name == "canvas-composite.png" + output = graph.nodes["canvas_output"] assert output.get_type() == "img_resize" - assert (output.width, output.height) == (640, 512) + assert (output.width, output.height) == BBOX + assert sources[("canvas_output", "image")] == EdgeConnection(node_id="canvas_provider_image", field="item") @pytest.mark.parametrize("mode", ["txt2img", "img2img"]) -def test_resized_image_retains_the_provider_output_metadata(mode: str) -> None: +def test_canvas_output_retains_the_provider_output_metadata(mode: str) -> None: graph = Graph.model_validate(GRAPHS[mode]) - sources = {(edge.destination.node_id, edge.destination.field): edge.source for edge in graph.edges} - output_metadata_source = sources[("canvas_output", "metadata")] - metadata_reader = graph.nodes[output_metadata_source.node_id] + sources = _sources(graph) + metadata_reader = graph.nodes[sources[("canvas_output", "metadata")].node_id] assert metadata_reader.get_type() == "metadata_from_image" - assert sources[(metadata_reader.id, "image")] == sources[("canvas_output", "image")] + # The reader and the output's image chain both start at the same iterate item, which pairs them per image. + image_source = sources[("canvas_output", "image")] + while image_source.field != "item": + image_source = sources[(image_source.node_id, "image")] + assert sources[(metadata_reader.id, "image")] == image_source + assert graph.nodes[image_source.node_id].get_type() == "iterate" # Core dimensions enter the provider, which adds its authoritative request identity, seed and options. provider_metadata = graph.nodes[sources[("canvas_processing_output", "metadata")].node_id] assert provider_metadata.get_type() == "core_metadata" - assert (provider_metadata.width, provider_metadata.height) == (640, 512) + assert (provider_metadata.width, provider_metadata.height) == BBOX + + +PROVIDER_COLORS = [(200, 30, 30), (30, 200, 30)] + + +class _FakeOpenAIProvider(ExternalProvider): + """Stands in for the network call: one solid-colored image per requested image, each with its own seed.""" + + provider_id = "openai" + + def __init__(self, app_config: InvokeAIAppConfig, logger: logging.Logger) -> None: + super().__init__(app_config, logger) + self.requests: list[ExternalGenerationRequest] = [] + + def is_configured(self) -> bool: + return True + + def generate(self, request: ExternalGenerationRequest) -> ExternalGenerationResult: + self.requests.append(request) + return ExternalGenerationResult( + images=[ + ExternalGeneratedImage(image=Image.new("RGB", PROVIDER_SIZE, PROVIDER_COLORS[index]), seed=1000 + index) + for index in range(request.num_images) + ], + provider_request_id="req-canvas", + provider_metadata={"revised_prompt": "canvas"}, + ) + + +@pytest.mark.parametrize("num_images", [1, 2]) +def test_each_provider_image_reaches_the_canvas_at_bbox_size_with_its_own_metadata( + num_images: int, mock_invoker: Invoker, mock_sqlite_database: Database, tmp_path: Path +) -> None: + services = mock_invoker.services + services.image_files = DiskImageFileStorage(tmp_path / "images") + services.names = SimpleNameService() + services.urls = LocalUrlService() + provider = _FakeOpenAIProvider(services.configuration, logging.getLogger()) + services.external_generation = ExternalGenerationService({"openai": provider}, logging.getLogger()) + services.image_files.start(mock_invoker) + services.images.start(mock_invoker) + + raw_graph = GRAPHS["txt2img"] + raw_model = raw_graph["nodes"]["canvas_processing_output"]["model"] + store = ModelRecordServiceSQL(mock_sqlite_database, logging.getLogger()) + store.add_model( + ExternalApiModelConfig( + key=raw_model["key"], + name=raw_model["name"], + provider_id=raw_model["provider_id"], + provider_model_id=raw_model["provider_model_id"], + capabilities=ExternalModelCapabilities.model_validate(raw_model["capabilities"]), + ) + ) + services.model_manager = SimpleNamespace(store=store) # type: ignore[assignment] + + raw_nodes = {**raw_graph["nodes"]} + raw_nodes["canvas_processing_output"] = {**raw_nodes["canvas_processing_output"], "num_images": num_images} + session = GraphExecutionState(graph=Graph.model_validate({**raw_graph, "nodes": raw_nodes})) + queue_item = SessionQueueItem( + item_id=1, + batch_id="batch-1", + session_id=session.id, + created_at="2026-01-01T00:00:00", + updated_at="2026-01-01T00:00:00", + started_at=None, + completed_at=None, + queue_id="default", + session=session, + ) + + executed_sources: list[str] = [] + while (invocation := session.next()) is not None: + source_id = session.prepared_source_mapping[invocation.id] + context = build_invocation_context( + services=services, + data=InvocationContextData(queue_item=queue_item, invocation=invocation, source_invocation_id=source_id), + is_canceled=lambda: False, + ) + session.complete(invocation.id, invocation.invoke(context)) + executed_sources.append(source_id) + + assert not session.has_error() + assert [(request.width, request.height) for request in provider.requests] == [PROVIDER_SIZE] + # One provider request, then the iterate expands every node downstream of it once per returned image. + assert executed_sources.count("canvas_processing_output") == 1 + assert executed_sources.count("canvas_provider_image") == num_images + assert executed_sources.count("canvas_provider_metadata") == num_images + + outputs = [ + session.results[node_id] + for node_id, source_id in session.prepared_source_mapping.items() + if source_id == "canvas_output" + ] + assert len(outputs) == num_images + seeds_by_color = {color: 1000 + index for index, color in enumerate(PROVIDER_COLORS[:num_images])} + output_seeds = [] + for output in outputs: + assert isinstance(output, ImageOutput) + assert (output.width, output.height) == BBOX + image = services.images.get_pil_image(output.image.image_name).convert("RGB") + assert image.size == BBOX + metadata = services.images.get_metadata(output.image.image_name) + assert metadata is not None + # The metadata travels with the image it was read from: its seed names the provider image of these pixels. + assert metadata.root["external_seed"] == seeds_by_color[image.getpixel((320, 256))] + assert metadata.root["external_request_id"] == "req-canvas" + assert metadata.root["external_provider_metadata"] == {"revised_prompt": "canvas"} + output_seeds.append(metadata.root["external_seed"]) + assert sorted(output_seeds) == sorted(seeds_by_color.values()) diff --git a/tests/app/services/external_generation/test_external_generation_service.py b/tests/app/services/external_generation/test_external_generation_service.py index 2b6ef3b6ca5..07c0a79b203 100644 --- a/tests/app/services/external_generation/test_external_generation_service.py +++ b/tests/app/services/external_generation/test_external_generation_service.py @@ -1,4 +1,5 @@ import logging +from fractions import Fraction import pytest from PIL import Image @@ -291,28 +292,31 @@ def test_generate_snaps_unlisted_ratio_when_model_has_no_bucket_sizes() -> None: assert (provider.last_request.width, provider.last_request.height) == (1360, 765) -def _starter_sizes() -> list[tuple[str, str, int, int]]: +def _starter_sizes() -> list[tuple[str, str, int, int, bool]]: """Every fixed size a txt2img external starter model offers: its resolution presets and ratio buckets.""" - cases: list[tuple[str, str, int, int]] = [] + cases: list[tuple[str, str, int, int, bool]] = [] for starter in STARTER_MODELS: capabilities = starter.capabilities if capabilities is None or "txt2img" not in capabilities.modes: continue cases.extend( - (starter.source, preset.label, preset.width, preset.height) + (starter.source, preset.aspect_ratio, preset.width, preset.height, False) for preset in capabilities.resolution_presets or [] ) cases.extend( - (starter.source, ratio, size.width, size.height) + (starter.source, ratio, size.width, size.height, True) for ratio, size in (capabilities.aspect_ratio_sizes or {}).items() ) return cases -@pytest.mark.parametrize("source, label, width, height", _starter_sizes()) -def test_starter_model_offered_sizes_are_accepted(source: str, label: str, width: int, height: int) -> None: +@pytest.mark.parametrize("source, ratio, width, height, is_bucket_size", _starter_sizes()) +def test_starter_model_offered_sizes_are_accepted( + source: str, ratio: str, width: int, height: int, is_bucket_size: bool +) -> None: """A size a model offers in its own presets must be accepted at (nearly) that size, even when its declared ratio is not in lowest terms ("21:9") and the preset is only approximately that ratio (1024x439).""" + # The frontend's Canvas picks its provider request size from these capabilities and relies on this acceptance. starter = next(model for model in STARTER_MODELS if model.source == source) assert starter.capabilities is not None provider_id, provider_model_id = source.removeprefix("external://").split("/", 1) @@ -328,10 +332,18 @@ def test_starter_model_offered_sizes_are_accepted(source: str, label: str, width service.generate(_build_request(model=model, width=width, height=height)) - assert provider.last_request is not None - # Snapping to an exact 21:9 moves in 21x9 px steps, so a small preset (512x219 -> 504x216) can shift ~2%. - assert provider.last_request.width == pytest.approx(width, rel=0.02) - assert provider.last_request.height == pytest.approx(height, rel=0.02) + sent = provider.last_request + assert sent is not None + if is_bucket_size: + # Bucket sizes pass unchanged: the Canvas frames its output crop around this exact size. + assert (sent.width, sent.height) == (width, height) + else: + # An approximate preset may snap onto its exact declared ratio (1024x819 -> 1025x820), never onto another + # ratio. Exact 21:9 moves in 21x9 px steps, so a small preset (512x219 -> 504x216) can shift ~2%. + ratio_width, ratio_height = (int(part) for part in ratio.split(":")) + assert Fraction(sent.width, sent.height) == Fraction(ratio_width, ratio_height) + assert sent.width == pytest.approx(width, rel=0.02) + assert sent.height == pytest.approx(height, rel=0.02) def test_generate_matches_declared_ratio_not_in_lowest_terms() -> None: From 99405a30d309960b46bd107d2ce4b624e344770a Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Mon, 19 Oct 2026 14:56:43 -0600 Subject: [PATCH 08/10] test(models): route the mock GPT Image fixture to a GPT Image model The mock again offers OpenAI's quality, background and input-fidelity options, which are limited to GPT Image models. --- invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs | 4 +++- .../frontend/webv2/scripts/mock-backend-fixtures.test.mjs | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs index 65b9fae6b54..f057ba26d2c 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs @@ -476,7 +476,9 @@ const EXTERNAL_MODEL_FIXTURES = [ name: 'Fixture GPT Image', path: 'external://openai/fixture-gpt-image', provider_id: 'openai', - provider_model_id: 'fixture-gpt-image', + // A GPT Image model id, so Generate offers OpenAI's quality, background, and input-fidelity options; the + // fixture source keeps the record out of the starter catalog and editable in the Model Manager. + provider_model_id: 'gpt-image-1', source: 'external://openai/fixture-gpt-image', source_type: 'external', source_url: null, diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs index 139318c817a..86f39de2806 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs @@ -1429,6 +1429,8 @@ test('hosted-provider credentials and the starter catalog agree with the hosted managed.map((model) => model.key), ['fixture-external-gemini-catalog'] ); + // The editable OpenAI record routes to a GPT Image model, the only OpenAI models that take provider options. + assert.equal(hosted.find((model) => model.key === 'fixture-external-openai').provider_model_id, 'gpt-image-1'); assert.equal(starters.find((starter) => starter.source === managed[0].source).is_installed, true); assert.ok(starters.filter((starter) => starter.type !== 'external_image_generator').every((s) => !s.is_installed)); From cbcd5ee93b3d84acc77f3333db670f7f90004ef3 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Mon, 19 Oct 2026 16:07:40 -0600 Subject: [PATCH 09/10] fix(generation): judge hosted Canvas frames by visible content Hosted models run text-to-image when nothing visible is in the frame; readiness checks each visible layer instead of the union of bounds. Model switches and recall keep the Canvas frame, the size hint names the requested size, and recall restores a Canvas output's named preset on Generate. A repeated catalog failure is announced again; palette notices and mock provider config branches gain tests. --- .../frontend/webv2/public/locales/en.json | 3 + .../scripts/mock-backend-fixtures.test.mjs | 22 ++++++ .../core/baseGenerationPolicies.test.ts | 21 ++++++ .../generation/core/baseGenerationPolicies.ts | 11 ++- .../generation/core/canvas/canvasMode.test.ts | 1 + .../generation/core/canvas/canvasMode.ts | 5 ++ .../generation/ui/GenerateDimensionFields.tsx | 1 + .../generation/ui/GenerateModelCard.tsx | 20 ++++- .../ExternalSizeSelect.browser.test.tsx | 32 +++++++- .../ui/shared/ExternalSizeSelect.tsx | 24 +++++- .../ExternalModelSection.browser.test.tsx | 38 ++++++++++ .../models/ui/detail/ExternalModelSection.tsx | 32 ++++---- .../workbench/canvas-engine/capabilities.ts | 1 + .../canvas-engine/render/rasterComposite.ts | 10 ++- .../compositeForGeneration.test.ts | 17 +++++ .../compositeForGeneration.ts | 50 +++++++++---- .../generationComposite.test.ts | 1 + .../canvas-operations/generationComposite.ts | 6 +- .../generationSettingsOrchestration.test.ts | 2 + .../generationSettingsOrchestration.ts | 4 +- .../workbench/hotkeys/firstPartyCommands.ts | 1 + .../image-actions/executeImageRecall.ts | 4 + .../image-actions/imageRecall.test.ts | 40 ++++++++++ .../workbench/image-actions/imageRecall.ts | 43 ++++++++--- .../image-actions/useImageActions.ts | 11 ++- .../webv2/src/workbench/invocation.test.ts | 75 +++++++++++++++---- .../webv2/src/workbench/invocation.ts | 67 ++++++++++++----- .../palette/WorkbenchCommandPaletteDialog.tsx | 26 +++---- .../palette/paletteProviders.test.ts | 75 +++++++++++++++---- .../src/workbench/palette/paletteProviders.ts | 27 ++++++- .../invoke/prepareCanvasInvocation.test.ts | 37 +++++++++ .../canvas/invoke/prepareCanvasInvocation.ts | 5 +- 32 files changed, 580 insertions(+), 132 deletions(-) diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index f1ceba831f6..71267b2f750 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -2891,6 +2891,9 @@ "pidFitHelp": "Generates at the requested size, then decodes 4x and scales back down.", "pidNativeHelp": "The requested size is the 4x target; generation runs at a quarter of it.", "providerSize": "Provider size", + "providerSizeCanvasCropHelp": "The Canvas generates {{width}}×{{height}}, then fits it to the frame: text to image is cropped at the edges, edits are resized.", + "providerSizeCanvasCropTextHelp": "The Canvas generates {{width}}×{{height}}, then crops it to the frame.", + "providerSizeCanvasResizeHelp": "The Canvas generates {{width}}×{{height}} and resizes it to the frame.", "providerSizeHelp": "Sizes this provider renders natively. Other sizes are mapped to the closest one.", "providerSizeCustom": "Custom", "providerOptions": { diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs index 86f39de2806..66fea95c9c2 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.test.mjs @@ -1444,6 +1444,28 @@ test('hosted-provider credentials and the starter catalog agree with the hosted 404 ); + // A body naming neither field is refused, and changes nothing. + const refused = await fetch(`${backend.origin}/api/v1/app/external_providers/config/gemini`, { + body: JSON.stringify({}), + method: 'POST', + }); + assert.equal(refused.status, 400); + assert.equal( + (await getJson(backend, '/api/v1/app/external_providers/config')).find((c) => c.provider_id === 'gemini') + .api_key_configured, + true + ); + + // Saving an empty key removes the key and the provider's hosted records, like a reset. + const cleared = await getJson(backend, '/api/v1/app/external_providers/config/gemini', { + body: JSON.stringify({ api_key: ' ' }), + method: 'POST', + }); + assert.equal(cleared.api_key_configured, false); + const afterClear = (await getJson(backend, '/api/v2/models/')).models; + assert.equal(afterClear.filter((model) => model.provider_id === 'gemini').length, 0); + assert.equal(afterClear.length, remaining.length - hosted.filter((model) => model.provider_id === 'gemini').length); + const empty = await startMockBackend(0, { profile: 'empty' }); try { const emptyConfigs = await getJson(empty, '/api/v1/app/external_providers/config'); diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts index a86c99f8750..3fc9bef48dd 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts @@ -958,6 +958,27 @@ describe('component policies', () => { expect(result.dimensionsAdjusted).toBe(true); }); + it('keeps the Canvas frame when switching to a provider with fixed sizes', () => { + const gptImage: GenerateModelConfig = { + ...externalModel, + key: 'gpt-image', + capabilities: { + modes: ['txt2img', 'img2img'], + aspect_ratio_sizes: { '1:1': { width: 1024, height: 1024 }, '3:2': { width: 1536, height: 1024 } }, + }, + }; + const currentValues = createSettings(createModel('sdxl'), { height: 512, width: 640 }); + + // Generate snaps to the provider's size; the Canvas sync would carry that to the frame, so a frame is kept. + expect(getGenerateModelSelectionResult({ currentValues, model: gptImage, models: [] }).settings).toMatchObject({ + height: 1024, + width: 1024, + }); + const canvas = getGenerateModelSelectionResult({ currentValues, isCanvasFrame: true, model: gptImage, models: [] }); + expect(canvas.settings).toMatchObject({ height: 512, width: 640 }); + expect(canvas.dimensionsAdjusted).toBe(false); + }); + it('reports the grid clamp from an off-grid provider size back to a local model as an adjustment', () => { const provider: GenerateModelConfig = { ...externalModel, key: 'gemini', provider_id: 'gemini' }; const result = getGenerateModelSelectionResult({ diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts index 29f171ec218..222b3bd350d 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts @@ -1481,7 +1481,8 @@ export const getCompatibleReferenceImages = ( const getSettingsWithCompatibleModelSelections = ( settings: GenerateSettings, model: GenerateModelConfig, - models: readonly ReferenceModelCandidate[] + models: readonly ReferenceModelCandidate[], + isCanvasFrame: boolean ): GenerateModelSelectionResult => { const nextSettings: GenerateSettings = { ...settings, modelKey: model.key }; const clearedLabels: string[] = []; @@ -1502,7 +1503,8 @@ const getSettingsWithCompatibleModelSelections = ( const supportedSizes = sizes.length > 0 ? sizes : getExternalAspectRatioSizes(model).map(([, size]) => size); const matchesSize = supportedSizes.some((size) => size.width === width && size.height === height); - if (!matchesSize && supportedSizes.length > 0) { + // The Canvas frame is kept: the Canvas asks a provider for its own size nearest the frame and restores the result. + if (!matchesSize && supportedSizes.length > 0 && !isCanvasFrame) { // Keep the frame's shape when possible, then its pixel budget within that shape. const distance = (size: { width: number; height: number }) => ({ area: Math.abs(Math.log((size.width * size.height) / (width * height))), @@ -1638,17 +1640,20 @@ const getSettingsWithCompatibleModelSelections = ( /** Normalize, reconcile, and apply component policy before persisting model selection. */ export const getGenerateModelSelectionResult = ({ currentValues, + isCanvasFrame = false, model, models, }: { currentValues: unknown; + /** The width and height are the Canvas frame (a Canvas source), which the Canvas sync resizes to follow them. */ + isCanvasFrame?: boolean; model: GenerateModelConfig; models: readonly ModelConfig[]; }): GenerateModelSelectionResult => { const currentSettings = isGenerateSettings(currentValues) ? currentValues : (normalizeGenerateSettings(currentValues) ?? getDefaultGenerateSettings(model)); - const result = getSettingsWithCompatibleModelSelections(currentSettings, model, models); + const result = getSettingsWithCompatibleModelSelections(currentSettings, model, models, isCanvasFrame); const componentSourceModel = getAutoFlux2ComponentSourceModel(model, result.settings, models); if (componentSourceModel === undefined || componentSourceModel?.key === result.settings.componentSourceModel?.key) { diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.test.ts index 9602f44b41c..61c732b40a6 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.test.ts @@ -10,6 +10,7 @@ const BBOX: Rect = { height: 100, width: 100, x: 0, y: 0 }; const input = (overrides: Partial = {}): CanvasModeInput => ({ bbox: BBOX, bboxFullyCovered: false, + bboxHasContent: false, contentBounds: null, hasActiveInpaintMask: false, ...overrides, diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.ts index 07ed4dc0e62..e1d176dd717 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/canvasMode.ts @@ -13,6 +13,11 @@ export interface CanvasModeInput { contentBounds: Rect | null; /** Opaque coverage is alpha-scanned and contains no holes. */ bboxFullyCovered: boolean; + /** + * Some composited pixel in the bbox is not fully transparent. Local modes ignore it: content whose bounds overlap + * the frame keeps it an outpaint, as before. A hosted model's strategy reads it instead. + */ + bboxHasContent: boolean; /** True when an active mask contains inpaint content. */ hasActiveInpaintMask: boolean; } diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx index e61dce60519..be5cfd452c0 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateDimensionFields.tsx @@ -523,6 +523,7 @@ export const GenerateDimensionFields = ({ {selectedModel?.type === 'external_image_generator' ? ( openManager({ modelType: 'main' }), [openManager]); // Retain only the selected model and recompute against the current draft on confirmation. const [pendingSwitchModel, setPendingSwitchModel] = useState(null); @@ -71,7 +73,12 @@ export const GenerateModelCard = ({ return NO_PENDING_SWITCH; } - const result = getGenerateModelSelectionResult({ currentValues: settings, model: pendingSwitchModel, models }); + const result = getGenerateModelSelectionResult({ + currentValues: settings, + isCanvasFrame, + model: pendingSwitchModel, + models, + }); return { adjustedSize: result.dimensionsAdjusted @@ -84,11 +91,18 @@ export const GenerateModelCard = ({ ); const commitModelSelection = (model: GenerateModelConfig) => { - onCommitSettings(getGenerateModelSelectionResult({ currentValues: draft.getSnapshot(), model, models }).settings); + onCommitSettings( + getGenerateModelSelectionResult({ currentValues: draft.getSnapshot(), isCanvasFrame, model, models }).settings + ); }; const selectModel = (model: GenerateModelConfig) => { - const result = getGenerateModelSelectionResult({ currentValues: draft.getSnapshot(), model, models }); + const result = getGenerateModelSelectionResult({ + currentValues: draft.getSnapshot(), + isCanvasFrame, + model, + models, + }); // Lossy switches confirm before committing; lossless ones stay instant and report a moved size. if (result.clearedLabels.length > 0) { diff --git a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx index e2994d74e12..c128494d519 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.browser.test.tsx @@ -23,6 +23,9 @@ void i18n.use(initReactI18next).init({ generate: { providerSize: 'Provider size', providerSizeCustom: 'Custom', + providerSizeCanvasCropHelp: 'Canvas generates {{width}}×{{height}}, cropped to the frame; edits resized.', + providerSizeCanvasCropTextHelp: 'Canvas generates {{width}}×{{height}}, cropped to the frame.', + providerSizeCanvasResizeHelp: 'Canvas generates {{width}}×{{height}}, resized to the frame.', providerSizeHelp: 'Sizes this provider renders natively.', }, }, @@ -68,7 +71,8 @@ const settle = (action: () => void): Promise => const render = async ( model: GenerateModelConfig, - settings: { externalImageSize: string | null; height: number; width: number } + settings: { externalImageSize: string | null; height: number; width: number }, + isCanvasFrame = false ) => { host = document.createElement('div'); document.body.append(host); @@ -79,7 +83,7 @@ const render = async ( root?.render( - + ); @@ -133,6 +137,30 @@ describe('ExternalSizeSelect', () => { await render(presetModel, { externalImageSize: '1K', height: 1000, width: 1024 }); expect(trigger()?.textContent).toContain('Custom'); + expect(host?.textContent).toContain('Sizes this provider renders natively.'); + }); + + it('tells a Canvas frame off the size table which size is generated and how it is fitted', async () => { + // A text-only model is never edited, so the hint leaves edits out. + await render(ratioModel, { externalImageSize: null, height: 512, width: 640 }, true); + expect(host?.textContent).toContain('Canvas generates 1024×1024, cropped to the frame.'); + + await settle(() => root?.unmount()); + host?.remove(); + await render( + external({ + aspect_ratio_sizes: { '1:1': { height: 1024, width: 1024 }, '7:4': { height: 1024, width: 1792 } }, + modes: ['txt2img', 'img2img'], + }), + { externalImageSize: null, height: 512, width: 640 }, + true + ); + expect(host?.textContent).toContain('Canvas generates 1024×1024, cropped to the frame; edits resized.'); + + await settle(() => root?.unmount()); + host?.remove(); + await render(ratioModel, { externalImageSize: null, height: 512, width: 896 }, true); + expect(host?.textContent).toContain('Canvas generates 1792×1024, resized to the frame.'); }); it('applies a fixed ratio size, falling back to a free ratio id for shapes outside the preset list', async () => { diff --git a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx index 13b541013c8..55760f14609 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/shared/ExternalSizeSelect.tsx @@ -6,6 +6,8 @@ import { findExternalResolutionPreset, getExternalAspectRatioSizes, getExternalResolutionPresets, + isExternalTextOnlyCanvasModel, + resolveExternalCanvasRequestSize, } from '@features/generation/core/externalProvider'; import { isAspectRatioId } from '@features/generation/core/settings'; import { Field } from '@platform/ui/Field'; @@ -16,6 +18,8 @@ import { useTranslation } from 'react-i18next'; type SizeSettings = Pick; interface ExternalSizeSelectProps { + /** The size is the Canvas frame, which the Canvas generates at the provider size nearest it. */ + isCanvasFrame: boolean; model: GenerateModelConfig | undefined; settings: SizeSettings; onCommit: (patch: Partial) => void; @@ -37,7 +41,7 @@ const sizePatch = (ratio: string, width: number, height: number): Partial { +export const ExternalSizeSelect = ({ isCanvasFrame, model, onCommit, settings }: ExternalSizeSelectProps) => { const { t } = useTranslation(); // One table per model: the policy reads fresh arrays off the record, so the record is the memo key. const { collection, presets, ratioSizes } = useMemo(() => { @@ -66,9 +70,25 @@ export const ExternalSizeSelect = ({ model, onCommit, settings }: ExternalSizeSe ? null : (ratioSizes.find(([, size]) => size.width === settings.width && size.height === settings.height)?.[0] ?? null); const value = currentPreset ? [presetKey(currentPreset)] : currentRatio ? [currentRatio] : []; + // Hosted providers have no grid, so on a Canvas route the width and height are the frame itself. + const canvasRequest = + isCanvasFrame && model && value.length === 0 ? resolveExternalCanvasRequestSize(model, settings) : null; + const canvasHelpKey = + canvasRequest && canvasRequest.width * settings.height === canvasRequest.height * settings.width + ? 'widgets.generate.providerSizeCanvasResizeHelp' + : model && isExternalTextOnlyCanvasModel(model) + ? 'widgets.generate.providerSizeCanvasCropTextHelp' + : 'widgets.generate.providerSizeCanvasCropHelp'; return ( - +