From b5f3ba52f3d4f367df85dd21ad25aae57b1950cc Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sat, 10 Oct 2026 21:55:17 -0600 Subject: [PATCH 1/5] feat(generation): restore the SDXL Refiner in Generate and Canvas Add refiner settings (model, steps, CFG, scheduler, start, aesthetic scores) with legacy defaults, a Refiner section shown for SDXL mains, and a refiner denoise stage grafted before the decode for Generate txt2img and Canvas img2img/inpaint/outpaint, with PiD rejected alongside it. Record refiner metadata and restore it through Recall All and Remix, resolving the model by key, hash or name and base. Extend the mock fixture with SDXL and refiner mains, the backend graph contract, and the Generate and SDXL docs. --- .../03.Models/Local Models/sdxl.mdx | 9 +- .../04.Image Generation/generate-panel.mdx | 6 +- .../frontend/webv2/public/locales/en.json | 44 +++- .../webv2/scripts/mock-backend-fixtures.mjs | 8 +- .../__snapshots__/generateGraphNodeTypes.json | 73 +++++- .../core/baseGenerationPolicies.test.ts | 74 ++++++ .../generation/core/baseGenerationPolicies.ts | 29 +++ .../core/canvas/compileCanvasGraph.test.ts | 89 +++++++ .../core/canvas/compileCanvasGraph.ts | 26 +- .../features/generation/core/graph.test.ts | 129 ++++++++++ .../src/features/generation/core/graph.ts | 10 +- .../generation/core/graphCoverage.test.ts | 33 ++- .../generation/core/sdxlRefiner.test.ts | 24 ++ .../features/generation/core/sdxlRefiner.ts | 104 ++++++++ .../generation/core/sdxlRefinerGraph.ts | 109 +++++++++ .../features/generation/core/settings.test.ts | 63 +++++ .../src/features/generation/core/settings.ts | 46 ++++ .../src/features/generation/core/types.ts | 10 + .../webv2/src/features/generation/settings.ts | 11 + .../GenerateRefinerSection.browser.test.tsx | 172 +++++++++++++ .../generation/ui/GenerateRefinerSection.tsx | 226 ++++++++++++++++++ .../generation/ui/GenerateSettingsForm.tsx | 9 + .../src/platform/ui/hints/hintRegistry.ts | 6 + .../importGalleryImages.test.ts | 7 + .../image-actions/imageRecall.test.ts | 168 +++++++++++++ .../workbench/image-actions/imageRecall.ts | 127 +++++++++- .../src/workbench/workbenchState.test.ts | 7 + 27 files changed, 1597 insertions(+), 22 deletions(-) create mode 100644 invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts create mode 100644 invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts create mode 100644 invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts create mode 100644 invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx create mode 100644 invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.tsx diff --git a/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx b/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx index 84a8e26b810..353c9903e7e 100644 --- a/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx @@ -181,8 +181,13 @@ image on its own. - **Installing** — install **SDXL Refiner** from the starter models, or any refiner checkpoint or Diffusers folder; InvokeAI recognizes refiners automatically and lists them under their own **SDXL Refiner** family. -- **Using it** — the refiner is not selectable in the Generate panel or on the Canvas. Use it in the - workflow editor. +- **Using it** — with an SDXL main model selected, the Generate panel shows a **Refiner** section. + Pick the refiner there and set **Refiner start** (the point where the base model hands over, 0.8 + by default), its steps, CFG, scheduler and the positive and negative **aesthetic scores**. The + same settings apply to SDXL generations from the Canvas, including inpainting and outpainting. The + refiner is never offered as a main model, and it cannot be combined with PiD decoding. An image + made with a refiner records it in its metadata, so **Recall All** and **Remix Image** restore it. + The workflow editor offers the same nodes for custom graphs. A refiner workflow runs two denoise passes over the same latents: diff --git a/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx b/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx index aa05b74a55b..53e0216e08f 100644 --- a/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx +++ b/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx @@ -62,11 +62,15 @@ The **Advanced** section appears for models with extra options, and its header c * **PiD Decode** — decodes the image at up to four times its size; see [PiD Super-Resolution Decode](/users-guide/image-generation/advanced-topics/pid-decode/). * **Seamless tiling** — makes the image tile along the X axis, the Y axis, or both. +### Refiner + +The **Refiner** section appears for SDXL models. Pick an **SDXL Refiner** model to have it finish the last part of the denoising schedule; the base model stops at **Refiner start** (0.8 by default, so the refiner handles the final 20%) and the refiner continues from there with its own **Refiner steps**, **Refiner CFG** and **Refiner scheduler**. The **Positive** and **Negative aesthetic score** condition the refiner on the aesthetic rating it was trained with. The controls are disabled until a refiner is selected, and refiner models are only offered here, never as the main model. The refiner also runs for SDXL generations from the Canvas. It cannot be combined with **PiD Decode**. See [SDXL](/users-guide/models/local-models/sdxl/#sdxl-refiner). + ### Resetting to defaults Each model has recommended settings, which you can change in the Model Manager. **Reset all to model defaults**, the ↺ button in the Generate panel's title bar, applies them: size, steps, CFG or guidance, scheduler and VAE settings. Your prompts are left alone. The button is disabled when your settings already match the defaults. -To return to the settings of an earlier image instead, right-click it in the Gallery or Preview and open **Recall Metadata**. **Recall All** restores its model, concepts, prompts, seed, size and render settings; **Remix Image** does the same but keeps a new seed; and **Use Prompt**, **Use Seed** and **Use Size** recall one thing at a time. Recalled concepts replace the ones in the panel at the weights the image used; any that are not installed or do not fit the model are listed in a notice. If the image's model is not installed, or the image was saved from the canvas, your current concepts are kept. +To return to the settings of an earlier image instead, right-click it in the Gallery or Preview and open **Recall Metadata**. **Recall All** restores its model, concepts, prompts, seed, size, render and refiner settings; **Remix Image** does the same but keeps a new seed; and **Use Prompt**, **Use Seed** and **Use Size** recall one thing at a time. Recalled concepts replace the ones in the panel at the weights the image used; any that are not installed or do not fit the model are listed in a notice. If the image's model is not installed, or the image was saved from the canvas, your current concepts are kept. To keep a combination of settings for later, use **Presets** in the panel's title bar to save the current settings under a name and apply them again. diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 728a9fad1f5..d4db3e4a2f5 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -1062,6 +1062,38 @@ "width": { "heading": "Width", "paragraphs": ["Width of the generated image. Must be a multiple of 8."] + }, + "refinerModel": { + "heading": "Refiner model", + "paragraphs": [ + "An SDXL Refiner model that takes over the last part of the denoising schedule from the SDXL base model, adding fine detail.", + "Only SDXL Refiner models are listed here; they cannot be used as the main model." + ] + }, + "refinerSteps": { + "heading": "Refiner steps", + "paragraphs": ["Number of steps the refiner schedule is divided into. Only the part after Refiner start runs."] + }, + "refinerCfgScale": { + "heading": "Refiner CFG", + "paragraphs": ["How strictly the refiner follows the prompt during its pass, like the main CFG."] + }, + "refinerScheduler": { + "heading": "Refiner scheduler", + "paragraphs": ["Scheduler used for the refiner pass, independent of the base model's scheduler."] + }, + "refinerStart": { + "heading": "Refiner start", + "paragraphs": [ + "Where in the schedule the base model hands over to the refiner.", + "0.8 means the base model denoises the first 80% and the refiner finishes the last 20%; 0 hands the whole schedule to the refiner." + ] + }, + "refinerAestheticScore": { + "heading": "Aesthetic score", + "paragraphs": [ + "The refiner was trained with an aesthetic rating from 1 to 10. The positive score steers the result toward images rated that highly; the negative score steers it away from images rated that low." + ] } }, "models": { @@ -2655,7 +2687,17 @@ "pidSteps": "PiD steps", "pidHelp": "Replaces the VAE decode with a 4x super-resolution decode.", "pidFitHelp": "Generates at the requested size, then decodes 4x and scales back down.", - "pidNativeHelp": "The requested size is the 4x target; generation runs at a quarter of it." + "pidNativeHelp": "The requested size is the 4x target; generation runs at a quarter of it.", + "refiner": "Refiner", + "refinerModel": "Refiner model", + "refinerNone": "None", + "noRefinerModels": "No SDXL Refiner models are installed.", + "refinerSteps": "Refiner steps", + "refinerCfgScale": "Refiner CFG", + "refinerScheduler": "Refiner scheduler", + "refinerStart": "Refiner start", + "refinerPositiveAestheticScore": "Positive aesthetic score", + "refinerNegativeAestheticScore": "Negative aesthetic score" }, "upscale": { "concepts": "Concepts", diff --git a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs index b29a99707e6..2103c11d90e 100644 --- a/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs +++ b/invokeai/frontend/webv2/scripts/mock-backend-fixtures.mjs @@ -395,14 +395,20 @@ const createQueueItems = (count) => const MODEL_BASES = ['sd-1', 'sdxl', 'flux', 'flux2', 'qwen-image']; const MODEL_TYPES = ['main', 'lora', 'controlnet', 'vae', 'embedding']; +// Main models cycle through their own bases so Generate sees SDXL mains and refiners, not only SD 1.5. +const MAIN_MODEL_BASES = ['sd-1', 'sdxl', 'sdxl-refiner']; const createModels = (count) => range(count, (index) => { const id = ordinal(index); const type = MODEL_TYPES[index % MODEL_TYPES.length]; + const base = + type === 'main' + ? MAIN_MODEL_BASES[Math.floor(index / MODEL_TYPES.length) % MAIN_MODEL_BASES.length] + : MODEL_BASES[index % MODEL_BASES.length]; return { - base: MODEL_BASES[index % MODEL_BASES.length], + base, cover_image: null, default_settings: type === 'main' ? { cfg_scale: 7, height: 1024, steps: 30, width: 1024 } : null, description: `Deterministic representative ${type} model ${id}.`, diff --git a/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json index 88a3f8d4de9..42b8f61ad36 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json +++ b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json @@ -40,7 +40,8 @@ "sdxl": { "componentsFilled": { "diffusers": [], - "standalone-components": [] + "standalone-components": [], + "diffusers-with-refiner": [] }, "nodeTypes": [ "collect", @@ -51,6 +52,8 @@ "noise", "sdxl_compel_prompt", "sdxl_model_loader", + "sdxl_refiner_compel_prompt", + "sdxl_refiner_model_loader", "string" ] }, @@ -597,6 +600,25 @@ "rand_device": [ "cpu" ], + "refiner_cfg_scale": [ + 7.5 + ], + "refiner_model": [], + "refiner_negative_aesthetic_score": [ + 2.5 + ], + "refiner_positive_aesthetic_score": [ + 6 + ], + "refiner_scheduler": [ + "euler" + ], + "refiner_start": [ + 0.8 + ], + "refiner_steps": [ + 20 + ], "scheduler": [ "euler_a", "euler" @@ -637,6 +659,7 @@ }, "denoise_latents": { "inputs": [ + "latents", "negative_conditioning", "noise", "positive_conditioning", @@ -647,13 +670,16 @@ 0 ], "cfg_scale": [ - 7 + 7, + 7.5 ], "denoising_end": [ + 0.8, 1 ], "denoising_start": [ - 0 + 0, + 0.8 ], "hidiffusion": [ false @@ -668,9 +694,11 @@ true ], "scheduler": [ - "euler_a" + "euler_a", + "euler" ], "steps": [ + 20, 30 ], "use_cache": [ @@ -1468,6 +1496,43 @@ "vae" ] }, + "sdxl_refiner_compel_prompt": { + "inputs": [ + "clip2", + "style" + ], + "literalInputs": { + "aesthetic_score": [ + 2.5, + 6 + ], + "is_intermediate": [ + true + ], + "use_cache": [ + true + ] + }, + "outputs": [ + "conditioning" + ] + }, + "sdxl_refiner_model_loader": { + "inputs": [], + "literalInputs": { + "is_intermediate": [ + true + ], + "model": [], + "use_cache": [ + true + ] + }, + "outputs": [ + "clip2", + "unet" + ] + }, "string": { "inputs": [], "literalInputs": { diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts index 68d069e8e72..b842b5db79c 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts @@ -1533,3 +1533,77 @@ describe('getGenerationDimensions and the variant it dispatches on', () => { expect(getGenerationDimensions({ base: 'flux', type: 'main' }).optimal).toBe(1024); }); }); + +describe('SDXL Refiner policy', () => { + const refiner: MainModelConfig = { base: 'sdxl-refiner', key: 'refiner', name: 'SDXL Refiner', type: 'main' }; + + it('offers the refiner section for SDXL only and never lists refiners as main models', () => { + expect(getGenerationUiPolicy(createModel('sdxl')).refinerVisible).toBe(true); + expect(getGenerationUiPolicy(createModel('sd-1')).refinerVisible).toBe(false); + expect(getGenerationUiPolicy(createModel('flux')).refinerVisible).toBe(false); + expect(isGenerateModelSelectable(refiner)).toBe(false); + expect(isSupportedGenerateModel(refiner)).toBe(false); + }); + + it('clears the refiner when leaving SDXL but keeps its tuning for the next SDXL pick', () => { + const result = getGenerateModelSelectionResult({ + currentValues: createSettings(createModel('sdxl'), { refinerModel: refiner, refinerStart: 0.6 }), + model: createModel('flux'), + models: [], + }); + + expect(result.settings.refinerModel).toBeNull(); + expect(result.settings.refinerStart).toBe(0.6); + expect(result.clearedLabels).toContain('Refiner'); + + const kept = getGenerateModelSelectionResult({ + currentValues: createSettings(createModel('sdxl'), { refinerModel: refiner }), + model: createModel('sdxl', { key: 'sdxl-other' }), + models: [], + }); + + expect(kept.settings.refinerModel).toEqual(refiner); + expect(kept.clearedLabels).toEqual([]); + }); + + it('validates the refiner only on SDXL, rejecting PiD and out-of-range tuning', () => { + const sdxl = createModel('sdxl'); + const withRefiner = (overrides: Partial = {}) => + getGenerationValidationReasons(sdxl, createSettings(sdxl, { refinerModel: refiner, ...overrides })); + + expect(withRefiner()).toEqual([]); + expect(withRefiner({ refinerStart: 1.2, refinerSteps: 0 })).toEqual([ + 'Refiner steps must be a whole number of at least 1.', + 'Refiner start must be between 0 and 1.', + ]); + expect(withRefiner({ refinerNegativeAestheticScore: 0 })).toEqual([ + 'Refiner negative aesthetic score must be between 1 and 10.', + ]); + expect( + withRefiner({ + gemma2EncoderModel: { base: 'any', key: 'gemma2', name: 'Gemma 2', type: 'gemma2_encoder' }, + pidDecoderModel: { base: 'sdxl', key: 'pid', name: 'PiD SDXL', type: 'pid_decoder' }, + pidMode: 'fit', + }) + ).toContain('PiD decode is not compatible with the SDXL Refiner. Turn PiD off or clear the refiner model.'); + expect(withRefiner({ refinerModel: sdxl })).toEqual([ + `${sdxl.name} is not an SDXL Refiner model. Pick a refiner or clear the slot.`, + ]); + + const sd1 = createModel('sd-1'); + + expect( + getGenerationValidationReasons(sd1, createSettings(sd1, { refinerModel: refiner, refinerStart: 5 })) + ).toEqual([]); + }); + + it('reports an uninstalled refiner', () => { + const sdxl = createModel('sdxl'); + const settings = createSettings(sdxl, { refinerModel: refiner }); + + expect(getGenerationModelAvailabilityReasons(sdxl, settings, [sdxl as never])).toEqual([ + 'Refiner "SDXL Refiner" is no longer installed.', + ]); + expect(getGenerationModelAvailabilityReasons(sdxl, settings, [sdxl as never, refiner as never])).toEqual([]); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts index e3e849062f2..6f277fe6041 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts @@ -55,6 +55,16 @@ import { getPidDecoderBaseForMainBase, getPidDimensionOverrides, } from './pid'; +import { + DEFAULT_REFINER_CFG_SCALE, + DEFAULT_REFINER_NEGATIVE_AESTHETIC_SCORE, + DEFAULT_REFINER_POSITIVE_AESTHETIC_SCORE, + DEFAULT_REFINER_SCHEDULER, + DEFAULT_REFINER_START, + DEFAULT_REFINER_STEPS, + getRefinerValidationReasons, + isRefinerSupported, +} from './sdxlRefiner'; import { clampDimension, DEFAULT_HIDIFFUSION_T1_RATIO, @@ -356,6 +366,7 @@ export const getGenerationUiPolicy = (model: GenerateModelConfig | undefined) => seedVisible, // Show PiD controls on supported bases even while PiD is disabled. pidVisible: getIsPidSupportedBase(model?.base), + refinerVisible: isRefinerSupported(model), }; }; @@ -478,6 +489,13 @@ export const getDefaultGenerateSettings = (model?: GenerateModelConfig): Generat gemma2EncoderModel: null, pidSteps: DEFAULT_PID_STEPS, referenceImages: [], + refinerModel: null, + refinerSteps: DEFAULT_REFINER_STEPS, + refinerCfgScale: DEFAULT_REFINER_CFG_SCALE, + refinerScheduler: DEFAULT_REFINER_SCHEDULER, + refinerPositiveAestheticScore: DEFAULT_REFINER_POSITIVE_AESTHETIC_SCORE, + refinerNegativeAestheticScore: DEFAULT_REFINER_NEGATIVE_AESTHETIC_SCORE, + refinerStart: DEFAULT_REFINER_START, scheduler: defaults.scheduler, seamlessXAxis: false, seamlessYAxis: false, @@ -1441,6 +1459,12 @@ const getSettingsWithCompatibleModelSelections = ( addClearedLabel(clearedLabels, 'HiDiffusion'); } + // The refiner only ever follows an SDXL base; its tuning values stay for the next SDXL pick. + if (!uiPolicy.refinerVisible && nextSettings.refinerModel) { + nextSettings.refinerModel = null; + addClearedLabel(clearedLabels, 'Refiner'); + } + if (!uiPolicy.seamlessVisible && (nextSettings.seamlessXAxis || nextSettings.seamlessYAxis)) { nextSettings.seamlessXAxis = false; nextSettings.seamlessYAxis = false; @@ -1507,6 +1531,10 @@ export const getGenerationModelAvailabilityReasons = ( } } + if (settings.refinerModel && isRefinerSupported(model) && !hasModelKey(models, settings.refinerModel.key, 'main')) { + reasons.push(`Refiner "${settings.refinerModel.name}" is no longer installed.`); + } + for (const lora of settings.loras) { if (!hasModelKey(models, lora.model.key, 'lora')) { reasons.push(`LoRA "${lora.model.name}" is no longer installed.`); @@ -1714,6 +1742,7 @@ export const getGenerationValidationReasons = (model: GenerateModelConfig, setti } reasons.push(...getModelFamilyValidationReasons(model, settings)); + reasons.push(...getRefinerValidationReasons(model, settings)); return reasons; }; diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index f0c5d3d695a..408e1063131 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -1113,3 +1113,92 @@ describe('compileCanvasGraph — regional guidance', () => { expect(collectedSources).toEqual(['krea2_seed_variance', 'rg_krea2_r1_seed_variance', 'rg_krea2_r2_seed_variance']); }); }); + +describe('compileCanvasGraph — SDXL Refiner', () => { + const refinerModel: MainModelConfig = { + base: 'sdxl-refiner', + key: 'sdxl-refiner-model', + name: 'SDXL Refiner', + type: 'main', + }; + const refinerSettings: Partial = { refinerModel, refinerStart: 0.8, refinerSteps: 10 }; + + it('hands txt2img latents to the refiner before the resize to the bbox', () => { + const { backendGraph } = compile(sdxlModel, 'txt2img', { settings: refinerSettings }); + + expect(backendGraph.nodes.denoise_latents).toMatchObject({ denoising_end: 0.8, denoising_start: 0 }); + expect(backendGraph.nodes.refiner_denoise_latents).toMatchObject({ denoising_end: 1, denoising_start: 0.8 }); + expect(getEdge(backendGraph, 'refiner_denoise_latents', 'latents')?.source.node_id).toBe('denoise_latents'); + expect(getEdge(backendGraph, 'canvas_output', 'latents')?.source.node_id).toBe('refiner_denoise_latents'); + }); + + it('stops the img2img base pass at the hand-over and keeps its start below it', () => { + const { backendGraph } = compile(sdxlModel, 'img2img', { settings: refinerSettings, strength: 0.6 }); + + // 1 - 0.6 = 0.4 is below the 0.8 hand-over, so the base pass runs 0.4 → 0.8. + expect(backendGraph.nodes.denoise_latents?.denoising_start).toBeCloseTo(0.4); + expect(backendGraph.nodes.denoise_latents?.denoising_end).toBe(0.8); + expect(backendGraph.nodes.refiner_denoise_latents).toMatchObject({ denoising_end: 1, denoising_start: 0.8 }); + expect(getEdge(backendGraph, 'denoise_latents', 'latents')?.source.node_id).toBe('canvas_i2l'); + expect(getEdge(backendGraph, 'refiner_denoise_latents', 'latents')?.source.node_id).toBe('denoise_latents'); + expect(getNodeByType(backendGraph, 'core_metadata')).toMatchObject({ + generation_mode: 'sdxl_img2img', + refiner_model: { key: 'sdxl-refiner-model' }, + refiner_start: 0.8, + strength: 0.6, + }); + }); + + it('caps a low-strength img2img start at the hand-over so the base pass is not inverted', () => { + const { backendGraph } = compile(sdxlModel, 'img2img', { settings: refinerSettings, strength: 0.1 }); + + expect(backendGraph.nodes.denoise_latents).toMatchObject({ denoising_end: 0.8, denoising_start: 0.8 }); + }); + + it.each(['inpaint', 'outpaint'] as const)( + 'keeps the mask on the base pass and decodes the refiner for %s', + (mode) => { + const { backendGraph } = compile(sdxlModel, mode, { settings: refinerSettings, strength: 0.5 }); + + expect(backendGraph.nodes.denoise_latents).toMatchObject({ denoising_end: 0.8, denoising_start: 0.5 }); + expect(getEdge(backendGraph, 'denoise_latents', 'denoise_mask')?.source.node_id).toBe('create_gradient_mask'); + expect(getEdge(backendGraph, 'refiner_denoise_latents', 'denoise_mask')).toBeUndefined(); + expect(getEdge(backendGraph, 'refiner_denoise_latents', 'latents')?.source.node_id).toBe('denoise_latents'); + expect(getEdge(backendGraph, 'canvas_l2i', 'latents')?.source.node_id).toBe('refiner_denoise_latents'); + expect(getNodeByType(backendGraph, 'core_metadata')?.generation_mode).toBe(`sdxl_${mode}`); + } + ); + + it('attaches control layers to the base pass, not the refiner', () => { + const { backendGraph } = compile(sdxlModel, 'txt2img', { + controlLayers: [ + { + beginEndStepPct: [0, 1], + controlMode: 'balanced', + id: 'control-1', + imageName: 'control.png', + kind: 'controlnet', + model: { base: 'sdxl', key: 'sdxl-controlnet', name: 'SDXL ControlNet', type: 'controlnet' }, + weight: 1, + }, + ], + settings: refinerSettings, + }); + + expect(getEdge(backendGraph, 'denoise_latents', 'control')).toBeDefined(); + expect(getEdge(backendGraph, 'refiner_denoise_latents', 'control')).toBeUndefined(); + }); + + it('still refuses PiD on the canvas when a refiner is selected', () => { + expect(() => + compile(sdxlModel, 'img2img', { + settings: { + ...refinerSettings, + gemma2EncoderModel: { base: 'any', key: 'g', name: 'Gemma 2', type: 'gemma2_encoder' }, + pidDecoderModel: { base: 'sdxl', key: 'pid', name: 'PiD SDXL', type: 'pid_decoder' }, + pidMode: 'fit', + }, + }) + ).toThrow('PiD decoding is not supported on the canvas yet.'); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts index 8155895bd1e..86d0ab893fb 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts @@ -12,6 +12,7 @@ import { GRAPH_BUILDERS } from '@features/generation/core/graph'; import { addEdge, addNode, toGraphContract } from '@features/generation/core/graphBuilder'; import { addKrea2ConditioningEnhancers } from '@features/generation/core/krea2Conditioning'; import { getIsPidSupportedBase } from '@features/generation/core/pid'; +import { getActiveRefiner, getRefinerDenoisingSplit } from '@features/generation/core/sdxlRefiner'; import type { CanvasCompositingSettings, @@ -51,6 +52,22 @@ const canvasDenoisingStart = (model: GenerateModelConfig, strength: number): num return 1 - strength ** (usesOptimizedCurve ? 0.2 : 1); }; +/** Applies the strength-derived range to the base denoise; with a refiner, the base pass stops at the hand-over. */ +const setCanvasDenoisingRange = ( + denoise: BackendInvocationContract, + model: GenerateModelConfig, + settings: GenerateSettings, + strength: number +): void => { + const denoisingStart = canvasDenoisingStart(model, strength); + const range = getActiveRefiner(settings, model) + ? getRefinerDenoisingSplit(settings.refinerStart, denoisingStart) + : { denoising_end: 1, denoising_start: denoisingStart }; + + denoise.denoising_start = range.denoising_start; + denoise.denoising_end = range.denoising_end; +}; + /** True for the image-referencing modes (everything but pure txt2img). */ const isImageMode = (mode: CompileCanvasGraphInput['mode']): boolean => mode !== 'txt2img'; @@ -274,8 +291,7 @@ const graftImageToImage = ( addEdge(graph, vaeSource.node, vaeSource.field, encode, 'vae'); addEdge(graph, encode, 'latents', denoise, 'latents'); - denoise.denoising_start = canvasDenoisingStart(model, strength); - denoise.denoising_end = 1; + setCanvasDenoisingRange(denoise, model, settings, strength); setMetadataMode(graph, 'img2img', strength); }; @@ -465,8 +481,7 @@ const graftInpaint = ( const strength = input.strength; const needsResize = !sizesMatch(input.bbox, processingSize); - denoise.denoising_start = canvasDenoisingStart(model, strength); - denoise.denoising_end = 1; + setCanvasDenoisingRange(denoise, model, input.settings, strength); const initialResize = needsResize ? addInputResizeNode(graph, 'canvas_resize_initial_to_processing', initialImageName, processingSize) @@ -530,8 +545,7 @@ const graftOutpaint = ( const strength = input.strength; const needsResize = !sizesMatch(input.bbox, processingSize); - denoise.denoising_start = canvasDenoisingStart(model, strength); - denoise.denoising_end = 1; + setCanvasDenoisingRange(denoise, model, input.settings, strength); // Infill the transparent region before encode (legacy `getInfill`). const infill = addInfillNode(graph, compositing.infillMethod, compositing); diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts index 789d05819c4..3db4143b399 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts @@ -1282,3 +1282,132 @@ describe('PiD decode', () => { ).toThrow(/multiple of 64/); }); }); + +describe('SDXL Refiner', () => { + const refinerModel: MainModelConfig = { + base: 'sdxl-refiner', + hash: 'blake3:refiner', + key: 'sdxl-refiner-model', + name: 'SDXL Refiner', + type: 'main', + }; + const refinerSettings: Partial = { + refinerCfgScale: 5, + refinerModel, + refinerNegativeAestheticScore: 2, + refinerPositiveAestheticScore: 7, + refinerScheduler: 'dpmpp_2m', + refinerStart: 0.7, + refinerSteps: 12, + }; + + it('leaves the SDXL graph unchanged without a refiner', () => { + const graph = compile(sdxlModel); + + expect(graph.nodes.refiner_denoise_latents).toBeUndefined(); + expect(graph.nodes.refiner_model_loader).toBeUndefined(); + expect(graph.nodes.denoise_latents).toMatchObject({ denoising_end: 1, denoising_start: 0 }); + expect(getEdge(graph, 'canvas_output', 'latents')?.source.node_id).toBe('denoise_latents'); + expect(getNodeByType(graph, 'core_metadata')).not.toHaveProperty('refiner_model'); + }); + + it('hands the base latents to a refiner stage that feeds the decode', () => { + const graph = compile(sdxlModel, refinerSettings); + const refinerDenoise = graph.nodes.refiner_denoise_latents; + + expect(graph.nodes.denoise_latents).toMatchObject({ denoising_end: 0.7, denoising_start: 0 }); + expect(graph.nodes.refiner_model_loader).toMatchObject({ + model: { base: 'sdxl-refiner', hash: 'blake3:refiner', key: 'sdxl-refiner-model', type: 'main' }, + type: 'sdxl_refiner_model_loader', + }); + expect(refinerDenoise).toMatchObject({ + cfg_scale: 5, + denoising_end: 1, + denoising_start: 0.7, + scheduler: 'dpmpp_2m', + steps: 12, + type: 'denoise_latents', + }); + expect(graph.nodes.refiner_pos_cond).toMatchObject({ aesthetic_score: 7, type: 'sdxl_refiner_compel_prompt' }); + expect(graph.nodes.refiner_neg_cond).toMatchObject({ aesthetic_score: 2, type: 'sdxl_refiner_compel_prompt' }); + + expect(getEdge(graph, 'refiner_denoise_latents', 'unet')?.source).toEqual({ + field: 'unet', + node_id: 'refiner_model_loader', + }); + expect(getEdge(graph, 'refiner_pos_cond', 'clip2')?.source.node_id).toBe('refiner_model_loader'); + expect(getEdge(graph, 'refiner_neg_cond', 'clip2')?.source.node_id).toBe('refiner_model_loader'); + expect(getEdge(graph, 'refiner_pos_cond', 'style')?.source.node_id).toBe('positive_prompt'); + expect(getEdge(graph, 'refiner_neg_cond', 'style')?.source.node_id).toBe('negative_prompt'); + expect(getEdge(graph, 'refiner_denoise_latents', 'positive_conditioning')?.source.node_id).toBe('refiner_pos_cond'); + expect(getEdge(graph, 'refiner_denoise_latents', 'negative_conditioning')?.source.node_id).toBe('refiner_neg_cond'); + expect(getEdge(graph, 'refiner_denoise_latents', 'latents')?.source.node_id).toBe('denoise_latents'); + expect(getEdge(graph, 'canvas_output', 'latents')?.source.node_id).toBe('refiner_denoise_latents'); + // The base VAE still decodes; the refiner has none of its own. + expect(getEdge(graph, 'canvas_output', 'vae')?.source.node_id).toBe('model_loader'); + }); + + it('records the refiner and its tuning in metadata', () => { + const graph = compile(sdxlModel, refinerSettings); + + expect(getNodeByType(graph, 'core_metadata')).toMatchObject({ + refiner_cfg_scale: 5, + refiner_model: { base: 'sdxl-refiner', hash: 'blake3:refiner', key: 'sdxl-refiner-model', type: 'main' }, + refiner_negative_aesthetic_score: 2, + refiner_positive_aesthetic_score: 7, + refiner_scheduler: 'dpmpp_2m', + refiner_start: 0.7, + refiner_steps: 12, + }); + }); + + it('wraps the refiner UNet in its own seamless node when tiling is on', () => { + const graph = compile(sdxlModel, { ...refinerSettings, seamlessXAxis: true }); + + expect(graph.nodes.refiner_seamless).toMatchObject({ seamless_x: true, seamless_y: false, type: 'seamless' }); + expect(getEdge(graph, 'refiner_seamless', 'unet')?.source.node_id).toBe('refiner_model_loader'); + expect(getEdge(graph, 'refiner_denoise_latents', 'unet')?.source.node_id).toBe('refiner_seamless'); + }); + + it('runs the refiner pass with the base HiDiffusion settings and coerces its scheduler', () => { + const graph = compile(sdxlModel, { + ...refinerSettings, + hiDiffusionEnabled: true, + hiDiffusionT1Ratio: 0.3, + refinerScheduler: 'not-a-scheduler', + }); + + expect(graph.nodes.refiner_denoise_latents).toMatchObject({ + hidiffusion: true, + hidiffusion_t1_ratio: 0.3, + scheduler: 'euler_a', + }); + expect(getNodeByType(graph, 'core_metadata')?.refiner_scheduler).toBe('euler_a'); + }); + + it('ignores a stored refiner on a non-SDXL main model', () => { + const graph = compile(sd1Model, refinerSettings); + + expect(graph.nodes.refiner_denoise_latents).toBeUndefined(); + expect(getNodeByType(graph, 'core_metadata')).not.toHaveProperty('refiner_model'); + }); + + it('rejects a refiner together with PiD decoding', () => { + const gemma2: ComponentModelConfig = { base: 'any', key: 'gemma2', name: 'Gemma 2 2B', type: 'gemma2_encoder' }; + + expect(() => + compile(sdxlModel, { + ...refinerSettings, + gemma2EncoderModel: gemma2, + pidDecoderModel: { base: 'sdxl', key: 'pid-sdxl', name: 'PiD SDXL', type: 'pid_decoder' }, + pidMode: 'fit', + }) + ).toThrow('PiD decode is not compatible with the SDXL Refiner.'); + }); + + it('refuses a non-refiner model stored in the refiner slot', () => { + expect(() => compile(sdxlModel, { ...refinerSettings, refinerModel: sdxlModel })).toThrow( + 'SDXL is not an SDXL Refiner model.' + ); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.ts index 02be3aae118..59637951803 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.ts @@ -54,6 +54,8 @@ import { import { addKrea2ConditioningEnhancers } from './krea2Conditioning'; import { addPidDecode, getPidDenoiseSize, getPidMetadata, shouldUsePidDecode } from './pidGraph'; import { getEffectiveReferenceImage } from './referenceImage'; +import { getActiveRefiner } from './sdxlRefiner'; +import { addSDXLRefinerStage, getRefinerMetadata } from './sdxlRefinerGraph'; const getCompatibleComponentSource = ( settings: GenerateSettings, @@ -430,9 +432,14 @@ const buildSDGraph = ( addEdge(graph, unetSource, 'unet', denoise, 'unet'); addEdge(graph, noise, 'noise', denoise, 'noise'); addIPAdapterReferenceImages(graph, settings, model, denoise); + // Validation rejects PiD alongside a refiner, so the refiner always feeds the ordinary VAE decode. + const refiner = getActiveRefiner(settings, model); + const refinerDenoise = refiner + ? addSDXLRefinerStage({ denoise, graph, model, negativePrompt, positivePrompt, refiner, seamless, settings }) + : null; // Create decode only after resolving the final VAE source. const output = addDecodeOutput({ - denoise, + denoise: refinerDenoise ?? denoise, graph, l2iProps: { color_compensation: colorCompensation, fp32: settings.vaePrecision === 'fp32' }, l2iType: 'l2i', @@ -455,6 +462,7 @@ const buildSDGraph = ( } : {}), ...(shouldUsePidDecode(settings, model.base) ? getPidMetadata(settings) : {}), + ...(refiner ? getRefinerMetadata(settings, model, refiner) : {}), scheduler, }); addReferenceImageMetadata(graph, output, settings); diff --git a/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.test.ts b/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.test.ts index 594703a4de0..c17a65de940 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.test.ts @@ -6,7 +6,7 @@ import { seedArchitectureCapabilities } from '@features/generation/core/architec import { describe, expect, it } from 'vitest'; import type { GenerateComponentValueKey, SupportedGenerateBase } from './baseGenerationPolicies'; -import type { ModelIdentifierConfig, VaeModelConfig } from './types'; +import type { MainModelConfig, ModelIdentifierConfig, VaeModelConfig } from './types'; import { getComponentSectionPolicy, @@ -36,6 +36,14 @@ const compileForShape = ( return { filled, graph: compileGenerateGraph(settings, model, 'gallery', { useCpuNoise: true }).backendGraph }; }; +const SDXL_REFINER_SHAPE: ModelShape = { label: 'diffusers-with-refiner', overrides: { format: 'diffusers' } }; +const SDXL_REFINER_MODEL: MainModelConfig = { + base: 'sdxl-refiner', + key: 'sdxl-refiner', + name: 'SDXL Refiner', + type: 'main', +}; + seedArchitectureCapabilities(); describe('generate graph coverage', () => { @@ -162,8 +170,27 @@ describe('generate graph coverage', () => { const fieldsFor = (nodeType: string) => (fields[nodeType] ??= { inputs: new Set(), literals: new Map(), outputs: new Set() }); - for (const { base, shape } of generateGraphCases) { - const { filled, graph } = compileForShape(base, shape); + // The refiner is a per-image choice, not a component slot, so it needs its own SDXL case. + const refinerCase = () => { + const { filled, model, settings } = satisfiedSettingsFor('sdxl', SDXL_REFINER_SHAPE); + const withRefiner = { ...settings, refinerModel: SDXL_REFINER_MODEL }; + + expect(getGenerationValidationReasons(model, withRefiner)).toEqual([]); + + return { + base: 'sdxl' as const, + filled, + graph: compileGenerateGraph(withRefiner, model, 'gallery', { useCpuNoise: true }).backendGraph, + shape: SDXL_REFINER_SHAPE, + }; + }; + const contractCases = [ + ...generateGraphCases.map(({ base, shape }) => () => ({ base, shape, ...compileForShape(base, shape) })), + refinerCase, + ]; + + for (const compileCase of contractCases) { + const { base, filled, graph, shape } = compileCase(); const entry = (byBase[base] ??= { componentsFilled: {}, nodeTypes: [] }); entry.componentsFilled[shape.label] = filled; diff --git a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts new file mode 100644 index 00000000000..a26e04539e7 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; + +import type { MainModelConfig } from './types'; + +import { getActiveRefiner, getRefinerDenoisingSplit, isRefinerModel } from './sdxlRefiner'; + +const sdxl: MainModelConfig = { base: 'sdxl', key: 'sdxl', name: 'SDXL', type: 'main' }; +const refiner: MainModelConfig = { base: 'sdxl-refiner', key: 'refiner', name: 'Refiner', type: 'main' }; + +describe('sdxlRefiner', () => { + it('activates a refiner only for a refiner-family model on an SDXL main', () => { + expect(getActiveRefiner({ refinerModel: refiner }, sdxl)).toBe(refiner); + expect(getActiveRefiner({ refinerModel: refiner }, { ...sdxl, base: 'sd-1' })).toBeNull(); + expect(getActiveRefiner({ refinerModel: sdxl }, sdxl)).toBeNull(); + expect(getActiveRefiner({ refinerModel: null }, sdxl)).toBeNull(); + expect(isRefinerModel({ base: 'sdxl-refiner', type: 'lora' })).toBe(false); + }); + + it('ends the base pass at the hand-over and never starts it later than that', () => { + expect(getRefinerDenoisingSplit(0.8)).toEqual({ denoising_end: 0.8, denoising_start: 0 }); + expect(getRefinerDenoisingSplit(0.8, 0.3)).toEqual({ denoising_end: 0.8, denoising_start: 0.3 }); + expect(getRefinerDenoisingSplit(0.8, 0.95)).toEqual({ denoising_end: 0.8, denoising_start: 0.8 }); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts new file mode 100644 index 00000000000..ecf7fd02cec --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts @@ -0,0 +1,104 @@ +/** SDXL Refiner policy: which models may fill the slot, its value ranges, and the denoise hand-over. */ + +import type { GenerateModelConfig, GenerateSettings, MainModelConfig } from './types'; + +export const DEFAULT_REFINER_STEPS = 20; +export const DEFAULT_REFINER_CFG_SCALE = 7.5; +export const DEFAULT_REFINER_SCHEDULER = 'euler'; +export const DEFAULT_REFINER_POSITIVE_AESTHETIC_SCORE = 6; +export const DEFAULT_REFINER_NEGATIVE_AESTHETIC_SCORE = 2.5; +export const DEFAULT_REFINER_START = 0.8; + +export const MIN_REFINER_STEPS = 1; +export const MIN_REFINER_CFG_SCALE = 1; +export const MIN_REFINER_AESTHETIC_SCORE = 1; +export const MAX_REFINER_AESTHETIC_SCORE = 10; +export const MIN_REFINER_START = 0; +export const MAX_REFINER_START = 1; + +export type RefinerSettings = Pick< + GenerateSettings, + | 'refinerModel' + | 'refinerSteps' + | 'refinerCfgScale' + | 'refinerScheduler' + | 'refinerPositiveAestheticScore' + | 'refinerNegativeAestheticScore' + | 'refinerStart' +>; + +/** The refiner is its own model family; only it may fill the slot, and never the main-model picker. */ +export const isRefinerModel = (model: { base: string; type: string } | null | undefined): boolean => + model?.type === 'main' && model.base === 'sdxl-refiner'; + +/** Only the SDXL base hands its latents to a refiner. */ +export const isRefinerSupported = (model: GenerateModelConfig | undefined): boolean => + model?.type === 'main' && model.base === 'sdxl'; + +/** The refiner that will run: a refiner model selected on an SDXL main model. */ +export const getActiveRefiner = ( + settings: Pick, + model: GenerateModelConfig | undefined +): MainModelConfig | null => + isRefinerSupported(model) && isRefinerModel(settings.refinerModel) ? settings.refinerModel : null; + +/** + * The base model's denoise range when a refiner finishes the schedule. An image-to-image start past the hand-over + * would leave the base model nothing to do, so the start is capped at the hand-over point (legacy parity). + */ +export const getRefinerDenoisingSplit = ( + refinerStart: number, + denoisingStart = 0 +): { denoising_start: number; denoising_end: number } => ({ + denoising_start: Math.min(refinerStart, denoisingStart), + denoising_end: refinerStart, +}); + +export const getRefinerValidationReasons = ( + model: GenerateModelConfig, + settings: RefinerSettings & Pick +): string[] => { + if (!settings.refinerModel || !isRefinerSupported(model)) { + return []; + } + + if (!isRefinerModel(settings.refinerModel)) { + return [`${settings.refinerModel.name} is not an SDXL Refiner model. Pick a refiner or clear the slot.`]; + } + + const reasons: string[] = []; + + // PiD replaces the VAE decode the refiner's second denoise stage feeds; the two cannot share a graph. + if (settings.pidMode !== 'off') { + reasons.push('PiD decode is not compatible with the SDXL Refiner. Turn PiD off or clear the refiner model.'); + } + + if (!Number.isInteger(settings.refinerSteps) || settings.refinerSteps < MIN_REFINER_STEPS) { + reasons.push(`Refiner steps must be a whole number of at least ${MIN_REFINER_STEPS}.`); + } + + if (!Number.isFinite(settings.refinerCfgScale) || settings.refinerCfgScale < MIN_REFINER_CFG_SCALE) { + reasons.push(`Refiner CFG must be at least ${MIN_REFINER_CFG_SCALE}.`); + } + + if ( + !Number.isFinite(settings.refinerStart) || + settings.refinerStart < MIN_REFINER_START || + settings.refinerStart > MAX_REFINER_START + ) { + reasons.push(`Refiner start must be between ${MIN_REFINER_START} and ${MAX_REFINER_START}.`); + } + + for (const [label, value] of [ + ['positive', settings.refinerPositiveAestheticScore], + ['negative', settings.refinerNegativeAestheticScore], + ] as const) { + if (!Number.isFinite(value) || value < MIN_REFINER_AESTHETIC_SCORE || value > MAX_REFINER_AESTHETIC_SCORE) { + reasons.push( + `Refiner ${label} aesthetic score must be between ${MIN_REFINER_AESTHETIC_SCORE} and ${MAX_REFINER_AESTHETIC_SCORE}.` + ); + } + } + + return reasons; +}; diff --git a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts new file mode 100644 index 00000000000..7e34c27b88f --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts @@ -0,0 +1,109 @@ +/** The SDXL Refiner's second denoise stage, grafted between the base denoise and the VAE decode. */ + +import type { BackendGraphContract, BackendInvocationContract } from '@features/generation/core/contracts'; + +import type { GenerateSettings, MainModelConfig } from './types'; + +import { coerceSchedulerForGraph } from './baseGenerationPolicies'; +import { addEdge, addNode, toModelIdentifier } from './graphBuilder'; +import { getRefinerDenoisingSplit } from './sdxlRefiner'; + +export const REFINER_DENOISE_NODE_ID = 'refiner_denoise_latents'; + +interface RefinerStageInput { + graph: BackendGraphContract; + settings: GenerateSettings; + /** The SDXL main model; the refiner shares its scheduler set. */ + model: MainModelConfig; + refiner: MainModelConfig; + /** The base denoise, whose latents the refiner finishes. Its `denoising_end` becomes the hand-over point. */ + denoise: BackendInvocationContract; + positivePrompt: BackendInvocationContract; + negativePrompt: BackendInvocationContract; + /** Set when the base graph tiles seamlessly; the refiner UNet gets the same wrapping. */ + seamless: BackendInvocationContract | null; +} + +/** Returns the refiner denoise, which replaces the base denoise as the decode's latents source. */ +export const addSDXLRefinerStage = ({ + denoise, + graph, + model, + negativePrompt, + positivePrompt, + refiner, + seamless, + settings, +}: RefinerStageInput): BackendInvocationContract => { + const scheduler = coerceSchedulerForGraph(model, settings.refinerScheduler); + const split = getRefinerDenoisingSplit(settings.refinerStart, denoise.denoising_start as number); + + denoise.denoising_start = split.denoising_start; + denoise.denoising_end = split.denoising_end; + + const modelLoader = addNode(graph, { + id: 'refiner_model_loader', + model: toModelIdentifier(refiner), + type: 'sdxl_refiner_model_loader', + }); + const posCond = addNode(graph, { + aesthetic_score: settings.refinerPositiveAestheticScore, + id: 'refiner_pos_cond', + type: 'sdxl_refiner_compel_prompt', + }); + const negCond = addNode(graph, { + aesthetic_score: settings.refinerNegativeAestheticScore, + id: 'refiner_neg_cond', + type: 'sdxl_refiner_compel_prompt', + }); + const refinerDenoise = addNode(graph, { + cfg_scale: settings.refinerCfgScale, + denoising_end: 1, + denoising_start: settings.refinerStart, + id: REFINER_DENOISE_NODE_ID, + // HiDiffusion is a UNet-level patch, so the refiner pass runs with the same settings as the base pass. + hidiffusion: denoise.hidiffusion, + hidiffusion_raunet: denoise.hidiffusion_raunet, + hidiffusion_window_attn: denoise.hidiffusion_window_attn, + hidiffusion_t1_ratio: denoise.hidiffusion_t1_ratio, + hidiffusion_t2_ratio: denoise.hidiffusion_t2_ratio, + scheduler, + steps: settings.refinerSteps, + type: 'denoise_latents', + }); + + if (seamless) { + const refinerSeamless = addNode(graph, { + id: 'refiner_seamless', + seamless_x: seamless.seamless_x, + seamless_y: seamless.seamless_y, + type: 'seamless', + }); + + addEdge(graph, modelLoader, 'unet', refinerSeamless, 'unet'); + addEdge(graph, refinerSeamless, 'unet', refinerDenoise, 'unet'); + } else { + addEdge(graph, modelLoader, 'unet', refinerDenoise, 'unet'); + } + + // The refiner only has the second text encoder; the prompts condition it as style text. + addEdge(graph, modelLoader, 'clip2', posCond, 'clip2'); + addEdge(graph, modelLoader, 'clip2', negCond, 'clip2'); + addEdge(graph, positivePrompt, 'value', posCond, 'style'); + addEdge(graph, negativePrompt, 'value', negCond, 'style'); + addEdge(graph, posCond, 'conditioning', refinerDenoise, 'positive_conditioning'); + addEdge(graph, negCond, 'conditioning', refinerDenoise, 'negative_conditioning'); + addEdge(graph, denoise, 'latents', refinerDenoise, 'latents'); + + return refinerDenoise; +}; + +export const getRefinerMetadata = (settings: GenerateSettings, model: MainModelConfig, refiner: MainModelConfig) => ({ + refiner_cfg_scale: settings.refinerCfgScale, + refiner_model: toModelIdentifier(refiner), + refiner_negative_aesthetic_score: settings.refinerNegativeAestheticScore, + refiner_positive_aesthetic_score: settings.refinerPositiveAestheticScore, + refiner_scheduler: coerceSchedulerForGraph(model, settings.refinerScheduler), + refiner_start: settings.refinerStart, + refiner_steps: settings.refinerSteps, +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts index bd621bc83ae..d835110974f 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts @@ -611,3 +611,66 @@ describe('planSeedSubmission', () => { ).toMatchObject({ lastSeed: 45, nextSeed: 46, sequenceLength: 4 }); }); }); + +describe('normalizeGenerateSettings refiner fields', () => { + const refiner: MainModelConfig = { base: 'sdxl-refiner', key: 'refiner', name: 'SDXL Refiner', type: 'main' }; + + it('defaults values saved before the refiner existed', () => { + expect(normalizeGenerateSettings(legacyStoredValues)).toMatchObject({ + refinerCfgScale: 7.5, + refinerModel: null, + refinerNegativeAestheticScore: 2.5, + refinerPositiveAestheticScore: 6, + refinerScheduler: 'euler', + refinerStart: 0.8, + refinerSteps: 20, + }); + }); + + it('keeps a refiner-family model and drops any other model stored in the slot', () => { + const stored = { ...legacyStoredValues, refinerModel: refiner }; + + expect(normalizeGenerateSettings(stored)?.refinerModel).toEqual(refiner); + expect( + normalizeGenerateSettings({ + ...legacyStoredValues, + refinerModel: { base: 'sdxl', key: 'sdxl', name: 'SDXL', type: 'main' }, + })?.refinerModel + ).toBeNull(); + expect( + normalizeGenerateSettings({ ...legacyStoredValues, refinerModel: { key: 'refiner' } })?.refinerModel + ).toBeNull(); + }); + + it('clamps the refiner tuning to its ranges and rounds steps', () => { + const normalized = normalizeGenerateSettings({ + ...legacyStoredValues, + refinerCfgScale: 0, + refinerNegativeAestheticScore: -3, + refinerPositiveAestheticScore: 42, + refinerScheduler: 7, + refinerStart: 1.5, + refinerSteps: 2.6, + }); + + expect(normalized).toMatchObject({ + refinerCfgScale: 1, + refinerNegativeAestheticScore: 1, + refinerPositiveAestheticScore: 10, + refinerScheduler: 'euler', + refinerStart: 1, + refinerSteps: 3, + }); + expect(normalized && isGenerateSettings(normalized)).toBe(true); + expect(isGenerateSettings({ ...normalized, refinerStart: 2 })).toBe(false); + }); + + it('refreshes a stale refiner snapshot from the installed catalog', () => { + const sdxl: MainModelConfig = { base: 'sdxl', key: 'legacy-model', name: 'SDXL', type: 'main' }; + const values = { ...normalizeGenerateSettings(legacyStoredValues)!, model: sdxl, refinerModel: refiner }; + const renamed = { ...refiner, name: 'SDXL Refiner 1.0' }; + + expect(syncGenerateWidgetValuesWithModels(values, [sdxl, renamed]).refinerModel).toEqual(renamed); + expect(syncGenerateWidgetValuesWithModels(values, [sdxl, refiner])).toBe(values); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.ts index 4b5cf3c4809..d8d630e83af 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.ts @@ -31,6 +31,21 @@ import { isDynamicPromptsSeedBehaviour, sanitizeMaxPrompts, sanitizeSampleSeed } import { clampPidSteps, DEFAULT_PID_STEPS, isPidMode } from './pid'; import { isCanonicalPromptTemplateSnapshot, sanitizePromptTemplateSnapshot } from './promptTemplates'; import { cloneCroppableImage, normalizeCroppableImage } from './referenceImage'; +import { + DEFAULT_REFINER_CFG_SCALE, + DEFAULT_REFINER_NEGATIVE_AESTHETIC_SCORE, + DEFAULT_REFINER_POSITIVE_AESTHETIC_SCORE, + DEFAULT_REFINER_SCHEDULER, + DEFAULT_REFINER_START, + DEFAULT_REFINER_STEPS, + isRefinerModel, + MAX_REFINER_AESTHETIC_SCORE, + MAX_REFINER_START, + MIN_REFINER_AESTHETIC_SCORE, + MIN_REFINER_CFG_SCALE, + MIN_REFINER_START, + MIN_REFINER_STEPS, +} from './sdxlRefiner'; /** Preset ratios, with the preset to switch to when dimensions are swapped. */ export const ASPECT_RATIO_MAP: Record, { ratio: number; inverseId: AspectRatioId }> = { @@ -190,6 +205,10 @@ const getModelIdentifierOrNull = (value: unknown): ComponentModelConfig | null = const getMainModelOrNull = (value: unknown): MainModelConfig | null => (isMainModelConfig(value) ? value : null); +/** Only a refiner-family model may persist in the slot; anything else stored there is dropped. */ +const getRefinerModelOrNull = (value: unknown): MainModelConfig | null => + isMainModelConfig(value) && isRefinerModel(value) ? value : null; + const isVaePrecision = (value: unknown): value is VaePrecision => value === 'fp16' || value === 'fp32'; /** Backend defaults from `Ideogram4DenoiseInvocation` / `Krea2*Invocation`. */ @@ -475,6 +494,7 @@ export const cloneGenerateWidgetValues = ( wanT5EncoderModel: values.wanT5EncoderModel ? { ...values.wanT5EncoderModel } : null, wanLowNoiseModel: values.wanLowNoiseModel ? { ...values.wanLowNoiseModel } : null, ideogram4UnconditionalModel: values.ideogram4UnconditionalModel ? { ...values.ideogram4UnconditionalModel } : null, + refinerModel: values.refinerModel ? { ...values.refinerModel } : null, ideogram4ColorPalette: [...values.ideogram4ColorPalette], referenceImages: cloneReferenceImages(values.referenceImages), t5EncoderModel: values.t5EncoderModel ? { ...values.t5EncoderModel } : null, @@ -550,6 +570,7 @@ export const syncGenerateWidgetValuesWithModels = ( : values.model; const vae = syncModelIdentifierWithModels(values.vae, modelsByKey); const componentSourceModel = syncModelIdentifierWithModels(values.componentSourceModel, modelsByKey); + const refinerModel = syncModelIdentifierWithModels(values.refinerModel, modelsByKey); const nextValues: GenerateWidgetValues = { ...values, clipEmbedModel: syncModelIdentifierWithModels(values.clipEmbedModel, modelsByKey), @@ -564,6 +585,7 @@ export const syncGenerateWidgetValuesWithModels = ( qwenVLEncoderModel: syncModelIdentifierWithModels(values.qwenVLEncoderModel, modelsByKey), qwen35EncoderModel: syncModelIdentifierWithModels(values.qwen35EncoderModel, modelsByKey), referenceImages: syncReferenceImagesWithModels(values.referenceImages, models), + refinerModel: isMainModelConfig(refinerModel) ? refinerModel : values.refinerModel, t5EncoderModel: syncModelIdentifierWithModels(values.t5EncoderModel, modelsByKey), vae: isVaeModelConfig(vae) ? vae : values.vae, }; @@ -581,6 +603,7 @@ export const syncGenerateWidgetValuesWithModels = ( nextValues.qwen35EncoderModel === values.qwen35EncoderModel && nextValues.referenceImages === values.referenceImages && nextValues.componentSourceModel === values.componentSourceModel && + nextValues.refinerModel === values.refinerModel && nextValues.modelKey === values.modelKey ? values : nextValues; @@ -794,6 +817,29 @@ export const normalizeGenerateSettings = (values: unknown): GenerateSettings | n pidDecoderModel: getModelIdentifierOrNull(values.pidDecoderModel), gemma2EncoderModel: getModelIdentifierOrNull(values.gemma2EncoderModel), pidSteps: clampPidSteps(hasFiniteNumber(values, 'pidSteps') ? (values.pidSteps as number) : DEFAULT_PID_STEPS), + refinerModel: getRefinerModelOrNull(values.refinerModel), + refinerSteps: hasFiniteNumber(values, 'refinerSteps') + ? Math.max(MIN_REFINER_STEPS, Math.round(values.refinerSteps as number)) + : DEFAULT_REFINER_STEPS, + refinerCfgScale: hasFiniteNumber(values, 'refinerCfgScale') + ? Math.max(MIN_REFINER_CFG_SCALE, values.refinerCfgScale as number) + : DEFAULT_REFINER_CFG_SCALE, + refinerScheduler: typeof values.refinerScheduler === 'string' ? values.refinerScheduler : DEFAULT_REFINER_SCHEDULER, + refinerPositiveAestheticScore: getClampedNumber( + values, + 'refinerPositiveAestheticScore', + MIN_REFINER_AESTHETIC_SCORE, + MAX_REFINER_AESTHETIC_SCORE, + DEFAULT_REFINER_POSITIVE_AESTHETIC_SCORE + ), + refinerNegativeAestheticScore: getClampedNumber( + values, + 'refinerNegativeAestheticScore', + MIN_REFINER_AESTHETIC_SCORE, + MAX_REFINER_AESTHETIC_SCORE, + DEFAULT_REFINER_NEGATIVE_AESTHETIC_SCORE + ), + refinerStart: getClampedNumber(values, 'refinerStart', MIN_REFINER_START, MAX_REFINER_START, DEFAULT_REFINER_START), width, }; }; diff --git a/invokeai/frontend/webv2/src/features/generation/core/types.ts b/invokeai/frontend/webv2/src/features/generation/core/types.ts index d32d37543de..fac0e15e889 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/types.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/types.ts @@ -269,6 +269,16 @@ export interface GenerateSettings { /** The shared Gemma-2 caption encoder PiD conditions its decode on. */ gemma2EncoderModel: ComponentModelConfig | null; pidSteps: number; + /** SDXL only: a refiner-family main model that finishes the schedule from `refinerStart`; null runs none. */ + refinerModel: MainModelConfig | null; + refinerSteps: number; + refinerCfgScale: number; + refinerScheduler: string; + /** Aesthetic-score conditioning for the refiner's positive and negative prompts, 1 to 10. */ + refinerPositiveAestheticScore: number; + refinerNegativeAestheticScore: number; + /** Fraction of the schedule at which the base model hands its latents to the refiner. */ + refinerStart: number; } /** diff --git a/invokeai/frontend/webv2/src/features/generation/settings.ts b/invokeai/frontend/webv2/src/features/generation/settings.ts index 8561bbc1a9b..6d5183850b8 100644 --- a/invokeai/frontend/webv2/src/features/generation/settings.ts +++ b/invokeai/frontend/webv2/src/features/generation/settings.ts @@ -71,6 +71,17 @@ export { type RebalancePreset, } from './core/conditioningRebalance'; export { MIN_BATCH_COUNT, sanitizeBatchCount } from './core/batch'; +export { + getActiveRefiner, + isRefinerModel, + isRefinerSupported, + MAX_REFINER_AESTHETIC_SCORE, + MAX_REFINER_START, + MIN_REFINER_AESTHETIC_SCORE, + MIN_REFINER_CFG_SCALE, + MIN_REFINER_START, + MIN_REFINER_STEPS, +} from './core/sdxlRefiner'; export { createDynamicPromptsSampleSeed, DYNAMIC_PROMPTS_DEFAULT_MAX_PROMPTS, diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx new file mode 100644 index 00000000000..9674d3a9adf --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx @@ -0,0 +1,172 @@ +/* oxlint-disable react-perf/jsx-no-new-object-as-prop, react-perf/jsx-no-new-array-as-prop */ +import type { GenerationModelCatalogItem } from '@features/generation/contracts'; +import type { MainModelConfig } from '@features/generation/core/types'; + +import { ChakraProvider } from '@chakra-ui/react'; +import { getDefaultGenerateSettings } from '@features/generation/core/baseGenerationPolicies'; +import { createExternalStoreCore } from '@platform/state/externalStoreCore'; +import { system } from '@theme/system'; +import { createInstance } from 'i18next'; +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { I18nextProvider, initReactI18next } from 'react-i18next'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import type { GenerationModelSelectProps } from './GenerationUiContext'; + +import { GenerateRefinerSection } from './GenerateRefinerSection'; + +const sdxlModel: MainModelConfig = { base: 'sdxl', key: 'sdxl', name: 'SDXL', type: 'main' }; +const fluxModel: MainModelConfig = { base: 'flux', key: 'flux', name: 'FLUX.1 dev', type: 'main' }; +const refinerModel: MainModelConfig = { base: 'sdxl-refiner', key: 'refiner', name: 'SDXL Refiner', type: 'main' }; +const CATALOG: readonly GenerationModelCatalogItem[] = [sdxlModel, refinerModel]; + +vi.mock('./GenerationUiContext', () => ({ + /** The picker reduced to one button per catalog model that passes the section's filter. */ + GenerationModelSelect: ({ filter, onChange, value }: GenerationModelSelectProps) => ( +
+ {CATALOG.filter((model) => filter?.(model) ?? true).map((model) => ( + + ))} +
+ ), + useGenerationUi: () => ({ + sectionPreferences: { sectionsOpen: { refiner: true }, setSectionOpen: vi.fn() }, + }), +})); + +const i18n = createInstance(); +void i18n.use(initReactI18next).init({ + fallbackLng: 'en', + initAsync: false, + lng: 'en', + resources: { + en: { + translation: { + widgets: { + generate: { + noRefinerModels: 'No SDXL Refiner models are installed.', + refiner: 'Refiner', + refinerCfgScale: 'Refiner CFG', + refinerModel: 'Refiner model', + refinerNegativeAestheticScore: 'Negative aesthetic score', + refinerNone: 'None', + refinerPositiveAestheticScore: 'Positive aesthetic score', + refinerScheduler: 'Refiner scheduler', + refinerStart: 'Refiner start', + refinerSteps: 'Refiner steps', + }, + }, + }, + }, + }, +}); + +let host: HTMLDivElement | null = null; +let root: Root | null = null; +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const settle = (action: () => void): Promise => + act(async () => { + action(); + await new Promise((resolve) => { + globalThis.setTimeout(resolve, 50); + }); + }); + +const render = async ( + model: MainModelConfig, + options: { models?: readonly GenerationModelCatalogItem[]; refinerModel?: MainModelConfig | null } = {} +) => { + host = document.createElement('div'); + document.body.append(host); + root = createRoot(host); + const settings = { ...getDefaultGenerateSettings(model), refinerModel: options.refinerModel ?? null }; + const onCommit = vi.fn(); + const onCommitImmediate = vi.fn(); + + await settle(() => { + root?.render( + + + + + + ); + }); + + return { onCommit, onCommitImmediate }; +}; + +/** Sliders are named by `aria-labelledby`, so resolve them through their label element. */ +const sliderByLabel = (label: string): HTMLElement | null => + [...(host?.querySelectorAll('[role="slider"]') ?? [])].find((control) => { + const labelId = control.getAttribute('aria-labelledby'); + + return labelId !== null && host?.querySelector(`#${CSS.escape(labelId)}`)?.textContent === label; + }) ?? null; + +const pickerButton = (name: string): HTMLButtonElement | null => + [...(host?.querySelectorAll('[data-testid="refiner-picker"] button') ?? [])].find( + (button) => button.textContent === name + ) ?? null; + +afterEach(async () => { + await settle(() => root?.unmount()); + host?.remove(); + host = null; + root = null; +}); + +describe('GenerateRefinerSection', () => { + it('renders for SDXL only', async () => { + await render(fluxModel); + + expect(host?.textContent).toBe(''); + + await settle(() => root?.unmount()); + host?.remove(); + + await render(sdxlModel); + + expect(host?.textContent).toContain('Refiner'); + expect(host?.querySelector('[data-testid="refiner-picker"]')).not.toBeNull(); + }); + + it('offers only refiner models and keeps the tuning disabled until one is picked', async () => { + const { onCommitImmediate } = await render(sdxlModel); + + expect(pickerButton('SDXL')).toBeNull(); + expect(pickerButton('SDXL Refiner')).not.toBeNull(); + expect(sliderByLabel('Refiner start')?.getAttribute('aria-disabled')).toBe('true'); + expect(sliderByLabel('Refiner steps')?.getAttribute('aria-disabled')).toBe('true'); + + await settle(() => pickerButton('SDXL Refiner')?.click()); + + expect(onCommitImmediate).toHaveBeenCalledWith({ refinerModel: refinerModel }); + }); + + it('enables the tuning once a refiner is selected and shows it in the header', async () => { + await render(sdxlModel, { refinerModel }); + + expect(host?.textContent).toContain('SDXL Refiner'); + expect(host?.querySelector('[data-testid="refiner-picker"]')?.getAttribute('data-value')).toBe('refiner'); + expect(sliderByLabel('Refiner start')?.getAttribute('aria-disabled')).not.toBe('true'); + expect(sliderByLabel('Positive aesthetic score')?.getAttribute('aria-valuenow')).toBe('6'); + expect(sliderByLabel('Negative aesthetic score')?.getAttribute('aria-valuenow')).toBe('2.5'); + }); + + it('explains an empty picker when no refiner is installed', async () => { + await render(sdxlModel, { models: [sdxlModel] }); + + expect(host?.textContent).toContain('No SDXL Refiner models are installed.'); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.tsx new file mode 100644 index 00000000000..131627398d4 --- /dev/null +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.tsx @@ -0,0 +1,226 @@ +/* oxlint-disable react-perf/jsx-no-new-function-as-prop, react-perf/jsx-no-jsx-as-prop */ +import type { GenerationModelCatalogItem as ModelConfig } from '@features/generation/contracts'; +import type { GenerateModelConfig, GenerateSettings } from '@features/generation/core/types'; + +import { Badge, Stack } from '@chakra-ui/react'; +import { getGenerationUiPolicy, getSchedulerOptions } from '@features/generation/core/baseGenerationPolicies'; +import { + DEFAULT_REFINER_CFG_SCALE, + DEFAULT_REFINER_NEGATIVE_AESTHETIC_SCORE, + DEFAULT_REFINER_POSITIVE_AESTHETIC_SCORE, + DEFAULT_REFINER_SCHEDULER, + DEFAULT_REFINER_START, + DEFAULT_REFINER_STEPS, + isRefinerModel, + MAX_REFINER_AESTHETIC_SCORE, + MAX_REFINER_START, + MIN_REFINER_AESTHETIC_SCORE, + MIN_REFINER_CFG_SCALE, + MIN_REFINER_START, + MIN_REFINER_STEPS, +} from '@features/generation/core/sdxlRefiner'; +import { isMainModelConfig } from '@features/generation/core/settings'; +import { useExternalStoreSelector } from '@platform/state/selectors'; +import { Combobox } from '@platform/ui/Combobox'; +import { Field } from '@platform/ui/Field'; +import { ScrubberField } from '@platform/ui/ScrubberField'; +import { useCallback } from 'react'; +import { useTranslation } from 'react-i18next'; + +import { type GenerateDraft, pickGenerateSettings } from './generateDebounce'; +import { GenerationModelSelect as ModelSelect } from './GenerationUiContext'; +import { GenerateCollapsibleSection } from './shared/GenerateCollapsibleSection'; +import { GenerateFieldContextMenu } from './shared/GenerateFieldContextMenu'; + +/** Practical track ranges; typed values may exceed them within validation bounds. */ +const STEPS_SLIDER_MAX = 100; +const CFG_SLIDER_MAX = 20; +const CFG_INPUT_MAX = 100; + +interface GenerateRefinerSectionProps { + draft: GenerateDraft; + models: readonly ModelConfig[]; + selectedModel: GenerateModelConfig | undefined; + onCommit: (patch: Partial) => void; + onCommitImmediate: (patch: Partial) => void; +} + +const selectRefinerSettings = pickGenerateSettings([ + 'refinerCfgScale', + 'refinerModel', + 'refinerNegativeAestheticScore', + 'refinerPositiveAestheticScore', + 'refinerScheduler', + 'refinerStart', + 'refinerSteps', +]); + +type RefinerNumberKey = + | 'refinerCfgScale' + | 'refinerNegativeAestheticScore' + | 'refinerPositiveAestheticScore' + | 'refinerStart' + | 'refinerSteps'; + +const refinerFilter = (model: ModelConfig) => isRefinerModel(model); +const MAIN_MODEL_TYPES = ['main']; + +export const GenerateRefinerSection = ({ + draft, + models, + onCommit, + onCommitImmediate, + selectedModel, +}: GenerateRefinerSectionProps) => { + const { t } = useTranslation(); + const settings = useExternalStoreSelector(draft.subscribe, draft.getSnapshot, selectRefinerSettings); + const commitRefinerModel = useCallback( + (model: ModelConfig | null) => { + onCommitImmediate({ refinerModel: isMainModelConfig(model) && isRefinerModel(model) ? model : null }); + }, + [onCommitImmediate] + ); + + if (!getGenerationUiPolicy(selectedModel).refinerVisible) { + return null; + } + + const hasInstalledRefiner = models.some(refinerFilter); + const refiner = settings.refinerModel; + const isActive = refiner !== null; + + const commitNumber = (key: RefinerNumberKey, min: number, max: number) => (value: number) => { + if (!Number.isFinite(value)) { + return; + } + + onCommit({ [key]: Math.min(max, Math.max(min, value)) }); + }; + + const numberField = ({ + defaultValue, + hint, + inputMax, + key, + label, + max, + min, + step, + }: { + defaultValue: number; + hint: 'refinerSteps' | 'refinerCfgScale' | 'refinerStart' | 'refinerAestheticScore'; + inputMax?: number; + key: RefinerNumberKey; + label: string; + max: number; + min: number; + step: number; + }) => ( + String(settings[key])} + isAtDefault={!isActive || settings[key] === defaultValue} + onReset={() => onCommit({ [key]: defaultValue })} + > + + + ); + + return ( + {refiner.name} : null} + sectionId="refiner" + > + + + + + {numberField({ + defaultValue: DEFAULT_REFINER_STEPS, + hint: 'refinerSteps', + inputMax: Number.MAX_SAFE_INTEGER, + key: 'refinerSteps', + label: t('widgets.generate.refinerSteps'), + max: STEPS_SLIDER_MAX, + min: MIN_REFINER_STEPS, + step: 1, + })} + {numberField({ + defaultValue: DEFAULT_REFINER_CFG_SCALE, + hint: 'refinerCfgScale', + inputMax: CFG_INPUT_MAX, + key: 'refinerCfgScale', + label: t('widgets.generate.refinerCfgScale'), + max: CFG_SLIDER_MAX, + min: MIN_REFINER_CFG_SCALE, + step: 0.5, + })} + settings.refinerScheduler} + isAtDefault={!isActive || settings.refinerScheduler === DEFAULT_REFINER_SCHEDULER} + onReset={() => onCommitImmediate({ refinerScheduler: DEFAULT_REFINER_SCHEDULER })} + > + + onCommitImmediate({ refinerScheduler })} + /> + + + {numberField({ + defaultValue: DEFAULT_REFINER_START, + hint: 'refinerStart', + key: 'refinerStart', + label: t('widgets.generate.refinerStart'), + max: MAX_REFINER_START, + min: MIN_REFINER_START, + step: 0.01, + })} + {numberField({ + defaultValue: DEFAULT_REFINER_POSITIVE_AESTHETIC_SCORE, + hint: 'refinerAestheticScore', + key: 'refinerPositiveAestheticScore', + label: t('widgets.generate.refinerPositiveAestheticScore'), + max: MAX_REFINER_AESTHETIC_SCORE, + min: MIN_REFINER_AESTHETIC_SCORE, + step: 0.5, + })} + {numberField({ + defaultValue: DEFAULT_REFINER_NEGATIVE_AESTHETIC_SCORE, + hint: 'refinerAestheticScore', + key: 'refinerNegativeAestheticScore', + label: t('widgets.generate.refinerNegativeAestheticScore'), + max: MAX_REFINER_AESTHETIC_SCORE, + min: MIN_REFINER_AESTHETIC_SCORE, + step: 0.5, + })} + + + ); +}; diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx index 86d79bbe0c8..6c2e9ba8931 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateSettingsForm.tsx @@ -22,6 +22,7 @@ import { useRegisterGenerateDraftFlusher } from './generateDraftRegistry'; import { getSettingsWithLatestPromptFields } from './generateFormViewModel'; import { GenerateGuidanceSection } from './GenerateGuidanceSection'; import { GenerateModelCard } from './GenerateModelCard'; +import { GenerateRefinerSection } from './GenerateRefinerSection'; import { GenerateRenderSection } from './GenerateRenderSection'; import { GeneratePromptFields } from './promptFields'; @@ -280,6 +281,14 @@ export const GenerateSettingsForm = ({ + + = {}): GenerateWi pidDecoderModel: null, gemma2EncoderModel: null, pidSteps: 4, + refinerModel: null, + refinerSteps: 20, + refinerCfgScale: 7.5, + refinerScheduler: 'euler', + refinerPositiveAestheticScore: 6, + refinerNegativeAestheticScore: 2.5, + refinerStart: 0.8, cfgRescaleMultiplier: 0, cfgScale: 7, clipEmbedModel: null, @@ -1233,3 +1240,164 @@ describe('remix round trip', () => { ]); }); }); + +describe('SDXL Refiner recall', () => { + const refiner: MainModelConfig = { + base: 'sdxl-refiner', + hash: 'blake3:refiner', + key: 'refiner-installed', + name: 'SDXL Refiner', + type: 'main', + }; + const recordedRefiner = { + refiner_cfg_scale: 5, + refiner_model: { base: 'sdxl-refiner', hash: 'blake3:refiner', key: 'refiner-recorded', name: 'SDXL Refiner' }, + refiner_negative_aesthetic_score: 2, + refiner_positive_aesthetic_score: 7, + refiner_scheduler: 'dpmpp_2m', + refiner_start: 0.7, + refiner_steps: 12, + }; + const recall = (metadata: Record, models: ComponentModelConfig[], kind: ImageRecallKind = 'all') => + buildImageRecallSettings({ + currentValues: createValues(), + image, + kind, + metadata: { model: { key: sdxlModel.key }, ...metadata }, + models, + supportedModels: [sdxlModel], + vaeModels: [], + }); + + it.each(['all', 'remix'] as const)('restores the refiner and its tuning with %s, resolving by hash', (kind) => { + const result = recall(recordedRefiner, [refiner], kind); + + expect(result?.values).toMatchObject({ + refinerCfgScale: 5, + refinerModel: refiner, + refinerNegativeAestheticScore: 2, + refinerPositiveAestheticScore: 7, + refinerScheduler: 'dpmpp_2m', + refinerStart: 0.7, + refinerSteps: 12, + }); + expect(result?.fields).toContain('refiner'); + }); + + it('resolves the refiner by key, then by name and base, and only when that is unambiguous', () => { + const byKey = { ...refiner, hash: 'blake3:other', key: 'refiner-recorded' }; + + expect(recall(recordedRefiner, [byKey, refiner])?.values.refinerModel).toEqual(byKey); + + const unhashed = { ...recordedRefiner, refiner_model: { base: 'sdxl-refiner', name: 'SDXL Refiner' } }; + const twin = { ...refiner, key: 'refiner-twin' }; + + expect(recall(unhashed, [refiner])?.values.refinerModel).toEqual(refiner); + expect(recall(unhashed, [refiner, twin])?.values.refinerModel).toBeNull(); + }); + + it('keeps the current refiner when the recorded one is not installed, but still restores the tuning', () => { + const current = createValues({ refinerModel: refiner }); + const result = buildImageRecallSettings({ + currentValues: current, + image, + kind: 'all', + metadata: { model: { key: sdxlModel.key }, ...recordedRefiner }, + models: [{ ...refiner, key: 'another-refiner', hash: 'blake3:another', name: 'Other Refiner' }], + supportedModels: [sdxlModel], + vaeModels: [], + }); + + expect(result?.values.refinerModel).toEqual(refiner); + expect(result?.values.refinerStart).toBe(0.7); + }); + + it('clears the refiner for a generation record that ran without one and ignores refiner data on other families', () => { + const cleared = buildImageRecallSettings({ + currentValues: createValues({ refinerModel: refiner }), + image, + kind: 'all', + metadata: { generation_mode: 'sdxl_txt2img', model: { key: sdxlModel.key } }, + models: [refiner], + supportedModels: [sdxlModel], + vaeModels: [], + }); + + expect(cleared?.values.refinerModel).toBeNull(); + expect(cleared?.fields).toContain('refiner'); + + // A canvas save names a model but is not a generation record, so it says nothing about the refiner. + const canvasSave = buildImageRecallSettings({ + currentValues: createValues({ refinerModel: refiner }), + image, + kind: 'all', + metadata: { model: { key: sdxlModel.key }, positive_prompt: 'kept' }, + models: [refiner], + supportedModels: [sdxlModel], + vaeModels: [], + }); + + expect(canvasSave?.values.refinerModel).toEqual(refiner); + expect(canvasSave?.fields).not.toContain('refiner'); + + const sd1 = buildImageRecallSettings({ + currentValues: createValues(), + image, + kind: 'all', + metadata: { model: { key: sd1Model.key }, ...recordedRefiner }, + models: [sd1Model, refiner], + supportedModels: [sd1Model], + vaeModels: [], + }); + + expect(sd1?.values.refinerModel).toBeNull(); + expect(sd1?.fields).not.toContain('refiner'); + }); + + it('drops out-of-range refiner tuning and unknown schedulers', () => { + const result = recall({ ...recordedRefiner, refiner_scheduler: 'bogus', refiner_start: 3, refiner_steps: 0 }, [ + refiner, + ]); + + expect(result?.values).toMatchObject({ + refinerModel: refiner, + refinerScheduler: 'euler', + refinerStart: 0.8, + refinerSteps: 20, + }); + }); + + it('round-trips the refiner a compiled SDXL graph records', () => { + const { model, settings } = satisfiedSettingsFor('sdxl', { + label: 'diffusers', + overrides: { format: 'diffusers' }, + }); + const graph = compileGenerateGraph({ ...settings, refinerModel: refiner, refinerStart: 0.65 }, model, 'gallery', { + useCpuNoise: true, + }).backendGraph; + const metadata = Object.values(graph.nodes).find((node) => node.type === 'core_metadata'); + const result = buildImageRecallSettings({ + currentValues: createValues({ model: sd1Model, modelKey: sd1Model.key }), + image, + kind: 'remix', + metadata, + models: [refiner], + supportedModels: [model], + vaeModels: [], + }); + + expect(result?.values.model).toEqual(model); + expect(result?.values.refinerModel).toEqual(refiner); + expect(result?.values.refinerStart).toBe(0.65); + expect( + getImageRecallCapabilities({ + currentValues: createValues(), + image, + metadata: { refiner_model: metadata?.refiner_model }, + models: [refiner], + supportedModels: [model], + vaeModels: [], + }) + ).toMatchObject({ all: true, remix: true }); + }); +}); diff --git a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts index 829d4119327..940c7c7196e 100644 --- a/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts +++ b/invokeai/frontend/webv2/src/workbench/image-actions/imageRecall.ts @@ -28,9 +28,16 @@ import { isMainModelConfig, isModelIdentifierConfig, isVaeCompatibleWithGenerateModel, + isRefinerModel, isValidKrea2RebalanceWeights, MAX_HIDIFFUSION_RATIO, + MAX_REFINER_AESTHETIC_SCORE, + MAX_REFINER_START, MIN_HIDIFFUSION_T1_RATIO, + MIN_REFINER_AESTHETIC_SCORE, + MIN_REFINER_CFG_SCALE, + MIN_REFINER_START, + MIN_REFINER_STEPS, normalizeReferenceImages, } from '@features/generation/settings'; import { SEED_MAX } from '@platform/core/seed'; @@ -73,7 +80,8 @@ type RecalledField = | 'components' | 'loras' | 'referenceImages' - | 'krea2Rebalance'; + | 'krea2Rebalance' + | 'refiner'; export type ImageRecallSkipReason = | 'ambiguous' @@ -542,6 +550,103 @@ const resolveRecordedLora = ( return matches.length > 1 ? 'ambiguous' : (matches[0] ?? null); }; +/** Resolve a recorded main model by install-local key, then content hash, then name and base, like a LoRA. */ +const resolveRecordedMainModel = ( + ref: RecordedModelRef, + models: readonly ComponentModelConfig[] +): MainModelConfig | null => { + const mains = models.filter(isMainModelConfig); + const byKey = ref.key ? mains.find((model) => model.key === ref.key) : undefined; + + if (byKey) { + return byKey; + } + + const matches = ref.hash + ? mains.filter((model) => model.hash === ref.hash) + : ref.name && ref.base + ? mains.filter((model) => model.name === ref.name && model.base === ref.base) + : []; + + return matches.length === 1 ? matches[0]! : null; +}; + +type RefinerPatch = Partial< + Pick< + GenerateWidgetValues, + | 'refinerModel' + | 'refinerSteps' + | 'refinerCfgScale' + | 'refinerScheduler' + | 'refinerPositiveAestheticScore' + | 'refinerNegativeAestheticScore' + | 'refinerStart' + > +>; + +const getNumberInRange = (metadata: unknown, key: string, min: number, max: number): number | null => { + const value = getNumber(metadata, key); + + return value !== null && value >= min && value <= max ? value : null; +}; + +/** + * The refiner an SDXL image ran with, resolved against installed refiner models, plus its tuning. A refiner that is + * not installed leaves the slot as it is, like an uninstalled VAE. Generation records only name a refiner when one + * ran, so a stamped record without one clears the slot, as a missing `loras` list means no concepts. + */ +const getRefinerPatch = ( + metadata: unknown, + model: GenerateModelConfig, + models: readonly ComponentModelConfig[] +): RefinerPatch => { + if (!getGenerationUiPolicy(model).refinerVisible || !isRecord(metadata)) { + return {}; + } + + const patch: RefinerPatch = {}; + + if ('refiner_model' in metadata) { + const ref = toRecordedModelRef(metadata.refiner_model); + const refiner = ref ? resolveRecordedMainModel(ref, models) : null; + + if (metadata.refiner_model === null) { + patch.refinerModel = null; + } else if (refiner && isRefinerModel(refiner)) { + patch.refinerModel = refiner; + } + } else if (isGenerationRecord(metadata)) { + patch.refinerModel = null; + } + + const steps = getInteger(metadata, 'refiner_steps'); + const cfgScale = getNumber(metadata, 'refiner_cfg_scale'); + const scheduler = getString(metadata, 'refiner_scheduler'); + const start = getNumberInRange(metadata, 'refiner_start', MIN_REFINER_START, MAX_REFINER_START); + const positiveScore = getNumberInRange( + metadata, + 'refiner_positive_aesthetic_score', + MIN_REFINER_AESTHETIC_SCORE, + MAX_REFINER_AESTHETIC_SCORE + ); + const negativeScore = getNumberInRange( + metadata, + 'refiner_negative_aesthetic_score', + MIN_REFINER_AESTHETIC_SCORE, + MAX_REFINER_AESTHETIC_SCORE + ); + + return { + ...patch, + ...(steps !== null && steps >= MIN_REFINER_STEPS ? { refinerSteps: steps } : {}), + ...(cfgScale !== null && cfgScale >= MIN_REFINER_CFG_SCALE ? { refinerCfgScale: cfgScale } : {}), + ...(scheduler !== null && isKnownScheduler(scheduler) ? { refinerScheduler: scheduler } : {}), + ...(start !== null ? { refinerStart: start } : {}), + ...(positiveScore !== null ? { refinerPositiveAestheticScore: positiveScore } : {}), + ...(negativeScore !== null ? { refinerNegativeAestheticScore: negativeScore } : {}), + }; +}; + /** * Generation graphs stamp `generation_mode` and omit `loras` when no concept was active, so on such a record a * missing list means "none". Other records that name a model, such as canvas saves, never record concepts. @@ -707,6 +812,7 @@ export const getImageRecallCapabilities = ({ const hasReferenceImages = getMetadataReferenceImages(metadata).length > 0; const hasModel = supportedMetadataModel !== null; const hasRebalance = hasKrea2Rebalance(metadata, clipSkipModel); + const hasRefiner = Object.keys(getRefinerPatch(metadata, clipSkipModel, models)).length > 0; const hasConcepts = (getMetadataConcepts(metadata, models, clipSkipModel, supportedMetadataModel).loras?.length ?? 0) > 0; const hasAnyMetadata = @@ -721,7 +827,8 @@ export const getImageRecallCapabilities = ({ hasComponents || hasReferenceImages || hasConcepts || - hasRebalance; + hasRebalance || + hasRefiner; const hasNonSeedMetadata = hasModel || hasVae || @@ -733,7 +840,8 @@ export const getImageRecallCapabilities = ({ hasComponents || hasReferenceImages || hasConcepts || - hasRebalance; + hasRebalance || + hasRefiner; return { all: hasAnyMetadata && isImageRecallKindAvailable('all'), @@ -919,6 +1027,19 @@ export const buildImageRecallSettings = ({ values = { ...values, ...hiDiffusionPatch }; fields.push('hiDiffusion'); } + + // Judged against the model the recall settled on: a non-SDXL main has no refiner to restore. + const refinerPatch = getRefinerPatch(metadata, values.model, models); + const refinerChanged = (Object.keys(refinerPatch) as (keyof RefinerPatch)[]).some((key) => + key === 'refinerModel' + ? refinerPatch.refinerModel?.key !== values.refinerModel?.key + : refinerPatch[key] !== values[key] + ); + + if (refinerChanged) { + values = { ...values, ...refinerPatch }; + fields.push('refiner'); + } } if (kind === 'all' || kind === 'remix' || kind === 'clipSkip') { diff --git a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts index c9db763eea2..757078a518e 100644 --- a/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts +++ b/invokeai/frontend/webv2/src/workbench/workbenchState.test.ts @@ -203,6 +203,13 @@ const createGenerateValues = (overrides: Partial = {}): Ge pidDecoderModel: null, gemma2EncoderModel: null, pidSteps: 4, + refinerModel: null, + refinerSteps: 20, + refinerCfgScale: 7.5, + refinerScheduler: 'euler', + refinerPositiveAestheticScore: 6, + refinerNegativeAestheticScore: 2.5, + refinerStart: 0.8, cfgRescaleMultiplier: 0, cfgScale: 7, clipEmbedModel: null, From b37d93ddb03a619081699f7caba5dadc6f307781 Mon Sep 17 00:00:00 2001 From: Josh Corbett Date: Sun, 11 Oct 2026 02:23:31 -0600 Subject: [PATCH 2/5] fix(generation): tighten the SDXL Refiner hand-over, recall and picker states The base denoise range has one owner, so Canvas grafts keep the hand-over, and inpaint/outpaint feed the gradient mask to the refiner pass too. A start of 1 skips the refiner instead of loading it for an empty pass; recall reports an uninstalled or ambiguous refiner instead of silently keeping the current one. The refiner picker owns its empty and loading states, steps/CFG use the legacy input limits, and docs describe the Canvas strength cap. --- .../03.Models/Local Models/sdxl.mdx | 9 +- .../04.Image Generation/generate-panel.mdx | 8 +- .../frontend/webv2/public/locales/en.json | 10 +- .../core/canvas/compileCanvasGraph.test.ts | 48 +++++++- .../core/canvas/compileCanvasGraph.ts | 18 +-- .../features/generation/core/graph.test.ts | 61 ++++++++++ .../src/features/generation/core/graph.ts | 5 +- .../generation/core/sdxlRefiner.test.ts | 30 ++++- .../features/generation/core/sdxlRefiner.ts | 26 +++- .../generation/core/sdxlRefinerGraph.ts | 8 +- .../GenerateRefinerSection.browser.test.tsx | 29 +++-- .../generation/ui/GenerateRefinerSection.tsx | 18 ++- .../generation/ui/GenerateSettingsForm.tsx | 1 - .../image-actions/executeImageRecall.test.ts | 30 +++++ .../image-actions/executeImageRecall.ts | 21 +++- .../image-actions/imageRecall.test.ts | 44 ++++--- .../workbench/image-actions/imageRecall.ts | 113 ++++++++++-------- 17 files changed, 345 insertions(+), 134 deletions(-) diff --git a/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx b/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx index 353c9903e7e..3d89326a1c6 100644 --- a/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/Local Models/sdxl.mdx @@ -184,10 +184,11 @@ image on its own. - **Using it** — with an SDXL main model selected, the Generate panel shows a **Refiner** section. Pick the refiner there and set **Refiner start** (the point where the base model hands over, 0.8 by default), its steps, CFG, scheduler and the positive and negative **aesthetic scores**. The - same settings apply to SDXL generations from the Canvas, including inpainting and outpainting. The - refiner is never offered as a main model, and it cannot be combined with PiD decoding. An image - made with a refiner records it in its metadata, so **Recall All** and **Remix Image** restore it. - The workflow editor offers the same nodes for custom graphs. + same settings apply to SDXL generations from the Canvas, including inpainting and outpainting; there + the base pass cannot start after the hand-over, so the effective denoising strength is capped at + 1 − Refiner start. The refiner is never offered as a main model, and it cannot be combined with PiD + decoding. An image made with a refiner records it in its metadata, so **Recall All** and + **Remix Image** restore it. The workflow editor offers the same nodes for custom graphs. A refiner workflow runs two denoise passes over the same latents: diff --git a/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx b/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx index 53e0216e08f..bc31fc0c7a9 100644 --- a/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx +++ b/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx @@ -51,6 +51,10 @@ A negative prompt lists what you do *not* want in the image, such as *blurry, wa Not every model uses a negative prompt. SD 1.5, SDXL, SD 3.5 and CogView4 always use it. Several newer models, including Z-Image, Qwen Image and Krea-2, use it only when CFG is above 1, and the box says so. FLUX.1, FLUX.2 and Ideogram 4 do not use a negative prompt, and the box is not shown for them. +### Refiner + +The **Refiner** section appears for SDXL models. Pick an **SDXL Refiner** model to have it finish the last part of the denoising schedule; the base model stops at **Refiner start** (0.8 by default, so the refiner handles the final 20%) and the refiner continues from there with its own **Refiner steps**, **Refiner CFG** and **Refiner scheduler**. The **Positive** and **Negative aesthetic score** condition the refiner on the aesthetic rating it was trained with. A **Refiner start** of 1 leaves the refiner nothing to do, so it is not run. The controls are disabled until a refiner is selected, and refiner models are only offered here, never as the main model. The refiner also runs for SDXL generations from the Canvas. It cannot be combined with **PiD Decode**. See [SDXL](/users-guide/models/local-models/sdxl/#sdxl-refiner). + ### Advanced settings The **Advanced** section appears for models with extra options, and its header counts the settings you have changed. Depending on the model it can include: @@ -62,10 +66,6 @@ The **Advanced** section appears for models with extra options, and its header c * **PiD Decode** — decodes the image at up to four times its size; see [PiD Super-Resolution Decode](/users-guide/image-generation/advanced-topics/pid-decode/). * **Seamless tiling** — makes the image tile along the X axis, the Y axis, or both. -### Refiner - -The **Refiner** section appears for SDXL models. Pick an **SDXL Refiner** model to have it finish the last part of the denoising schedule; the base model stops at **Refiner start** (0.8 by default, so the refiner handles the final 20%) and the refiner continues from there with its own **Refiner steps**, **Refiner CFG** and **Refiner scheduler**. The **Positive** and **Negative aesthetic score** condition the refiner on the aesthetic rating it was trained with. The controls are disabled until a refiner is selected, and refiner models are only offered here, never as the main model. The refiner also runs for SDXL generations from the Canvas. It cannot be combined with **PiD Decode**. See [SDXL](/users-guide/models/local-models/sdxl/#sdxl-refiner). - ### Resetting to defaults Each model has recommended settings, which you can change in the Model Manager. **Reset all to model defaults**, the ↺ button in the Generate panel's title bar, applies them: size, steps, CFG or guidance, scheduler and VAE settings. Your prompts are left alone. The button is disabled when your settings already match the defaults. diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index d4db3e4a2f5..ff09d1acc93 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -1086,7 +1086,8 @@ "heading": "Refiner start", "paragraphs": [ "Where in the schedule the base model hands over to the refiner.", - "0.8 means the base model denoises the first 80% and the refiner finishes the last 20%; 0 hands the whole schedule to the refiner." + "0.8 means the base model denoises the first 80% and the refiner finishes the last 20%; 0 hands the whole schedule to the refiner, and 1 leaves it nothing to do, so the refiner is not run.", + "On the Canvas the base pass cannot start after the hand-over, so an image-to-image strength above 1 − Refiner start is capped there." ] }, "refinerAestheticScore": { @@ -2691,7 +2692,7 @@ "refiner": "Refiner", "refinerModel": "Refiner model", "refinerNone": "None", - "noRefinerModels": "No SDXL Refiner models are installed.", + "refinerModelsScope": "SDXL Refiner models", "refinerSteps": "Refiner steps", "refinerCfgScale": "Refiner CFG", "refinerScheduler": "Refiner scheduler", @@ -4170,6 +4171,7 @@ "nothingRecalled": "No recallable image data", "partiallyRecalled": "Some image data was not recalled", "unnamedConcept": "unnamed concept", + "unnamedRefiner": "unnamed refiner", "skippedConcept": { "ambiguous": "{{name}} (matches more than one installed model)", "duplicate": "{{name}} (listed more than once)", @@ -4177,6 +4179,10 @@ "invalid": "{{name}} (unreadable metadata)", "modelUnavailable": "{{name}} (the image's model is not available)", "unresolved": "{{name}} (not installed)" + }, + "skippedRefiner": { + "ambiguous": "Refiner not restored: {{name}} matches more than one installed model; the current refiner was kept.", + "unresolved": "Refiner not restored: {{name}} is not installed; the current refiner was kept." } }, "star": { diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index 408e1063131..22cc84c844c 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -1156,19 +1156,63 @@ describe('compileCanvasGraph — SDXL Refiner', () => { }); it.each(['inpaint', 'outpaint'] as const)( - 'keeps the mask on the base pass and decodes the refiner for %s', + 'feeds the same gradient mask to both passes and decodes the refiner for %s', (mode) => { const { backendGraph } = compile(sdxlModel, mode, { settings: refinerSettings, strength: 0.5 }); expect(backendGraph.nodes.denoise_latents).toMatchObject({ denoising_end: 0.8, denoising_start: 0.5 }); expect(getEdge(backendGraph, 'denoise_latents', 'denoise_mask')?.source.node_id).toBe('create_gradient_mask'); - expect(getEdge(backendGraph, 'refiner_denoise_latents', 'denoise_mask')).toBeUndefined(); + // The refiner respects the same region, or its unmasked latents would bleed past the coherence edge. + expect(getEdge(backendGraph, 'refiner_denoise_latents', 'denoise_mask')?.source.node_id).toBe( + 'create_gradient_mask' + ); expect(getEdge(backendGraph, 'refiner_denoise_latents', 'latents')?.source.node_id).toBe('denoise_latents'); expect(getEdge(backendGraph, 'canvas_l2i', 'latents')?.source.node_id).toBe('refiner_denoise_latents'); expect(getNodeByType(backendGraph, 'core_metadata')?.generation_mode).toBe(`sdxl_${mode}`); } ); + it.each(['txt2img', 'img2img', 'inpaint', 'outpaint'] as const)( + 'never lets the %s graft restore the full base schedule while a refiner is active', + (mode) => { + const { backendGraph } = compile(sdxlModel, mode, { settings: refinerSettings, strength: 0.9 }); + + // The canvas re-derives the base range from the strength after the refiner stage is grafted. + expect(backendGraph.nodes.denoise_latents?.denoising_end).toBe(0.8); + expect(backendGraph.nodes.refiner_denoise_latents?.denoising_start).toBe(0.8); + } + ); + + it('keeps regional guidance on the base pass', () => { + const { backendGraph } = compile(sdxlModel, 'txt2img', { + regionalGuidance: [ + { + autoNegative: false, + id: 'r1', + maskImageName: 'r1-mask.png', + negativePrompt: null, + positivePrompt: 'a cat', + referenceImages: [], + }, + ], + settings: refinerSettings, + }); + + const collected = backendGraph.edges + .filter((edge) => edge.destination.node_id === 'pos_cond_collect') + .map((edge) => edge.source.node_id); + expect(collected).toEqual(['pos_cond', 'rg_pos_cond_r1']); + expect(getEdge(backendGraph, 'denoise_latents', 'positive_conditioning')?.source.node_id).toBe('pos_cond_collect'); + expect(getEdge(backendGraph, 'refiner_denoise_latents', 'positive_conditioning')?.source.node_id).toBe( + 'refiner_pos_cond' + ); + expect( + backendGraph.edges + .filter((edge) => edge.destination.node_id === 'refiner_denoise_latents') + .map((edge) => edge.destination.field) + ).toEqual(['unet', 'positive_conditioning', 'negative_conditioning', 'latents']); + }); + it('attaches control layers to the base pass, not the refiner', () => { const { backendGraph } = compile(sdxlModel, 'txt2img', { controlLayers: [ diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts index 86d0ab893fb..3038008632f 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.ts @@ -12,7 +12,8 @@ import { GRAPH_BUILDERS } from '@features/generation/core/graph'; import { addEdge, addNode, toGraphContract } from '@features/generation/core/graphBuilder'; import { addKrea2ConditioningEnhancers } from '@features/generation/core/krea2Conditioning'; import { getIsPidSupportedBase } from '@features/generation/core/pid'; -import { getActiveRefiner, getRefinerDenoisingSplit } from '@features/generation/core/sdxlRefiner'; +import { getBaseDenoisingRange } from '@features/generation/core/sdxlRefiner'; +import { REFINER_DENOISE_NODE_ID } from '@features/generation/core/sdxlRefinerGraph'; import type { CanvasCompositingSettings, @@ -52,20 +53,14 @@ const canvasDenoisingStart = (model: GenerateModelConfig, strength: number): num return 1 - strength ** (usesOptimizedCurve ? 0.2 : 1); }; -/** Applies the strength-derived range to the base denoise; with a refiner, the base pass stops at the hand-over. */ +/** Re-derives the base denoise range from the strength; the refiner hand-over rule stays with its owner. */ const setCanvasDenoisingRange = ( denoise: BackendInvocationContract, model: GenerateModelConfig, settings: GenerateSettings, strength: number ): void => { - const denoisingStart = canvasDenoisingStart(model, strength); - const range = getActiveRefiner(settings, model) - ? getRefinerDenoisingSplit(settings.refinerStart, denoisingStart) - : { denoising_end: 1, denoising_start: denoisingStart }; - - denoise.denoising_start = range.denoising_start; - denoise.denoising_end = range.denoising_end; + Object.assign(denoise, getBaseDenoisingRange(settings, model, canvasDenoisingStart(model, strength))); }; /** True for the image-referencing modes (everything but pure txt2img). */ @@ -377,6 +372,11 @@ const graftMaskTail = ( addEdge(graph, unetSource.node, unetSource.field, gradientMask, 'unet'); } addEdge(graph, gradientMask, 'denoise_mask', denoise, 'denoise_mask'); + // The refiner pass must respect the same region, or its unmasked latents would bleed past the coherence edge. + const refinerDenoise = graph.nodes[REFINER_DENOISE_NODE_ID]; + if (refinerDenoise) { + addEdge(graph, gradientMask, 'denoise_mask', refinerDenoise, 'denoise_mask'); + } const expandMask = addNode(graph, { fade_size_px: compositing.maskBlur, diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts index 3db4143b399..250bd6908b1 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts @@ -1410,4 +1410,65 @@ describe('SDXL Refiner', () => { 'SDXL is not an SDXL Refiner model.' ); }); + + it('hands the whole schedule to the refiner at a start of 0', () => { + const graph = compile(sdxlModel, { ...refinerSettings, refinerStart: 0 }); + + // The base pass only seeds the latents from noise; the refiner denoises every step. + expect(graph.nodes.denoise_latents).toMatchObject({ denoising_end: 0, denoising_start: 0 }); + expect(graph.nodes.refiner_denoise_latents).toMatchObject({ denoising_end: 1, denoising_start: 0 }); + expect(getEdge(graph, 'canvas_output', 'latents')?.source.node_id).toBe('refiner_denoise_latents'); + expect(getNodeByType(graph, 'core_metadata')?.refiner_start).toBe(0); + }); + + it('runs without the refiner at a start of 1, where its pass would be empty', () => { + const graph = compile(sdxlModel, { + ...refinerSettings, + gemma2EncoderModel: { base: 'any', key: 'gemma2', name: 'Gemma 2 2B', type: 'gemma2_encoder' }, + pidDecoderModel: { base: 'sdxl', key: 'pid-sdxl', name: 'PiD SDXL', type: 'pid_decoder' }, + pidMode: 'fit', + refinerStart: 1, + }); + + // Neither the refiner model nor its PiD conflict applies: the graph is the plain SDXL graph. + expect(graph.nodes.refiner_model_loader).toBeUndefined(); + expect(graph.nodes.refiner_denoise_latents).toBeUndefined(); + expect(graph.nodes.denoise_latents).toMatchObject({ denoising_end: 1, denoising_start: 0 }); + expect(getNodeByType(graph, 'core_metadata')).not.toHaveProperty('refiner_model'); + }); + + it('keeps IP Adapter reference images on the base pass', () => { + const sdxlIpAdapter: ComponentModelConfig = { + base: 'sdxl', + key: 'sdxl-ip-adapter', + name: 'SDXL IP Adapter', + type: 'ip_adapter', + }; + const graph = compile(sdxlModel, { + ...refinerSettings, + referenceImages: [ + { + id: 'ref-1', + isEnabled: true, + config: { + beginEndStepPct: [0, 1], + clipVisionModel: 'ViT-H', + image: refImage, + method: 'full', + model: sdxlIpAdapter, + type: 'ip_adapter', + weight: 1, + }, + }, + ], + }); + + expect(getEdge(graph, 'denoise_latents', 'ip_adapter')?.source.node_id).toContain('ip_adapter_collector'); + expect(getEdge(graph, 'refiner_denoise_latents', 'ip_adapter')).toBeUndefined(); + expect( + graph.edges + .filter((edge) => edge.destination.node_id === 'refiner_denoise_latents') + .map((e) => e.destination.field) + ).toEqual(['unet', 'positive_conditioning', 'negative_conditioning', 'latents']); + }); }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.ts index 59637951803..7378dec48d0 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.ts @@ -54,7 +54,7 @@ import { import { addKrea2ConditioningEnhancers } from './krea2Conditioning'; import { addPidDecode, getPidDenoiseSize, getPidMetadata, shouldUsePidDecode } from './pidGraph'; import { getEffectiveReferenceImage } from './referenceImage'; -import { getActiveRefiner } from './sdxlRefiner'; +import { getActiveRefiner, getBaseDenoisingRange } from './sdxlRefiner'; import { addSDXLRefinerStage, getRefinerMetadata } from './sdxlRefinerGraph'; const getCompatibleComponentSource = ( @@ -342,8 +342,7 @@ const buildSDGraph = ( const denoise = addNode(graph, { cfg_rescale_multiplier: settings.cfgRescaleMultiplier, cfg_scale: settings.cfgScale, - denoising_end: 1, - denoising_start: 0, + ...getBaseDenoisingRange(settings, model, 0), id: 'denoise_latents', ...(supportsHiDiffusion ? { diff --git a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts index a26e04539e7..02b449f6f89 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.test.ts @@ -2,23 +2,43 @@ import { describe, expect, it } from 'vitest'; import type { MainModelConfig } from './types'; -import { getActiveRefiner, getRefinerDenoisingSplit, isRefinerModel } from './sdxlRefiner'; +import { getActiveRefiner, getBaseDenoisingRange, getRefinerDenoisingSplit, isRefinerModel } from './sdxlRefiner'; const sdxl: MainModelConfig = { base: 'sdxl', key: 'sdxl', name: 'SDXL', type: 'main' }; const refiner: MainModelConfig = { base: 'sdxl-refiner', key: 'refiner', name: 'Refiner', type: 'main' }; +const withRefiner = { refinerModel: refiner, refinerStart: 0.8 }; describe('sdxlRefiner', () => { it('activates a refiner only for a refiner-family model on an SDXL main', () => { - expect(getActiveRefiner({ refinerModel: refiner }, sdxl)).toBe(refiner); - expect(getActiveRefiner({ refinerModel: refiner }, { ...sdxl, base: 'sd-1' })).toBeNull(); - expect(getActiveRefiner({ refinerModel: sdxl }, sdxl)).toBeNull(); - expect(getActiveRefiner({ refinerModel: null }, sdxl)).toBeNull(); + expect(getActiveRefiner(withRefiner, sdxl)).toBe(refiner); + expect(getActiveRefiner(withRefiner, { ...sdxl, base: 'sd-1' })).toBeNull(); + expect(getActiveRefiner({ ...withRefiner, refinerModel: sdxl }, sdxl)).toBeNull(); + expect(getActiveRefiner({ ...withRefiner, refinerModel: null }, sdxl)).toBeNull(); expect(isRefinerModel({ base: 'sdxl-refiner', type: 'lora' })).toBe(false); }); + it('treats a start of 1 as off, since the refiner would have no schedule left', () => { + expect(getActiveRefiner({ ...withRefiner, refinerStart: 1 }, sdxl)).toBeNull(); + expect(getActiveRefiner({ ...withRefiner, refinerStart: 0 }, sdxl)).toBe(refiner); + }); + it('ends the base pass at the hand-over and never starts it later than that', () => { expect(getRefinerDenoisingSplit(0.8)).toEqual({ denoising_end: 0.8, denoising_start: 0 }); expect(getRefinerDenoisingSplit(0.8, 0.3)).toEqual({ denoising_end: 0.8, denoising_start: 0.3 }); expect(getRefinerDenoisingSplit(0.8, 0.95)).toEqual({ denoising_end: 0.8, denoising_start: 0.8 }); }); + + it('owns the base range for every caller: a strength-derived start never restores the full schedule', () => { + expect(getBaseDenoisingRange(withRefiner, sdxl, 0)).toEqual({ denoising_end: 0.8, denoising_start: 0 }); + expect(getBaseDenoisingRange(withRefiner, sdxl, 0.4)).toEqual({ denoising_end: 0.8, denoising_start: 0.4 }); + expect(getBaseDenoisingRange(withRefiner, sdxl, 0.95)).toEqual({ denoising_end: 0.8, denoising_start: 0.8 }); + expect(getBaseDenoisingRange({ ...withRefiner, refinerModel: null }, sdxl, 0.4)).toEqual({ + denoising_end: 1, + denoising_start: 0.4, + }); + expect(getBaseDenoisingRange({ ...withRefiner, refinerStart: 1 }, sdxl, 0.4)).toEqual({ + denoising_end: 1, + denoising_start: 0.4, + }); + }); }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts index ecf7fd02cec..54d538f094d 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefiner.ts @@ -35,12 +35,17 @@ export const isRefinerModel = (model: { base: string; type: string } | null | un export const isRefinerSupported = (model: GenerateModelConfig | undefined): boolean => model?.type === 'main' && model.base === 'sdxl'; -/** The refiner that will run: a refiner model selected on an SDXL main model. */ +/** + * The refiner that will run: a refiner model selected on an SDXL main model. A start of 1 leaves the refiner no + * schedule, so it is treated as off rather than loading a model for an empty pass. + */ export const getActiveRefiner = ( - settings: Pick, + settings: Pick, model: GenerateModelConfig | undefined ): MainModelConfig | null => - isRefinerSupported(model) && isRefinerModel(settings.refinerModel) ? settings.refinerModel : null; + isRefinerSupported(model) && isRefinerModel(settings.refinerModel) && settings.refinerStart < MAX_REFINER_START + ? settings.refinerModel + : null; /** * The base model's denoise range when a refiner finishes the schedule. An image-to-image start past the hand-over @@ -54,6 +59,19 @@ export const getRefinerDenoisingSplit = ( denoising_end: refinerStart, }); +/** + * The single owner of the base denoise range. `denoisingStart` is the strength-derived start (0 for text-to-image); + * with an active refiner the base pass ends at the hand-over, otherwise it runs to the end of the schedule. + */ +export const getBaseDenoisingRange = ( + settings: Pick, + model: GenerateModelConfig, + denoisingStart: number +): { denoising_start: number; denoising_end: number } => + getActiveRefiner(settings, model) + ? getRefinerDenoisingSplit(settings.refinerStart, denoisingStart) + : { denoising_end: 1, denoising_start: denoisingStart }; + export const getRefinerValidationReasons = ( model: GenerateModelConfig, settings: RefinerSettings & Pick @@ -69,7 +87,7 @@ export const getRefinerValidationReasons = ( const reasons: string[] = []; // PiD replaces the VAE decode the refiner's second denoise stage feeds; the two cannot share a graph. - if (settings.pidMode !== 'off') { + if (settings.pidMode !== 'off' && settings.refinerStart < MAX_REFINER_START) { reasons.push('PiD decode is not compatible with the SDXL Refiner. Turn PiD off or clear the refiner model.'); } diff --git a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts index 7e34c27b88f..18a25c3feb1 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/sdxlRefinerGraph.ts @@ -6,7 +6,6 @@ import type { GenerateSettings, MainModelConfig } from './types'; import { coerceSchedulerForGraph } from './baseGenerationPolicies'; import { addEdge, addNode, toModelIdentifier } from './graphBuilder'; -import { getRefinerDenoisingSplit } from './sdxlRefiner'; export const REFINER_DENOISE_NODE_ID = 'refiner_denoise_latents'; @@ -16,7 +15,7 @@ interface RefinerStageInput { /** The SDXL main model; the refiner shares its scheduler set. */ model: MainModelConfig; refiner: MainModelConfig; - /** The base denoise, whose latents the refiner finishes. Its `denoising_end` becomes the hand-over point. */ + /** The base denoise, whose latents the refiner finishes; `getBaseDenoisingRange` already ends it at the hand-over. */ denoise: BackendInvocationContract; positivePrompt: BackendInvocationContract; negativePrompt: BackendInvocationContract; @@ -36,11 +35,6 @@ export const addSDXLRefinerStage = ({ settings, }: RefinerStageInput): BackendInvocationContract => { const scheduler = coerceSchedulerForGraph(model, settings.refinerScheduler); - const split = getRefinerDenoisingSplit(settings.refinerStart, denoise.denoising_start as number); - - denoise.denoising_start = split.denoising_start; - denoise.denoising_end = split.denoising_end; - const modelLoader = addNode(graph, { id: 'refiner_model_loader', model: toModelIdentifier(refiner), diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx index 9674d3a9adf..de524b586bc 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateRefinerSection.browser.test.tsx @@ -23,8 +23,8 @@ const CATALOG: readonly GenerationModelCatalogItem[] = [sdxlModel, refinerModel] vi.mock('./GenerationUiContext', () => ({ /** The picker reduced to one button per catalog model that passes the section's filter. */ - GenerationModelSelect: ({ filter, onChange, value }: GenerationModelSelectProps) => ( -
+ GenerationModelSelect: ({ filter, onChange, scopeLabel, value }: GenerationModelSelectProps) => ( +
{CATALOG.filter((model) => filter?.(model) ?? true).map((model) => (