diff --git a/docs/src/content/docs/Users Guide/03.Models/02.model-families.mdx b/docs/src/content/docs/Users Guide/03.Models/02.model-families.mdx index 10d4d5543d0..ca8c0f3c991 100644 --- a/docs/src/content/docs/Users Guide/03.Models/02.model-families.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/02.model-families.mdx @@ -123,7 +123,7 @@ faster generation. | **Z-Image** | Images | 6B model in a fast, few-step Turbo variant and a Base variant. Understands English and Chinese prompts, with ControlNets. | Apache 2.0 | [Z-Image](/users-guide/models/local-models/z-image/) | | **ERNIE-Image** | Images | Baidu's 8B text-to-image model, in standard and distilled Turbo variants. | Apache 2.0 | [ERNIE-Image](/users-guide/models/local-models/ernie-image/) | | **Qwen Image** | Images | Text-to-image (2512) and instruction-based image editing (Edit 2511), with fast Lightning variants. | Apache 2.0 | [Qwen Image](/users-guide/models/local-models/qwen-image/) | -| **Qwen-Image-2.1** | Images | 7B text-to-image model with strong text rendering and transparent (RGBA) output, in a base and an 8-step Turbo variant. | Non-commercial | [Qwen-Image-2.1](/users-guide/models/local-models/qwen-image-2-1/) | +| **Qwen-Image-2.1** | Images | 7B text-to-image model with strong text rendering and transparent (RGBA) output, in a base and an 8-step Turbo variant. Edits from reference images. | Non-commercial | [Qwen-Image-2.1](/users-guide/models/local-models/qwen-image-2-1/) | | **Krea-2** | Images | 12B model in fast Turbo and undistilled Raw variants, with training-free style reference. | Community, free under $1M revenue | [Krea-2](/users-guide/models/local-models/krea-2/) | | **Ideogram 4** | Images | Structured prompts that place described elements in regions of the image. | Non-commercial | [Ideogram 4](/users-guide/models/local-models/ideogram4/) | | **Anima** | Images | Compact 2B anime-focused model, with ControlNet-LLLite adapters for inpainting and sketch guidance. | Non-commercial | [Anima](/users-guide/models/local-models/anima/) | diff --git a/docs/src/content/docs/Users Guide/03.Models/Local Models/qwen-image-2-1.mdx b/docs/src/content/docs/Users Guide/03.Models/Local Models/qwen-image-2-1.mdx index f7d9298c745..8ae96a8bc6f 100644 --- a/docs/src/content/docs/Users Guide/03.Models/Local Models/qwen-image-2-1.mdx +++ b/docs/src/content/docs/Users Guide/03.Models/Local Models/qwen-image-2-1.mdx @@ -1,7 +1,7 @@ --- title: Qwen-Image-2.1 description: Generate images with Qwen-Image-2.1 and its 8-step Turbo variant, from the Diffusers folders or a GGUF transformer, including transparent (RGBA) output. -lastUpdated: 2026-10-17 +lastUpdated: 2026-10-22 sidebar: order: 8 --- @@ -14,7 +14,7 @@ Two variants are available, and the variant is detected on install: - **Qwen-Image-2.1** — the base model. 40 steps by default. - **Qwen-Image-2.1 Turbo** — distilled for 8 steps on a fixed schedule of its own. -Text-to-image, image-to-image, inpainting and outpainting on the Canvas are supported. Reference images (image editing), LoRAs and regional guidance are not supported yet. +Text-to-image, image-to-image, inpainting and outpainting on the Canvas are supported, and with the Diffusers models also edits from [reference images](#reference-images). LoRAs and regional guidance are not supported yet. ### License @@ -57,6 +57,17 @@ The text encoder runs first and is unloaded from the GPU when the transformer ne The VAE needs about 8 GB of working memory to decode a 1024×1024 image, and is priced the same to encode one for image-to-image, inpainting or outpainting (twice that with `precision: float32`). When an image is too large for the card to handle in one piece, Invoke decodes and encodes it in tiles; see [Large decodes](/configuration/optimization/low-vram-mode/). Invoke picks 1024-pixel tiles, and 768 or 512 only on cards with less memory: measured at 2048×2048, 1024-pixel tiles came close to an untiled decode (47.7 dB PSNR), while the 256-pixel tiles other VAEs default to left visible seams (35.6 dB). +### Reference images + +With a Qwen-Image-2.1 **Diffusers** model, add up to four images with **Add reference image** in the Guidance section (see [Adding reference images](/users-guide/image-generation/generate-panel/#adding-reference-images)) and say in the prompt what to do with them, for example *put the jacket from the first image on the person in the second* or *the same cat, as a watercolor*. The model reads each reference twice: its Qwen3-VL encoder looks at it alongside the prompt, and the transformer sees it as clean latents next to the image it generates. + +- **Diffusers models only.** Only the encoder that comes with the Diffusers model keeps the Qwen3-VL vision tower, which reads the images. A single-file or GGUF transformer with a standalone encoder does not offer reference images, and on a Diffusers model the references refuse to run while a standalone **Qwen3-VL Encoder** is selected under **Components**; clear it to edit. +- **Size.** Each reference is read at about one megapixel at its own aspect ratio, whatever its size. The generated image takes the size set in the panel; **Use size** on a reference's thumbnail matches it. +- **Transparency.** The encoder sees a transparent reference over white, as the model was trained; its latents keep the transparency. +- **At most four.** A graph or API request with more than four references is refused with an error. +- **Memory.** The encoder's vision tower, about 1.1 GB, is loaded only for generations with references and joins the encoder while it reads them; plain prompts never load it, and the encoder itself is not loaded again when you switch between edits and plain prompts. Each reference adds about 4,100 tokens to the start of the sequence, which the model keeps for the whole generation: about 2 GB per reference, twice that with **CFG** above 1. Where that would take more than half of the GPU, Invoke reprocesses the references at every step instead of keeping them, which uses less memory but takes noticeably longer. Reading the references also needs encoder memory that grows quickly with their number: about 7 GB beyond its weights for four, so on a 24 GB card part of the encoder then runs from system memory. +- **Reproducing an edit.** Keeping the references and reprocessing them give slightly different images from the same seed, and which one runs depends on the GPU's memory. The image's metadata records which one ran, as `qwen_image_2_1_prefix_kv_cache`: `true` when the references were kept, `false` when they were reprocessed. Generations without references record it too. + ### Transparent images The Qwen-Image-2.1 VAE decodes an alpha channel along with the colors, so a prompt such as *a sticker of a red fox on a transparent background* produces a cut-out image. An image is saved with transparency only when it has some: when more than 0.1% of its pixels are noticeably transparent. Opaque images are saved as ordinary RGB, so the faint edges tiled decoding leaves in the alpha channel never turn an opaque image transparent. diff --git a/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx b/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx index 1ec9cbe5742..b5f3de5a4ce 100644 --- a/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx +++ b/docs/src/content/docs/Users Guide/04.Image Generation/generate-panel.mdx @@ -120,7 +120,7 @@ What a reference does, and what options it has, depends on the model family: | **SD 1.5, SDXL** | An IP-Adapter model carries the reference's look into the image. | **Mode**: **Style**, **Composition** or **Both**. **Weight**. Under **Advanced**: the IP-Adapter model, a **Style variant** and the **Active steps** range. | | **FLUX.1** | An IP-Adapter, or a **FLUX Redux** model, which transfers the reference's content. | IP-Adapter options as above, or **Image influence** (Lowest to Highest) for Redux. | | **FLUX.1 Kontext** | Kontext edits or recombines the reference images according to your prompt. | None beyond the model. | -| **FLUX.2, Qwen Image Edit** | The model reads the references directly and uses them as your prompt instructs, for example *put the jacket from the first image on the person in the second*. | None. | +| **FLUX.2, Qwen Image Edit, Qwen-Image-2.1** | The model reads the references directly and uses them as your prompt instructs, for example *put the jacket from the first image on the person in the second*. | None. | | **Krea-2** | One reference's look — palette, texture, rendering — is transferred into the image while the prompt keeps driving the content; see [Krea-2](/users-guide/models/local-models/krea-2/#style-reference). | **Style strength** (0–2; 0 turns the reference off). | Reference images require an extra model for SD 1.5, SDXL and FLUX.1 (an IP-Adapter or FLUX Redux model). These are available as starter models in the Model Manager, and an installed IP-Adapter is selected automatically. diff --git a/docs/src/content/docs/contributing/new-model-integration.mdx b/docs/src/content/docs/contributing/new-model-integration.mdx index c6325083423..50bbd3af62e 100644 --- a/docs/src/content/docs/contributing/new-model-integration.mdx +++ b/docs/src/content/docs/contributing/new-model-integration.mdx @@ -885,8 +885,8 @@ Paths below are relative to `invokeai/frontend/webv2/src/features/generation/cor ### Reference images -- Backend: set `FeaturesFacet.max_reference_images` (and `reference_images_require_variant` if only one variant supports them). Wire the images into the text encoder and/or the denoise node, as the reference pipeline does. -- webv2: extend the reference-image config union (`types.ts`), `getDefaultReferenceImageConfig` and `getReferenceImageConfigSupported` (`baseGenerationPolicies.ts`), and wire the images in the builder. +- Backend: set `FeaturesFacet.max_reference_images`, plus `reference_images_require_variant` if only one variant supports them or `reference_images_require_format` if only one format does. Wire the images into the text encoder and/or the denoise node, as the reference pipeline does. +- webv2: extend the reference-image config union (`types.ts`); the config-type set and the normalize, clone and sync switches in `settings.ts`; the persisted union in `src/workbench/canvasMigration.ts`; and `getDefaultReferenceImageConfig` and `getReferenceImageConfigSupported` (`baseGenerationPolicies.ts`). Wire the images in the builder, and report any setting that keeps them from running in `getReferenceImageValidationReasons`, not only from the builder. ### Control and regional guidance diff --git a/docs/src/content/docs/development/Guides/recall-api.mdx b/docs/src/content/docs/development/Guides/recall-api.mdx index 8478a3fe4a9..771f1d917de 100644 --- a/docs/src/content/docs/development/Guides/recall-api.mdx +++ b/docs/src/content/docs/development/Guides/recall-api.mdx @@ -250,12 +250,13 @@ separate adapter model: - **FLUX.2 Klein** — built-in reference image support. - **FLUX Kontext** — reference image associated with the main model. - **Qwen Image Edit** — reference image associated with the main model. +- **Qwen-Image-2.1** — reference images read by its own encoder, on the Diffusers model only. Because there is no adapter model to resolve, these entries carry only `image_name`. When the frontend receives them, it picks the appropriate config flavor (`flux2_reference_image`, `flux_kontext_reference_image`, -or `qwen_image_reference_image`) based on the currently-selected main -model, matching the behavior of a manual drag-and-drop. +`qwen_image_reference_image` or `qwen_image_2_1_reference_image`) based on +the currently-selected main model, matching the behavior of a manual drag-and-drop. ## Usage Examples diff --git a/invokeai/app/api/routers/recall_parameters.py b/invokeai/app/api/routers/recall_parameters.py index 20caef0d1b5..f32e18bf008 100644 --- a/invokeai/app/api/routers/recall_parameters.py +++ b/invokeai/app/api/routers/recall_parameters.py @@ -65,10 +65,10 @@ class ReferenceImageRecallParameter(BaseModel): Used for reference images that feed directly into the main model rather than through a separate IP-Adapter / ControlNet model — for example - FLUX.2 Klein, FLUX Kontext, and Qwen Image Edit. The receiving frontend - picks the correct config type (``flux2_reference_image`` / - ``qwen_image_reference_image`` / ``flux_kontext_reference_image``) based - on the currently-selected main model. + FLUX.2 Klein, FLUX Kontext, Qwen Image Edit and Qwen-Image-2.1. The + receiving frontend picks the correct config type (``flux2_reference_image`` / + ``qwen_image_reference_image`` / ``qwen_image_2_1_reference_image`` / + ``flux_kontext_reference_image``) based on the currently-selected main model. """ image_name: str = Field(description="The filename of the reference image in outputs/images") diff --git a/invokeai/app/invocations/qwen_image_2_1/qwen_image_2_1_denoise.py b/invokeai/app/invocations/qwen_image_2_1/qwen_image_2_1_denoise.py index 9c428c8a729..2fe3dfa4ae3 100644 --- a/invokeai/app/invocations/qwen_image_2_1/qwen_image_2_1_denoise.py +++ b/invokeai/app/invocations/qwen_image_2_1/qwen_image_2_1_denoise.py @@ -1,3 +1,4 @@ +import itertools import math from typing import Optional @@ -14,12 +15,13 @@ Input, InputField, LatentsField, + MetadataField, QwenImage21ConditioningField, WithBoard, WithMetadata, ) +from invokeai.app.invocations.metadata_linked import LatentsMetaOutput from invokeai.app.invocations.model import TransformerField -from invokeai.app.invocations.primitives import LatentsOutput from invokeai.app.services.shared.invocation_context import InvocationContext from invokeai.app.util.startup_utils import log_attention_backends from invokeai.backend.flux.sampling_utils import clip_timestep_schedule_fractional @@ -30,13 +32,16 @@ from invokeai.backend.stable_diffusion.diffusers_pipeline import PipelineIntermediateState from invokeai.backend.stable_diffusion.diffusion.conditioning_data import QwenImage21ConditioningInfo from invokeai.backend.util.attention import sdpa_score_matrix_bytes -from invokeai.backend.util.devices import TorchDevice +from invokeai.backend.util.devices import TorchDevice, device_has_dedicated_vram from invokeai.backend.util.logging import InvokeAILogger LATENT_CHANNELS = 64 VAE_SCALE_FACTOR = 16 # One vision-language slot of the transformer's joint sequence covers 2x2 latent tokens. TOKENS_PER_SLOT = 4 +# The metadata entry that records whether a run kept the prefix KV cache (True) or recomputed the prefix on every +# step (False). +PREFIX_CACHE_METADATA_KEY = "qwen_image_2_1_prefix_kv_cache" logger = InvokeAILogger.get_logger(__name__) @@ -63,6 +68,98 @@ def clip_sigmas( return list(sigmas[first_step : last_step + 1]), first_step +def slot_runs(image_pad_mask: torch.Tensor | None) -> list[int]: + """The lengths of the runs of image slots in a prompt's mask: one run per reference image, in order.""" + if image_pad_mask is None: + return [] + return [len(list(run)) for is_slot, run in itertools.groupby(image_pad_mask[0].tolist()) if is_slot] + + +def _describe_grids(grids: list[tuple[int, int]]) -> str: + # A slot is 32x32 pixels of the reference as the encoder read it. + return ", ".join(f"{columns * 32}x{rows * 32}" for rows, columns in grids) or "none" + + +def check_reference_slots( + info: QwenImage21ConditioningInfo, reference_shapes: list[tuple[int, int, int]], prompt: str +) -> None: + """Refuse references whose latents do not match the prompt's image slots, reference by reference. + + Each reference's latents fill one run of the prompt's slots, 2x2 latents per slot, at the slot grid the + encoder read it at. This catches a different number of references, or one at a different size, than the + prompt was encoded with -- including references of different sizes in another order. It cannot tell apart + same-size references in another order, or a same-size image swapped for another: the slots carry no identity. + """ + expected = [(h // 2, w // 2) for _, h, w in reference_shapes] + found = list(info.reference_grids) + if found != expected or slot_runs(info.image_pad_mask) != [rows * columns for rows, columns in found]: + raise ValueError( + f"The {prompt} prompt was encoded with {len(found)} reference image(s) ({_describe_grids(found)}), but " + f"the denoise node received {len(expected)} reference latent(s) ({_describe_grids(expected)}): the " + "counts or sizes differ. Encode each reference with Image to Latents in reference mode, from the same " + "images and in the same order as the prompts." + ) + + +def prefix_length(info: QwenImage21ConditioningInfo) -> int: + """Tokens before the target in the joint sequence: the prompt, each reference slot widened to 2x2 latents.""" + slots = 0 if info.image_pad_mask is None else int(info.image_pad_mask.sum()) + return info.prompt_embeds.shape[1] + (TOKENS_PER_SLOT - 1) * slots + + +def kv_bytes_per_prefix_token(transformer: torch.nn.Module, dtype: torch.dtype) -> int: + """K and V of one prefix token in every block of `transformer`, in the compute dtype the caches are kept in. + + 2 x 4096 x 32 x 2 bytes for the released model in bf16 (32 heads of 128, 32 blocks): 0.5 MiB. + """ + config = transformer.config + hidden = config.num_attention_heads * config.attention_head_dim + return 2 * hidden * len(transformer.transformer_blocks) * dtype.itemsize + + +def prefix_cache_fits(cache_bytes: int, device: torch.device) -> bool: + """Whether the prefix caches can stay on the GPU for the whole run. + + They cannot be offloaded the way the transformer's weights can, so caches larger than half the card would + leave too little for the model however it streams: four references at CFG above 1 hold ~16 GiB. Past that the + prefix is recomputed every step instead -- slower, but within the memory of an ordinary run. Memory that is + not a dedicated card's (CPU, MPS, an integrated XPU) is not limited here, nor a card that reports no size. + """ + if not device_has_dedicated_vram(device): + return True + backend = torch.xpu if device.type == "xpu" else torch.cuda + total = int(backend.get_device_properties(device).total_memory) + return total == 0 or cache_bytes <= total // 2 + + +def prefix_segments(info: QwenImage21ConditioningInfo) -> list[int]: + """The prefix's blocks in the joint sequence, in order: each run of text, and each reference widened to 2x2 + latents per slot. The prefill attends each block over every key up to its end.""" + if info.image_pad_mask is None: + return [info.prompt_embeds.shape[1]] + return [ + len(list(run)) * (TOKENS_PER_SLOT if is_slot else 1) + for is_slot, run in itertools.groupby(info.image_pad_mask[0].tolist()) + ] + + +def largest_attention_call(infos: list[QwenImage21ConditioningInfo], image_seq_len: int) -> int: + """Queries x keys of the largest attention call a run makes, over every prompt it runs. + + The prefill runs each prefix block over the keys before and in it, and the target over every key; a cached + step repeats only the target's call. With large references and a small target, the last reference's call is + the largest: four 1 MP references before a 512x512 target make it ~4x the target's. + """ + largest = 0 + for info in infos: + end = 0 + for length in prefix_segments(info): + end += length + largest = max(largest, length * end) + largest = max(largest, image_seq_len * (end + image_seq_len)) + return largest + + def pack_latents(latents: torch.Tensor) -> torch.Tensor: """(B, C, h, w) -> (B, h*w, C): one token per latent pixel, in raster order (patch size 1).""" batch, channels, height, width = latents.shape @@ -80,7 +177,7 @@ def unpack_latents(latents: torch.Tensor, height: int, width: int) -> torch.Tens title="Denoise - Qwen-Image-2.1", tags=["image", "qwen_image_2_1", "qwen-image-2.1"], category="image", - version="1.0.1", + version="1.1.0", classification=Classification.Prototype, ) class QwenImage21DenoiseInvocation(BaseInvocation, WithMetadata, WithBoard): @@ -92,6 +189,13 @@ class QwenImage21DenoiseInvocation(BaseInvocation, WithMetadata, WithBoard): denoise_mask: Optional[DenoiseMaskField] = InputField( default=None, description=FieldDescriptions.denoise_mask, input=Input.Connection ) + reference_latents: LatentsField | list[LatentsField] | None = InputField( + default=None, + description="Latents of the reference images an edit reads, from Image to Latents in reference mode, in the " + "order the prompts were encoded with them.", + input=Input.Connection, + title="Reference Latents", + ) denoising_start: float = InputField(default=0.0, ge=0, le=1, description=FieldDescriptions.denoising_start) denoising_end: float = InputField(default=1.0, ge=0, le=1, description=FieldDescriptions.denoising_end) transformer: TransformerField = InputField( @@ -125,21 +229,48 @@ def validate_cfg_scale_is_finite(cls, value: float | list[float]) -> float | lis return value @torch.no_grad() - def invoke(self, context: InvocationContext) -> LatentsOutput: - latents = self._run_diffusion(context).detach().to("cpu") - name = context.tensors.save(tensor=latents) + def invoke(self, context: InvocationContext) -> LatentsMetaOutput: + latents, used_prefix_cache = self._run_diffusion(context) + name = context.tensors.save(tensor=latents.detach().to("cpu")) + # The two modes sample visibly differently in bf16, and which one runs depends on the card's memory: the + # image's metadata has to say, or nothing tells why one graph and seed give another image on another card. + # A graph routes this output to the node that saves the image. + metadata = dict(self.metadata.root) if self.metadata is not None else {} + metadata[PREFIX_CACHE_METADATA_KEY] = used_prefix_cache # `LatentsOutput.build` assumes 8x latents; these are 16x, so report the requested size directly. - return LatentsOutput( - latents=LatentsField(latents_name=name, seed=self.seed), width=self.width, height=self.height + return LatentsMetaOutput( + latents=LatentsField(latents_name=name, seed=self.seed), + width=self.width, + height=self.height, + metadata=MetadataField.model_validate(metadata), ) def _load_conditioning( self, context: InvocationContext, field: QwenImage21ConditioningField, dtype: torch.dtype, device: torch.device - ) -> torch.Tensor: + ) -> QwenImage21ConditioningInfo: data = context.conditioning.load(field.conditioning_name) if len(data.conditionings) != 1 or not isinstance(data.conditionings[0], QwenImage21ConditioningInfo): raise ValueError("Expected exactly one Qwen-Image-2.1 conditioning.") - return data.conditionings[0].to(dtype=dtype, device=device).prompt_embeds + return data.conditionings[0].to(dtype=dtype, device=device) + + def _load_references(self, context: InvocationContext, device: torch.device) -> list[torch.Tensor]: + """The reference latents, `(1, C, h, w)` each, in the order the prompts were encoded with them.""" + if self.reference_latents is None: + return [] + fields = self.reference_latents if isinstance(self.reference_latents, list) else [self.reference_latents] + references = [] + for field in fields: + reference = context.tensors.load(field.latents_name).to(device=device, dtype=torch.float32) + if reference.dim() == 5: + reference = reference.squeeze(2) + if reference.shape[1] != LATENT_CHANNELS or reference.shape[-2] % 2 or reference.shape[-1] % 2: + raise ValueError( + f"Reference latents are {tuple(reference.shape[1:])}; Qwen-Image-2.1 reads {LATENT_CHANNELS} " + "channels with even sides. Encode the reference with Image to Latents - Qwen-Image-2.1 in " + "reference mode." + ) + references.append(reference) + return references def _prepare_cfg_scale(self, num_steps: int) -> list[float]: if isinstance(self.cfg_scale, list): @@ -163,7 +294,8 @@ def _prep_inpaint_mask(self, context: InvocationContext, latents: torch.Tensor) ) return mask.to(device=latents.device, dtype=latents.dtype) - def _run_diffusion(self, context: InvocationContext) -> torch.Tensor: + def _run_diffusion(self, context: InvocationContext) -> tuple[torch.Tensor, bool]: + """The denoised latents, and whether the prefix KV cache ran (rather than the prefix being recomputed).""" from diffusers.models.transformers.transformer_qwenimage21 import QwenImage21KVCache if self.denoising_start >= self.denoising_end: @@ -174,7 +306,6 @@ def _run_diffusion(self, context: InvocationContext) -> torch.Tensor: device = TorchDevice.choose_torch_device() dtype = TorchDevice.choose_bfloat16_safe_dtype(device) transformer_config = context.models.get_config(self.transformer.transformer) - transformer_info = context.models.load(self.transformer.transformer) latent_height, latent_width = self.height // VAE_SCALE_FACTOR, self.width // VAE_SCALE_FACTOR image_seq_len = latent_height * latent_width @@ -190,13 +321,26 @@ def _run_diffusion(self, context: InvocationContext) -> torch.Tensor: sigmas = torch.tensor(sigmas, dtype=torch.float32, device=device) cfg_scale = self._prepare_cfg_scale(len(full_sigmas) - 1)[first_step : first_step + len(sigmas) - 1] - pos_embeds = self._load_conditioning(context, self.positive_conditioning, dtype, device) + pos = self._load_conditioning(context, self.positive_conditioning, dtype, device) do_cfg = self.negative_conditioning is not None and any(value > 1.0 for value in cfg_scale) - neg_embeds = ( + neg = ( self._load_conditioning(context, self.negative_conditioning, dtype, device) if do_cfg and self.negative_conditioning is not None else None ) + pos_embeds = pos.prompt_embeds + neg_embeds = neg.prompt_embeds if neg is not None else None + + references = self._load_references(context, device) + reference_shapes = [(1, r.shape[-2], r.shape[-1]) for r in references] + check_reference_slots(pos, reference_shapes, "positive") + if neg is not None: + check_reference_slots(neg, reference_shapes, "negative") + # Clean latents, condition images first: the transformer drops them into the prompt's image slots. + reference_tokens = torch.cat([pack_latents(r) for r in references], dim=1).to(dtype) if references else None + + # Only now, with every input checked: loading reads the whole transformer into RAM. + transformer_info = context.models.load(self.transformer.transformer) # Drawn as the pipeline draws it, (B, frame, C, h, w), on the CPU so a seed means the same on any device. noise = torch.randn( @@ -234,27 +378,43 @@ def _run_diffusion(self, context: InvocationContext) -> torch.Tensor: # upcast); matching that is what makes this node reproduce its outputs. latents = pack_latents(latents).to(dtype) - def image_slot_mask(text_len: int) -> torch.Tensor: - # The joint sequence's slots: the text, then one slot per 2x2 group of target latents. Condition - # images would sit between the two. - text = torch.zeros(1, text_len, dtype=torch.bool, device=device) + def image_slot_mask(info: QwenImage21ConditioningInfo) -> torch.Tensor: + # The joint sequence's slots: the prompt with its reference slots, then one slot per 2x2 group of + # target latents. + text_len = info.prompt_embeds.shape[1] + prompt = ( + info.image_pad_mask.bool() + if info.image_pad_mask is not None + else torch.zeros(1, text_len, dtype=torch.bool, device=device) + ) target = torch.ones(1, image_seq_len // TOKENS_PER_SLOT, dtype=torch.bool, device=device) - return torch.cat([text, target], dim=1) - - img_shapes = [[(1, latent_height, latent_width)]] - pos_img_mask = image_slot_mask(pos_embeds.shape[1]) - neg_img_mask = image_slot_mask(neg_embeds.shape[1]) if neg_embeds is not None else None - + return torch.cat([prompt, target], dim=1) + + img_shapes = [[*reference_shapes, (1, latent_height, latent_width)]] + pos_img_mask = image_slot_mask(pos) + neg_img_mask = image_slot_mask(neg) if neg is not None else None + + # Each reference slot widens to 2x2 latent tokens in the joint sequence; with the text they are the + # prefix the cache holds. + pos_prefix = prefix_length(pos) + neg_prefix = prefix_length(neg) if neg is not None else None + # Text and reference keys and values do not depend on the step under `causal_condition`: the first step + # prefills them, the rest compute the image tokens only -- when the caches fit. + kv_bytes_per_token = kv_bytes_per_prefix_token(transformer_info.model, dtype) + cache_bytes = kv_bytes_per_token * (pos_prefix + (neg_prefix or 0)) + use_cache = bool(getattr(transformer_info.model.config, "causal_condition", False)) + if use_cache and not prefix_cache_fits(cache_bytes, device): + use_cache = False + logger.info( + f"Qwen-Image-2.1: the prefix caches would hold {cache_bytes / 1024**3:.1f} GiB, more than half of " + "this GPU; recomputing the prefix every step instead, which is slower." + ) working_memory = self._estimate_working_memory( - image_seq_len, - pos_embeds.shape[1], - None if neg_embeds is None else neg_embeds.shape[1], - do_cfg, - dtype.itemsize, + image_seq_len, pos_prefix, neg_prefix, do_cfg, use_cache, dtype.itemsize, kv_bytes_per_token ) working_memory += self._attention_score_bytes( transformer_info.model, - image_seq_len + max(pos_embeds.shape[1], 0 if neg_embeds is None else neg_embeds.shape[1]), + largest_attention_call([pos] if neg is None else [pos, neg], image_seq_len), device, dtype, ) @@ -274,16 +434,15 @@ def image_slot_mask(text_len: int) -> torch.Tensor: ) with transformer_info.model_on_device(working_mem_bytes=working_memory) as (_, transformer): - # Text keys and values do not depend on the step under `causal_condition`: the first step prefills - # them, the rest compute the image tokens only. One cache per guidance branch, for this run only. + # One cache per guidance branch, for this run only. num_blocks = len(transformer.transformer_blocks) - use_cache = bool(transformer.config.causal_condition) pos_cache = QwenImage21KVCache(num_blocks) if use_cache else None neg_cache = QwenImage21KVCache(num_blocks) if use_cache and neg_embeds is not None else None def predict(embeds: torch.Tensor, img_mask: torch.Tensor, cache, mode, timestep) -> torch.Tensor: + hidden_states = latents if reference_tokens is None else torch.cat([reference_tokens, latents], dim=1) out = transformer( - hidden_states=latents, + hidden_states=hidden_states, encoder_hidden_states=embeds, encoder_hidden_states_mask=None, timestep=timestep, @@ -331,45 +490,57 @@ def predict(embeds: torch.Tensor, img_mask: torch.Tensor, cache, mode, timestep) ) ) - return unpack_latents(latents, latent_height, latent_width).float() + return unpack_latents(latents, latent_height, latent_width).float(), use_cache @staticmethod def _estimate_working_memory( - image_seq_len: int, pos_text_len: int, neg_text_len: int | None, do_cfg: bool, element_size: int + image_seq_len: int, + pos_prefix: int, + neg_prefix: int | None, + do_cfg: bool, + use_cache: bool, + element_size: int, + kv_bytes_per_token: int, ) -> int: """Peak transformer activations, in bytes, for the model cache to keep free. Measured on an RTX 4090 in bf16 with the transformer fully resident: 0.55 GiB of activations at 1024x1024 (4096 image tokens) and 2.2 GiB at 2048x2048 (16384), about 0.14 MiB per token. 0.25 MiB per - token plus a 1 GiB base leaves room for the allocator and the prefix cache, which holds K and V of every - text token in all 32 blocks (~0.5 MiB per text token, per guidance branch). Both per-token terms are bf16 - figures and scale with `element_size`, the compute dtype's width: a float32 fallback needs twice as much. + token plus a 1 GiB base leaves room for the allocator. That term is a bf16 figure and scales with + `element_size`, the compute dtype's width: a float32 fallback needs twice as much. The prefix cache adds + `kv_bytes_per_token` (K and V in every block, already in that dtype: 0.5 MiB for the released model in + bf16) for every prefix token -- text and reference latents -- per guidance branch. """ mib, gib = 1024**2, 1024**3 width = element_size / 2 - text_len = max(pos_text_len, neg_text_len or 0) - estimated = int((image_seq_len + text_len) * 0.25 * mib * width) + gib - branches = 2 if do_cfg else 1 - estimated += int(branches * (pos_text_len if neg_text_len is None else text_len) * 0.5 * mib * width) + prefix = max(pos_prefix, neg_prefix or 0) + estimated = int((image_seq_len + prefix) * 0.25 * mib * width) + gib + if use_cache: + branches = 2 if do_cfg else 1 + prefix_tokens = branches * (pos_prefix if neg_prefix is None else prefix) + estimated += prefix_tokens * kv_bytes_per_token return estimated @staticmethod - def _attention_score_bytes(transformer: object, seq_len: int, device: torch.device, dtype: torch.dtype) -> int: - """The score matrix attention materializes on a build without a fused kernel for it (0 where it has one).""" + def _attention_score_bytes(transformer: object, largest_call: int, device: torch.device, dtype: torch.dtype) -> int: + """The score matrix attention materializes on a build without a fused kernel for it (0 where it has one). + + `largest_call` is queries x keys of the largest call, from `largest_attention_call`. + """ config = getattr(transformer, "config", None) num_heads = getattr(config, "num_attention_heads", None) head_dim = getattr(config, "attention_head_dim", None) if not isinstance(num_heads, int) or not isinstance(head_dim, int): return 0 - # The largest call, the image queries over every key, goes unmasked both in the prefill and on decode steps: - # only the text segments carry a mask, and they are as short as the prompt. All of them go through diffusers' - # attention dispatch. + # The largest call is a reference block's or the target's, both unmasked: only the text blocks carry a mask, + # and they are as short as the prompt. All of them go through diffusers' attention dispatch. The square + # helper prices the call at the side with the same area, rounded up. return sdpa_score_matrix_bytes( device=device, dtype=dtype, num_heads=num_heads, head_dim=head_dim, - seq_len=seq_len, + seq_len=math.isqrt(largest_call - 1) + 1 if largest_call > 0 else 0, via_diffusers_dispatch=True, ) diff --git a/invokeai/app/invocations/text_encoder/qwen_image_2_1_text_encoder.py b/invokeai/app/invocations/text_encoder/qwen_image_2_1_text_encoder.py index 0bea270aa49..31b5384f4be 100644 --- a/invokeai/app/invocations/text_encoder/qwen_image_2_1_text_encoder.py +++ b/invokeai/app/invocations/text_encoder/qwen_image_2_1_text_encoder.py @@ -1,56 +1,143 @@ +from contextlib import nullcontext + import torch from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation -from invokeai.app.invocations.fields import FieldDescriptions, Input, InputField, UIComponent -from invokeai.app.invocations.model import Qwen3VLEncoderField +from invokeai.app.invocations.fields import FieldDescriptions, ImageField, Input, InputField, UIComponent +from invokeai.app.invocations.model import ModelIdentifierField, Qwen3VLEncoderField from invokeai.app.invocations.primitives import QwenImage21ConditioningOutput from invokeai.app.services.shared.invocation_context import InvocationContext +from invokeai.backend.architectures import FeaturesFacet, require from invokeai.backend.model_manager.load.model_cache.utils import get_effective_device +from invokeai.backend.model_manager.taxonomy import BaseModelType, ModelFormat, ModelType, SubModelType from invokeai.backend.quantization.dequantizing_linear import peak_dequant_transient_bytes -from invokeai.backend.qwen_image_2_1.text_encoding import encode_prompt +from invokeai.backend.qwen_image_2_1.text_encoding import ( + attached_vision_tower, + encode_prompt, + sequence_length, + vision_attention_bytes, + working_memory_bytes, +) +from invokeai.backend.qwen_image_2_1.vae import to_reference from invokeai.backend.stable_diffusion.diffusion.conditioning_data import ( ConditioningFieldData, QwenImage21ConditioningInfo, ) from invokeai.backend.util.devices import TorchDevice +_NO_VISION_TOWER = ( + "Reference images are read by the Qwen3-VL vision tower, which only the Qwen-Image-2.1 Diffusers model's own " + "encoder brings. Clear the standalone encoder, or use a Qwen-Image-2.1 Diffusers model." +) + @invocation( "qwen_image_2_1_text_encoder", title="Prompt - Qwen-Image-2.1", tags=["prompt", "conditioning", "qwen_image_2_1", "qwen-image-2.1"], category="conditioning", - version="1.0.0", + version="1.1.0", classification=Classification.Prototype, idle_gpu_offloadable=True, ) class QwenImage21TextEncoderInvocation(BaseInvocation): - """Encodes a text prompt for Qwen-Image-2.1 with its Qwen3-VL-8B encoder.""" + """Encodes a prompt for Qwen-Image-2.1 with its Qwen3-VL-8B encoder, with the reference images an edit reads. + + Each reference also goes to the denoise node, encoded by Image to Latents in its reference mode, in the same + order. + """ prompt: str = InputField(description="Text prompt describing the desired image.", ui_component=UIComponent.Textarea) qwen3_vl_encoder: Qwen3VLEncoderField = InputField( title="Qwen3-VL Encoder", description=FieldDescriptions.qwen3_vl_encoder, input=Input.Connection ) + reference_images: list[ImageField] = InputField( + default=[], + description="Reference images to edit or draw from, in the order the denoise node receives their latents, " + f"at most {require(BaseModelType.QwenImage21, FeaturesFacet).max_reference_images}. Needs a Qwen-Image-2.1 " + "Diffusers model's own encoder.", + ) @torch.no_grad() def invoke(self, context: InvocationContext) -> QwenImage21ConditioningOutput: - tokenizer_info = context.models.load(self.qwen3_vl_encoder.tokenizer) + max_references = require(BaseModelType.QwenImage21, FeaturesFacet).max_reference_images + if len(self.reference_images) > max_references: + raise ValueError( + f"Qwen-Image-2.1 reads at most {max_references} reference images; this prompt has " + f"{len(self.reference_images)}. Remove references until {max_references} remain." + ) + images = [to_reference(context.images.get_pil(field.image_name)) for field in self.reference_images] + tower_id, processor_id = self._edit_identifiers(context) if images else (None, None) + + # The encoder first: loading it evicts unlocked small entries from the RAM cache, which would otherwise drop + # the tokenizer, tower and processor loaded after it. text_encoder_info = context.models.load(self.qwen3_vl_encoder.text_encoder) + tokenizer_info = context.models.load(self.qwen3_vl_encoder.tokenizer) + tower_info = context.models.load(tower_id) if tower_id is not None else None + processor_info = context.models.load(processor_id) if processor_id is not None else None + if tower_info is not None and tower_info.compute_device != text_encoder_info.compute_device: + raise RuntimeError( + f"The vision tower runs on {tower_info.compute_device} and the encoder on " + f"{text_encoder_info.compute_device}; an edit attaches one to the other, so they need one device." + ) + compute_dtype = TorchDevice.choose_bfloat16_safe_dtype(text_encoder_info.compute_device) # A GGUF or int8 encoder dequantizes one layer at a time; that transient is the working memory. - dequant_bytes = peak_dequant_transient_bytes( - text_encoder_info.model, TorchDevice.choose_bfloat16_safe_dtype(text_encoder_info.compute_device) + dequant_bytes = peak_dequant_transient_bytes(text_encoder_info.model, compute_dtype) + vision_bytes = ( + vision_attention_bytes(tower_info.model, images, tower_info.compute_device, compute_dtype) + if tower_info is not None + else 0 ) context.util.signal_progress("Running Qwen3-VL text encoder") - with ( - tokenizer_info as tokenizer, - text_encoder_info.model_on_device(working_mem_bytes=dequant_bytes) as ( - _, - text_encoder, - ), - ): - embeds = encode_prompt(text_encoder, tokenizer, self.prompt, get_effective_device(text_encoder)) - - conditioning = QwenImage21ConditioningInfo(prompt_embeds=embeds.detach().cpu()) + with tokenizer_info as tokenizer: + # Reference images make the sequence long, and its attention grows with the square of it: room the + # model cache has to make, or the driver spills the encode to system memory and it crawls. + num_tokens = sequence_length(tokenizer, self.prompt, images) + working_memory = dequant_bytes + working_memory_bytes(num_tokens, compute_dtype.itemsize, vision_bytes) + # The tower locks before the encoder, so the room the encoder reserves is still free once both are on + # the device. + with ( + processor_info.model_on_device() if processor_info is not None else nullcontext((None, None)) as ( + _, + processor, + ), + tower_info.model_on_device() if tower_info is not None else nullcontext((None, None)) as (_, tower), + text_encoder_info.model_on_device(working_mem_bytes=working_memory) as (_, text_encoder), + attached_vision_tower(text_encoder, tower) if tower is not None else nullcontext(), + ): + encoding = encode_prompt( + text_encoder, + tokenizer, + self.prompt, + get_effective_device(text_encoder), + processor=processor, + images=images, + ) + + conditioning = QwenImage21ConditioningInfo( + prompt_embeds=encoding.embeds.detach().cpu(), + image_pad_mask=None if encoding.image_pad_mask is None else encoding.image_pad_mask.cpu(), + reference_grids=encoding.reference_grids, + ) name = context.conditioning.save(ConditioningFieldData(conditionings=[conditioning])) return QwenImage21ConditioningOutput.build(name) + + def _edit_identifiers(self, context: InvocationContext) -> tuple[ModelIdentifierField, ModelIdentifierField]: + """The vision tower and the image processor an edit adds, which only a Qwen-Image-2.1 pipeline brings. + + Checked on the configs, before anything loads: a graph can pair a standalone encoder with this pipeline's + tokenizer, or the other way round. + """ + for identifier in (self.qwen3_vl_encoder.tokenizer, self.qwen3_vl_encoder.text_encoder): + config = context.models.get_config(identifier) + if ( + config.base is not BaseModelType.QwenImage21 + or config.type is not ModelType.Main + or config.format is not ModelFormat.Diffusers + ): + raise ValueError(_NO_VISION_TOWER) + return ( + self.qwen3_vl_encoder.text_encoder.model_copy(update={"submodel_type": SubModelType.VisionTower}), + self.qwen3_vl_encoder.tokenizer.model_copy(update={"submodel_type": SubModelType.Processor}), + ) diff --git a/invokeai/app/invocations/vae/qwen_image_2_1_image_to_latents.py b/invokeai/app/invocations/vae/qwen_image_2_1_image_to_latents.py index 991e744184d..e8aa7d57e9b 100644 --- a/invokeai/app/invocations/vae/qwen_image_2_1_image_to_latents.py +++ b/invokeai/app/invocations/vae/qwen_image_2_1_image_to_latents.py @@ -14,7 +14,13 @@ from invokeai.app.invocations.model import VAEField from invokeai.app.invocations.primitives import LatentsOutput from invokeai.app.services.shared.invocation_context import InvocationContext -from invokeai.backend.qwen_image_2_1.vae import SPATIAL_SCALE, choose_tile_size, to_vae_input, working_memory_bytes +from invokeai.backend.qwen_image_2_1.vae import ( + SPATIAL_SCALE, + choose_tile_size, + to_reference, + to_vae_input, + working_memory_bytes, +) from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.qwen_image_vae import patch_qwen_image_vae_tiling @@ -27,26 +33,41 @@ title="Image to Latents - Qwen-Image-2.1", tags=["image", "latents", "vae", "i2l", "qwen_image_2_1"], category="image", - version="1.0.0", + version="1.1.0", classification=Classification.Prototype, ) class QwenImage21ImageToLatentsInvocation(BaseInvocation, WithMetadata, WithBoard): - """Encodes an image into Qwen-Image-2.1 latents. The image is encoded opaque; its alpha is not read.""" + """Encodes an image into Qwen-Image-2.1 latents. + + For image-to-image and the canvas, the image is encoded opaque at its own size; its alpha is not read. As a + reference for an edit, it is resized to the area the text encoder reads it at and keeps its alpha. + """ image: ImageField = InputField(description="The image to encode.") vae: VAEField = InputField(description=FieldDescriptions.vae, input=Input.Connection) tiled: bool = InputField(default=False, description=FieldDescriptions.tiled) tile_size: int = InputField(default=0, ge=0, multiple_of=16, description=FieldDescriptions.vae_tile_size) + reference: bool = InputField( + default=False, + description="Encode as a reference image for the denoise node's reference latents: resized to ~1 megapixel, " + "as the text encoder reads it, with its alpha kept.", + ) @torch.no_grad() def invoke(self, context: InvocationContext) -> LatentsOutput: image = context.images.get_pil(self.image.image_name) - # Crop-free resize onto the 32px grid the transformer needs, as the canvas sizes its own images. - width, height = image.width // _GRID * _GRID, image.height // _GRID * _GRID - if width == 0 or height == 0: - raise ValueError(f"Qwen-Image-2.1 encodes images of at least {_GRID}x{_GRID}; this one is {image.size}.") - if (width, height) != image.size: - image = image.resize((width, height), resample=PILImage.LANCZOS) + if self.reference: + image = to_reference(image) + width, height = image.size + else: + # Crop-free resize onto the 32px grid the transformer needs, as the canvas sizes its own images. + width, height = image.width // _GRID * _GRID, image.height // _GRID * _GRID + if width == 0 or height == 0: + raise ValueError( + f"Qwen-Image-2.1 encodes images of at least {_GRID}x{_GRID}; this one is {image.size}." + ) + if (width, height) != image.size: + image = image.resize((width, height), resample=PILImage.LANCZOS) vae_info = context.models.load(self.vae.vae) device = vae_info.compute_device @@ -66,12 +87,16 @@ def invoke(self, context: InvocationContext) -> LatentsOutput: working_memory = working_memory_bytes(height, width, tile_size, element_size) with vae_info.model_on_device(working_mem_bytes=working_memory) as (_, vae): - pixels = to_vae_input(image).to(device=device, dtype=vae.dtype).unsqueeze(2) + pixels = to_vae_input(image, keep_alpha=self.reference).to(device=device, dtype=vae.dtype).unsqueeze(2) with torch.inference_mode(), patch_qwen_image_vae_tiling(vae, tile_size, SPATIAL_SCALE): - z = vae.encode(pixels).latent_dist.mode().float()[:, :, 0] - mean = torch.tensor(vae.config.latents_mean, device=z.device).view(1, -1, 1, 1) - std = torch.tensor(vae.config.latents_std, device=z.device).view(1, -1, 1, 1) - latents = ((z - mean) / std).cpu() + z = vae.encode(pixels).latent_dist.mode()[:, :, 0] + # A reference is normalized in the VAE's dtype, as the pipeline normalizes it, so the transformer reads + # the same values; the canvas's latents only seed a sampler, and keep fp32. + z = z if self.reference else z.float() + # float32 first, then the latents' dtype: straight from Python floats to bf16 can round differently. + mean = torch.tensor(vae.config.latents_mean).to(z.device, z.dtype).view(1, -1, 1, 1) + std = torch.tensor(vae.config.latents_std).to(z.device, z.dtype).view(1, -1, 1, 1) + latents = ((z - mean) / std).float().cpu() TorchDevice.empty_cache() name = context.tensors.save(tensor=latents) diff --git a/invokeai/backend/architectures/capabilities.py b/invokeai/backend/architectures/capabilities.py index cbfccd56192..e72665da0d4 100644 --- a/invokeai/backend/architectures/capabilities.py +++ b/invokeai/backend/architectures/capabilities.py @@ -72,6 +72,10 @@ class ArchitectureFeatures(BaseModel): default=None, description="If set, reference images are only accepted for models of this variant.", ) + reference_images_require_format: str | None = Field( + default=None, + description="If set, reference images are only accepted for models of this format.", + ) supports_regional_guidance: bool = False regional_negative: bool = Field( default=False, description="Whether a region's negative prompt is masked, rather than applied globally." @@ -137,6 +141,7 @@ def _features_of(facet: FeaturesFacet, variant: AnyVariant | None = None) -> Arc control_kinds=sorted(facet.control_kinds), max_reference_images=facet.max_reference_images, reference_images_require_variant=facet.reference_images_require_variant, + reference_images_require_format=facet.reference_images_require_format, supports_regional_guidance=facet.supports_regional_guidance, regional_negative=facet.regional_negative, clip_skip_max=facet.clip_skip_max, @@ -171,7 +176,8 @@ def architecture_capabilities() -> list[ArchitectureCapabilities]: is rendered in full, so a client never has to know which fields a variant row is allowed to omit. Differences too small to have earned a mapping are expressed on the base row instead, by - `features.reference_images_require_variant`; Qwen-Image is the only one. + `features.reference_images_require_variant` (Qwen-Image) and + `features.reference_images_require_format` (Qwen-Image-2.1). Sorted by base value, then variant, so the response is stable and diffable. """ diff --git a/invokeai/backend/architectures/defs/qwen_image_2_1.py b/invokeai/backend/architectures/defs/qwen_image_2_1.py index 0a523835091..8dea696ac0b 100644 --- a/invokeai/backend/architectures/defs/qwen_image_2_1.py +++ b/invokeai/backend/architectures/defs/qwen_image_2_1.py @@ -34,6 +34,9 @@ dimension_grid=32, guidance_label="CFG", scheduler_set="flow", + # Each reference holds ~1 megapixel of latents in the prefix the cache keeps: ~2 GiB per guidance branch. + max_reference_images=4, + reference_images_require_format="diffusers", ), # Values are qwen_image_21_* because variant strings resolve without the base and must be unique. VariantFacet({ModelType.Main: QwenImage21VariantType}), diff --git a/invokeai/backend/architectures/facets/features.py b/invokeai/backend/architectures/facets/features.py index f892306bfa4..027f27ad3b4 100644 --- a/invokeai/backend/architectures/facets/features.py +++ b/invokeai/backend/architectures/facets/features.py @@ -121,8 +121,11 @@ class FeaturesFacet(Facet): control_kinds: frozenset[ControlKind] = frozenset() max_reference_images: int = 0 reference_images_require_variant: str | None = None - """Qwen-Image accepts reference images only as the `edit` variant — the one feature in this + """Qwen-Image accepts reference images only as the `edit` variant — one of the two features in this table that a base alone cannot answer.""" + reference_images_require_format: str | None = None + """The other: Qwen-Image-2.1 reads reference images only as a Diffusers pipeline, whose encoder is + the only one that keeps the vision tower.""" supports_regional_guidance: bool = False regional_negative: bool = False diff --git a/invokeai/backend/model_manager/load/load_default.py b/invokeai/backend/model_manager/load/load_default.py index 5f68261da37..1ba0800730f 100644 --- a/invokeai/backend/model_manager/load/load_default.py +++ b/invokeai/backend/model_manager/load/load_default.py @@ -51,6 +51,9 @@ # quality, which fp8 rounding degrades. Its tokenizer is listed for symmetry (it is not an nn.Module, # so casting is a no-op today only by accident). # +# The vision tower follows the text encoder it attaches to. It also could not take the cast as it stands: +# `Qwen3VLModel` casts the pixels to `visual.dtype`, which a cast tower reports as float8. +# # Unlike the model-kind rules in `fp8_capability.py`, these are about which *component* of a model is # being loaded, which no client can see and nothing outside this file asks about. _FP8_EXCLUDED_SUBMODEL_TYPES: frozenset[SubModelType] = frozenset( @@ -58,6 +61,7 @@ SubModelType.TextEncoder, SubModelType.TextEncoder2, SubModelType.TextEncoder3, + SubModelType.VisionTower, SubModelType.Tokenizer, SubModelType.Tokenizer2, SubModelType.Tokenizer3, @@ -342,7 +346,12 @@ def _get_execution_device( if hasattr(config, "default_settings") and config.default_settings is not None: if hasattr(config.default_settings, "cpu_only") and config.default_settings.cpu_only is True: # Only apply CPU execution to text encoder submodels - if submodel_type in [SubModelType.TextEncoder, SubModelType.TextEncoder2, SubModelType.TextEncoder3]: + if submodel_type in [ + SubModelType.TextEncoder, + SubModelType.TextEncoder2, + SubModelType.TextEncoder3, + SubModelType.VisionTower, + ]: return torch.device("cpu") # Check if this is a standalone text encoder config with cpu_only field (T5Encoder, Qwen3Encoder, etc.) diff --git a/invokeai/backend/model_manager/load/model_loaders/qwen_image_2_1.py b/invokeai/backend/model_manager/load/model_loaders/qwen_image_2_1.py index f94637da230..549ee7f5be0 100644 --- a/invokeai/backend/model_manager/load/model_loaders/qwen_image_2_1.py +++ b/invokeai/backend/model_manager/load/model_loaders/qwen_image_2_1.py @@ -1,5 +1,7 @@ """Loaders for Qwen-Image-2.1 main models: diffusers pipelines, single-file and GGUF transformers.""" +import json +import math from pathlib import Path from typing import Any, Optional @@ -62,12 +64,29 @@ def _build_transformer(sd: dict[str, Any]) -> torch.nn.Module: class QwenImage21DiffusersModel(GenericDiffusersLoader): """Loads the submodels of a Qwen-Image-2.1 diffusers pipeline. - The tokenizer lives in `processor/` (the pipeline declares a `Qwen3VLProcessor`, of which text prompts - need only the tokenizer). The text encoder loads as `Qwen3VLModel` rather than the declared - `Qwen3VLForConditionalGeneration`: the LM head is ~1.2 GiB the encoder never runs, and `Qwen3VLModel` is - what the standalone Qwen3-VL loaders return, so the text encoder node sees one class. + The tokenizer lives in `processor/` (the pipeline declares a `Qwen3VLProcessor`); the processor itself, which + also cuts reference images into patches, loads only for an edit. The text encoder loads as `Qwen3VLModel` + rather than the declared `Qwen3VLForConditionalGeneration`: the LM head is ~1.2 GiB the encoder never runs, + and `Qwen3VLModel` is what the standalone Qwen3-VL loaders return, so the text encoder node sees one class. + + `TextEncoder` drops the vision tower, as the standalone loaders do: text prompts never reach it. + `VisionTower` is that tower alone (~1.1 GiB), read from the same folder for edits, which attach it to the + cached encoder while they encode. The language model's weights are cached once for both. """ + def get_size_fs( + self, config: AnyModelConfig, model_path: Path, submodel_type: Optional[SubModelType] = None + ) -> int: + if submodel_type is SubModelType.VisionTower: + # A few hundred tensors of the encoder's folder, not the folder: reserving its ~17 GB for the tower + # would evict models for nothing. Sized at the dtype they load in, whatever the file stores. + element_size = TorchDevice.choose_bfloat16_safe_dtype(TorchDevice.choose_torch_device()).itemsize + return sum( + math.prod(shape) * element_size + for _shard, _key, shape in _vision_tower_tensors(model_path / SubModelType.TextEncoder.value) + ) + return super().get_size_fs(config, model_path, submodel_type) + def _load_model(self, config: AnyModelConfig, submodel_type: Optional[SubModelType] = None) -> AnyModel: if isinstance(config, Checkpoint_Config_Base): raise NotImplementedError( @@ -79,6 +98,10 @@ def _load_model(self, config: AnyModelConfig, submodel_type: Optional[SubModelTy model_path = Path(config.path) if submodel_type is SubModelType.Tokenizer: return AutoTokenizer.from_pretrained(model_path / "processor", local_files_only=True) + if submodel_type is SubModelType.Processor: + from transformers import AutoProcessor + + return AutoProcessor.from_pretrained(model_path / "processor", local_files_only=True) if submodel_type is SubModelType.VAE: # The configured precision, except float16, which is unstable on this autoencoder: bfloat16 where the @@ -86,6 +109,11 @@ def _load_model(self, config: AnyModelConfig, submodel_type: Optional[SubModelTy dtype = self._torch_dtype_avoiding_float16() else: dtype = TorchDevice.choose_bfloat16_safe_dtype(TorchDevice.choose_torch_device()) + + if submodel_type is SubModelType.VisionTower: + tower = _load_vision_tower(model_path / SubModelType.TextEncoder.value, dtype) + return self._apply_fp8_layerwise_casting(tower, config, submodel_type) + repo_variant = config.repo_variant if isinstance(config, Diffusers_Config_Base) else None variant = repo_variant.value if repo_variant else None if submodel_type is SubModelType.TextEncoder: @@ -109,12 +137,64 @@ def _load_model(self, config: AnyModelConfig, submodel_type: Optional[SubModelTy raise if submodel_type is SubModelType.TextEncoder: - # Text prompts never reach the vision tower; reference images (which would) are not wired up. + # Text prompts never reach the vision tower; edits load it as `VisionTower` and attach it while they + # encode. drop_qwen3vl_visual_tower(result) return self._apply_fp8_layerwise_casting(result, config, submodel_type) +# The tower's tensors in an encoder folder: `model.visual.*` as `Qwen3VLForConditionalGeneration` saves them (the +# released pipeline), `visual.*` as `Qwen3VLModel` does. +_VISION_TOWER_PREFIXES = ("model.visual.", "visual.") + + +def _vision_tower_tensors(folder: Path) -> list[tuple[Path, str, list[int]]]: + """(shard, key, shape) of every vision-tower tensor in the folder, read from the headers. + + The shard index names the shards that hold them; without an index, every safetensors file in the folder. + """ + from safetensors import safe_open + + index = folder / "model.safetensors.index.json" + if index.exists(): + weight_map: dict[str, str] = json.loads(index.read_text(encoding="utf-8"))["weight_map"] + shards = sorted({folder / shard for key, shard in weight_map.items() if key.startswith(_VISION_TOWER_PREFIXES)}) + else: + shards = sorted(folder.glob("*.safetensors")) + tensors = [] + for shard in shards: + with safe_open(shard, framework="pt", device="cpu") as f: + for key in f.keys(): + if key.startswith(_VISION_TOWER_PREFIXES): + tensors.append((shard, key, f.get_slice(key).get_shape())) + if not tensors: + raise ValueError(f"{folder} holds no Qwen3-VL vision tower ({' or '.join(_VISION_TOWER_PREFIXES)} tensors).") + return tensors + + +def _load_vision_tower(folder: Path, dtype: torch.dtype) -> torch.nn.Module: + """The encoder's vision tower alone, as `Qwen3VLModel.visual` would hold it, without reading the language model.""" + from safetensors import safe_open + from transformers import AutoConfig + from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLVisionModel + + vision_config = AutoConfig.from_pretrained(folder, local_files_only=True).vision_config + with accelerate.init_empty_weights(): + tower = Qwen3VLVisionModel._from_config(vision_config, torch_dtype=dtype) + + state_dict: dict[str, torch.Tensor] = {} + tensors = _vision_tower_tensors(folder) + for shard in sorted({shard for shard, *_ in tensors}): + with safe_open(shard, framework="pt", device="cpu") as f: + for _shard, key, _shape in (t for t in tensors if t[0] == shard): + prefix = next(p for p in _VISION_TOWER_PREFIXES if key.startswith(p)) + state_dict[key[len(prefix) :]] = f.get_tensor(key).to(dtype) + load_state_dict_ignoring_extras(tower, state_dict, source=f"Qwen3-VL vision tower in {folder}", assign=True) + reject_incomplete_load(tower, what="Qwen3-VL vision tower") + return tower.eval() + + @ModelLoaderRegistry.register(base=BaseModelType.QwenImage21, type=ModelType.Main, format=ModelFormat.Checkpoint) class QwenImage21CheckpointModel(ModelLoader): """Loads a Qwen-Image-2.1 transformer from a single safetensors file. diff --git a/invokeai/backend/model_manager/taxonomy.py b/invokeai/backend/model_manager/taxonomy.py index 3a328b436ed..4a45c175c52 100644 --- a/invokeai/backend/model_manager/taxonomy.py +++ b/invokeai/backend/model_manager/taxonomy.py @@ -123,6 +123,9 @@ class SubModelType(str, Enum): TextEncoder = "text_encoder" TextEncoder2 = "text_encoder_2" TextEncoder3 = "text_encoder_3" + # A vision-language encoder's vision tower alone, which the text encoder is loaded without; edits attach it + # while they encode, so the language model is cached once. Qwen-Image-2.1 only. + VisionTower = "vision_tower" Tokenizer = "tokenizer" Tokenizer2 = "tokenizer_2" Tokenizer3 = "tokenizer_3" diff --git a/invokeai/backend/qwen_image_2_1/text_encoding.py b/invokeai/backend/qwen_image_2_1/text_encoding.py index 45f95084271..bb1a80c7b31 100644 --- a/invokeai/backend/qwen_image_2_1/text_encoding.py +++ b/invokeai/backend/qwen_image_2_1/text_encoding.py @@ -1,17 +1,84 @@ -"""Qwen-Image-2.1's prompt encoding, as `QwenImage21Pipeline._get_qwen_prompt_embeds` does it for text prompts.""" +"""Qwen-Image-2.1's prompt encoding, as `QwenImage21Pipeline._get_qwen_prompt_embeds` does it.""" + +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from typing import NamedTuple import torch +from PIL import Image + +from invokeai.backend.util.attention import sdpa_score_matrix_bytes SYSTEM_PROMPT = "Comprehend and analyze the provided prompt." SYSTEM_BLOCK = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n" # A raw string handed to the tokenizer, not `apply_chat_template`: the two tokenize differently, and the # checkpoint was trained on this one. T2I_TEMPLATE = SYSTEM_BLOCK + "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" +IMAGE_PAD = "<|image_pad|>" +# Pixels per side of one vision token: 16px patches, merged 2x2. +_PIXELS_PER_TOKEN = 32 +# The vision tower reads each reference's 16px patches before the 2x2 merge: four per vision token. +_PATCHES_PER_TOKEN = 4 + +# What one encode needs above the weights for a sequence of T tokens, measured on an RTX 4090 in bf16 (torch 2.13, +# transformers 5.10): ~286 B * T^2 + 224 KiB * T at peak. The quadratic term is the language model's attention, +# which SDPA runs on its math kernel for grouped-query heads; four 1-megapixel references (~4.1k tokens) need +# ~5.4 GiB. Both terms are activations, so they scale with the compute dtype's width. The allocator reserves about +# a fifth more than it hands out. +_BYTES_PER_TOKEN_PAIR = 286 +_BYTES_PER_TOKEN = 224 * 1024 +_RESERVED_OVER_ALLOCATED = 1.25 -def format_prompt(prompt: str) -> str: +def format_prompt(prompt: str, num_images: int = 0) -> str: + """The prompt as the checkpoint was trained on it, with one vision placeholder per reference image first.""" + if num_images and IMAGE_PAD in prompt: + # The processor expands every placeholder into one image's tokens, so a literal one would claim an image + # that is not there. + raise ValueError(f"A prompt that reads reference images cannot contain {IMAGE_PAD}. Remove it from the prompt.") + placeholders = " ".join(f"<|vision_start|>{IMAGE_PAD}<|vision_end|>" for i in range(1, num_images + 1)) # Qwen has no BOS token, so an empty prompt would leave the encoder nothing to read. - return T2I_TEMPLATE.format(prompt or " ") + return T2I_TEMPLATE.format(placeholders + (prompt or " ")) + + +def sequence_length(tokenizer, prompt: str, images: Sequence[Image.Image] = ()) -> int: + """Tokens the encoder reads: the formatted prompt, each reference's placeholder one token per 32x32 pixels. + + `images` are the references at `vae.reference_size`, which the image processor leaves at their size. + """ + text = len(tokenizer(format_prompt(prompt, len(images))).input_ids) - len(images) + return text + sum((w // _PIXELS_PER_TOKEN) * (h // _PIXELS_PER_TOKEN) for w, h in (i.size for i in images)) + + +def vision_attention_bytes( + tower: torch.nn.Module, images: Sequence[Image.Image], device: torch.device, dtype: torch.dtype +) -> int: + """The score matrix the vision tower's attention materializes for the largest reference, 0 where it is fused. + + The tower attends within one image at a time, over its patches: four per vision token, ~4k for 1 megapixel. + """ + if not images: + return 0 + patches = max((w // _PIXELS_PER_TOKEN) * (h // _PIXELS_PER_TOKEN) for w, h in (i.size for i in images)) + return sdpa_score_matrix_bytes( + device=device, + dtype=dtype, + num_heads=tower.config.num_heads, + head_dim=tower.config.hidden_size // tower.config.num_heads, + seq_len=_PATCHES_PER_TOKEN * patches, + ) + + +def working_memory_bytes(num_tokens: int, element_size: int, vision_attention: int = 0) -> int: + """Device memory an encode of `num_tokens` tokens needs above the encoder's weights. + + `element_size` is the compute dtype's width. `vision_attention` is the tower's score matrix + (`vision_attention_bytes`): the tower runs, and frees it, before the language model starts, so only the larger + of the two attention peaks counts. + """ + width = element_size / 2 + attention = max(_BYTES_PER_TOKEN_PAIR * num_tokens**2 * width, vision_attention) + return int((attention + _BYTES_PER_TOKEN * num_tokens * width) * _RESERVED_OVER_ALLOCATED) def system_prefix_length(tokenizer) -> int: @@ -29,20 +96,100 @@ def _base_model(text_encoder: torch.nn.Module) -> torch.nn.Module: return text_encoder if hasattr(text_encoder, "language_model") else text_encoder.model +def has_vision_tower(text_encoder: torch.nn.Module) -> bool: + """Whether this encoder can read reference images: whether a vision tower is attached (`attached_vision_tower`). + + Every encoder loads with an `Identity` where the tower was. + """ + visual = getattr(_base_model(text_encoder), "visual", None) + return isinstance(visual, torch.nn.Module) and not isinstance(visual, torch.nn.Identity) + + +@contextmanager +def attached_vision_tower(text_encoder: torch.nn.Module, tower: torch.nn.Module) -> Iterator[None]: + """Attach the pipeline's vision tower to its cached encoder for one encode. + + The encoder and the tower are separate model-cache entries, so the language model is cached once for text + prompts and edits. Both must be locked on the same device for the whole block. Whatever the block does, the + encoder leaves it with the module it had, so the cache never holds an encoder with a tower it does not own. + """ + base = _base_model(text_encoder) + previous = base.visual + base.visual = tower + try: + yield + finally: + base.visual = previous + + +def on_white(image: Image.Image) -> Image.Image: + """A reference as the vision encoder reads it: alpha composited over white, as the checkpoint was trained.""" + if image.mode != "RGBA": + return image.convert("RGB") + white = Image.new("RGB", image.size, (255, 255, 255)) + white.paste(image, mask=image.getchannel("A")) + return white + + +class PromptEncoding(NamedTuple): + embeds: torch.Tensor + """`(1, L, hidden)`, the system turn dropped.""" + image_pad_mask: torch.Tensor | None + """`(1, L)`, True at each `<|image_pad|>` slot, where the transformer places 2x2 of a reference's latents.""" + reference_grids: tuple[tuple[int, int], ...] + """Per reference, in order: its slots as (rows, columns), one slot per 32x32 pixels.""" + + @torch.no_grad() -def encode_prompt(text_encoder: torch.nn.Module, tokenizer, prompt: str, device: torch.device) -> torch.Tensor: - """Encode a prompt into `(1, L, hidden)` embeddings, as the pipeline encodes a single prompt. +def encode_prompt( + text_encoder: torch.nn.Module, + tokenizer, + prompt: str, + device: torch.device, + *, + processor=None, + images: Sequence[Image.Image] = (), +) -> PromptEncoding: + """Encode a prompt, and the reference images it edits. The transformer reads the last decoder layer's output BEFORE the final RMSNorm. transformers 5 ties the last hidden state to the normalized output, so a forward hook returning the norm's input neutralizes the norm for this call -- the pipeline's approach. The system turn every prompt shares is dropped from the result. + + `images` are the references, already at `vae.reference_size`; reading them needs the vision tower and the + `processor` that cuts them into patches. """ - inputs = tokenizer([format_prompt(prompt)], return_tensors="pt").to(device) base = _base_model(text_encoder) + if images: + if processor is None or not has_vision_tower(text_encoder): + raise ValueError("Reading reference images needs the Qwen3-VL vision tower and its image processor.") + inputs = processor( + text=[format_prompt(prompt, len(images))], images=[on_white(i) for i in images], return_tensors="pt" + ).to(device) + forward_kwargs = { + "input_ids": inputs.input_ids, + "attention_mask": inputs.attention_mask, + "pixel_values": inputs.pixel_values, + "image_grid_thw": inputs.image_grid_thw, + } + if "mm_token_type_ids" in inputs: + forward_kwargs["mm_token_type_ids"] = inputs.mm_token_type_ids + else: + inputs = tokenizer([format_prompt(prompt)], return_tensors="pt").to(device) + forward_kwargs = {"input_ids": inputs.input_ids, "attention_mask": inputs.attention_mask} + handle = base.language_model.norm.register_forward_hook(lambda module, args, output: args[0]) try: - hidden = base(input_ids=inputs.input_ids, attention_mask=inputs.attention_mask).last_hidden_state + # One forward, so the KV cache the language model would build over the whole sequence is waste. + hidden = base(**forward_kwargs, use_cache=False).last_hidden_state finally: handle.remove() - return hidden[:, system_prefix_length(tokenizer) :] + + drop = system_prefix_length(tokenizer) + if not images: + return PromptEncoding(hidden[:, drop:], None, ()) + image_pad_id = tokenizer.convert_tokens_to_ids(IMAGE_PAD) + merge = processor.image_processor.merge_size + grids = tuple((int(h) // merge, int(w) // merge) for _, h, w in inputs.image_grid_thw.tolist()) + return PromptEncoding(hidden[:, drop:], (inputs.input_ids == image_pad_id)[:, drop:], grids) diff --git a/invokeai/backend/qwen_image_2_1/vae.py b/invokeai/backend/qwen_image_2_1/vae.py index f787f2bb445..4fe48a69c90 100644 --- a/invokeai/backend/qwen_image_2_1/vae.py +++ b/invokeai/backend/qwen_image_2_1/vae.py @@ -4,6 +4,8 @@ and Wan VAEs (same tiling attributes, 16x instead of 8x). """ +import math + import numpy as np import torch from PIL import Image @@ -96,11 +98,38 @@ def to_image(decoded: torch.Tensor) -> Image.Image: return Image.fromarray(pixels, mode="RGBA") -def to_vae_input(image: Image.Image) -> torch.Tensor: - """An image as the VAE's `(1, 4, H, W)` input in [-1, 1], encoded opaque. +def to_vae_input(image: Image.Image, keep_alpha: bool = False) -> torch.Tensor: + """An image as the VAE's `(1, 4, H, W)` input in [-1, 1]. - Alpha is set to 1 rather than read from the image: the canvas hands its masks over in alpha, and a - transparent region there means "repaint", not "transparent pixels". + For image-to-image and the canvas, alpha is set to 1 rather than read from the image: the canvas hands its masks + over in alpha, and a transparent region there means "repaint", not "transparent pixels". A reference image keeps + its alpha (`keep_alpha`), as the pipeline encodes it. """ - rgb = torch.from_numpy(np.asarray(image.convert("RGB"), dtype=np.float32)).permute(2, 0, 1) / 127.5 - 1 - return torch.cat([rgb, torch.ones_like(rgb[:1])], dim=0)[None] + rgba = image.convert("RGBA") + # In the pipeline's `VaeImageProcessor` order, so the result matches it to the bit. + pixels = torch.from_numpy(np.asarray(rgba).astype(np.float32) / 255.0).permute(2, 0, 1) * 2.0 - 1.0 + if not keep_alpha: + pixels[3] = 1.0 + return pixels[None] + + +# The area every reference image is resized to, as the pipeline's default `output_resolution` sets it. +REFERENCE_AREA = 1024 * 1024 + + +def reference_size(width: int, height: int) -> tuple[int, int]: + """The size a `width` x `height` reference is read at: `REFERENCE_AREA` at its aspect ratio, on the 32px grid. + + The text encoder and the VAE must both read it at this size: each `<|image_pad|>` slot the encoder emits + stands for 2x2 of the reference's latents, and the transformer drops the latents into those slots. + """ + ratio = width / height + scaled_width = math.sqrt(REFERENCE_AREA * ratio) + # Python's round (half to even) is the pipeline's `calculate_dimensions`. + return round(scaled_width / 32) * 32, round(scaled_width / ratio / 32) * 32 + + +def to_reference(image: Image.Image) -> Image.Image: + """A reference image as both the text encoder and the VAE read it: RGBA, resized to `reference_size`.""" + rgba = image.convert("RGBA") + return rgba.resize(reference_size(*rgba.size), resample=Image.LANCZOS) diff --git a/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py b/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py index 04591b0f238..43c24a6d11a 100644 --- a/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py +++ b/invokeai/backend/stable_diffusion/diffusion/conditioning_data.py @@ -166,8 +166,18 @@ class QwenImage21ConditioningInfo: prompt_embeds: torch.Tensor """Shape: (1, seq_len, hidden=4096). One prompt, so no padding and no mask.""" + image_pad_mask: torch.Tensor | None = None + """Shape: (1, seq_len), True at the reference images' `<|image_pad|>` slots; None without references. Each + slot stands for 2x2 latent tokens of a reference, in the order the references were encoded.""" + + reference_grids: tuple[tuple[int, int], ...] = () + """Per reference, in order: its slots as (rows, columns), so the denoise node can match each reference's + latents by shape, not only by count.""" + def to(self, device: torch.device | None = None, dtype: torch.dtype | None = None): self.prompt_embeds = self.prompt_embeds.to(device=device, dtype=dtype) + if self.image_pad_mask is not None: + self.image_pad_mask = self.image_pad_mask.to(device=device) return self diff --git a/invokeai/frontend/api/openapi.json b/invokeai/frontend/api/openapi.json index cb42543e7ee..0ecd08049f0 100644 --- a/invokeai/frontend/api/openapi.json +++ b/invokeai/frontend/api/openapi.json @@ -20796,6 +20796,18 @@ "title": "Reference Images Require Variant", "description": "If set, reference images are only accepted for models of this variant." }, + "reference_images_require_format": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reference Images Require Format", + "description": "If set, reference images are only accepted for models of this format." + }, "supports_regional_guidance": { "type": "boolean", "title": "Supports Regional Guidance", @@ -57777,7 +57789,7 @@ "$ref": "#/components/schemas/PromptTemplateOutput" }, "qwen_image_2_1_denoise": { - "$ref": "#/components/schemas/LatentsOutput" + "$ref": "#/components/schemas/LatentsMetaOutput" }, "qwen_image_2_1_i2l": { "$ref": "#/components/schemas/LatentsOutput" @@ -96264,6 +96276,29 @@ "orig_default": null, "orig_required": false }, + "reference_latents": { + "anyOf": [ + { + "$ref": "#/components/schemas/LatentsField" + }, + { + "items": { + "$ref": "#/components/schemas/LatentsField" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Latents of the reference images an edit reads, from Image to Latents in reference mode, in the order the prompts were encoded with them.", + "field_kind": "input", + "input": "connection", + "orig_default": null, + "orig_required": false, + "title": "Reference Latents" + }, "denoising_start": { "default": 0.0, "description": "When to start denoising, expressed a percentage of total steps", @@ -96412,16 +96447,16 @@ "tags": ["image", "qwen_image_2_1", "qwen-image-2.1"], "title": "Denoise - Qwen-Image-2.1", "type": "object", - "version": "1.0.1", + "version": "1.1.0", "output": { - "$ref": "#/components/schemas/LatentsOutput" + "$ref": "#/components/schemas/LatentsMetaOutput" } }, "QwenImage21ImageToLatentsInvocation": { "category": "image", "class": "invocation", "classification": "prototype", - "description": "Encodes an image into Qwen-Image-2.1 latents. The image is encoded opaque; its alpha is not read.", + "description": "Encodes an image into Qwen-Image-2.1 latents.\n\nFor image-to-image and the canvas, the image is encoded opaque at its own size; its alpha is not read. As a\nreference for an edit, it is resized to the area the text encoder reads it at and keeps its alpha.", "node_pack": "invokeai", "properties": { "board": { @@ -96532,6 +96567,16 @@ "title": "Tile Size", "type": "integer" }, + "reference": { + "default": false, + "description": "Encode as a reference image for the denoise node's reference latents: resized to ~1 megapixel, as the text encoder reads it, with its alpha kept.", + "field_kind": "input", + "input": "any", + "orig_default": false, + "orig_required": false, + "title": "Reference", + "type": "boolean" + }, "type": { "const": "qwen_image_2_1_i2l", "default": "qwen_image_2_1_i2l", @@ -96544,7 +96589,7 @@ "tags": ["image", "latents", "vae", "i2l", "qwen_image_2_1"], "title": "Image to Latents - Qwen-Image-2.1", "type": "object", - "version": "1.0.0", + "version": "1.1.0", "output": { "$ref": "#/components/schemas/LatentsOutput" } @@ -96818,7 +96863,7 @@ "category": "conditioning", "class": "invocation", "classification": "prototype", - "description": "Encodes a text prompt for Qwen-Image-2.1 with its Qwen3-VL-8B encoder.", + "description": "Encodes a prompt for Qwen-Image-2.1 with its Qwen3-VL-8B encoder, with the reference images an edit reads.\n\nEach reference also goes to the denoise node, encoded by Image to Latents in its reference mode, in the same\norder.", "node_pack": "invokeai", "properties": { "id": { @@ -96878,6 +96923,19 @@ "orig_required": true, "title": "Qwen3-VL Encoder" }, + "reference_images": { + "default": [], + "description": "Reference images to edit or draw from, in the order the denoise node receives their latents, at most 4. Needs a Qwen-Image-2.1 Diffusers model's own encoder.", + "field_kind": "input", + "input": "any", + "items": { + "$ref": "#/components/schemas/ImageField" + }, + "orig_default": [], + "orig_required": false, + "title": "Reference Images", + "type": "array" + }, "type": { "const": "qwen_image_2_1_text_encoder", "default": "qwen_image_2_1_text_encoder", @@ -96890,7 +96948,7 @@ "tags": ["prompt", "conditioning", "qwen_image_2_1", "qwen-image-2.1"], "title": "Prompt - Qwen-Image-2.1", "type": "object", - "version": "1.0.0", + "version": "1.1.0", "output": { "$ref": "#/components/schemas/QwenImage21ConditioningOutput" } @@ -99696,7 +99754,7 @@ "type": "object", "required": ["image_name"], "title": "ReferenceImageRecallParameter", - "description": "Global reference-image configuration for recall.\n\nUsed for reference images that feed directly into the main model rather\nthan through a separate IP-Adapter / ControlNet model \u2014 for example\nFLUX.2 Klein, FLUX Kontext, and Qwen Image Edit. The receiving frontend\npicks the correct config type (``flux2_reference_image`` /\n``qwen_image_reference_image`` / ``flux_kontext_reference_image``) based\non the currently-selected main model." + "description": "Global reference-image configuration for recall.\n\nUsed for reference images that feed directly into the main model rather\nthan through a separate IP-Adapter / ControlNet model \u2014 for example\nFLUX.2 Klein, FLUX Kontext, Qwen Image Edit and Qwen-Image-2.1. The\nreceiving frontend picks the correct config type (``flux2_reference_image`` /\n``qwen_image_reference_image`` / ``qwen_image_2_1_reference_image`` /\n``flux_kontext_reference_image``) based on the currently-selected main model." }, "RemoteModelFile": { "properties": { @@ -106071,6 +106129,7 @@ "text_encoder", "text_encoder_2", "text_encoder_3", + "vision_tower", "tokenizer", "tokenizer_2", "tokenizer_3", diff --git a/invokeai/frontend/api/schema.ts b/invokeai/frontend/api/schema.ts index ce3e700a895..ffa15f76713 100644 --- a/invokeai/frontend/api/schema.ts +++ b/invokeai/frontend/api/schema.ts @@ -5778,6 +5778,11 @@ export type components = { * @description If set, reference images are only accepted for models of this variant. */ reference_images_require_variant?: string | null; + /** + * Reference Images Require Format + * @description If set, reference images are only accepted for models of this format. + */ + reference_images_require_format?: string | null; /** * Supports Regional Guidance * @default false @@ -22993,7 +22998,7 @@ export type components = { pidi_edge_detection: components["schemas"]["ImageOutput"]; prompt_from_file: components["schemas"]["StringCollectionOutput"]; prompt_template: components["schemas"]["PromptTemplateOutput"]; - qwen_image_2_1_denoise: components["schemas"]["LatentsOutput"]; + qwen_image_2_1_denoise: components["schemas"]["LatentsMetaOutput"]; qwen_image_2_1_i2l: components["schemas"]["LatentsOutput"]; qwen_image_2_1_l2i: components["schemas"]["ImageOutput"]; qwen_image_2_1_model_loader: components["schemas"]["QwenImage21ModelLoaderOutput"]; @@ -41591,6 +41596,12 @@ export type components = { * @default null */ denoise_mask?: components["schemas"]["DenoiseMaskField"] | null; + /** + * Reference Latents + * @description Latents of the reference images an edit reads, from Image to Latents in reference mode, in the order the prompts were encoded with them. + * @default null + */ + reference_latents?: components["schemas"]["LatentsField"] | components["schemas"]["LatentsField"][] | null; /** * Denoising Start * @description When to start denoising, expressed a percentage of total steps @@ -41658,7 +41669,10 @@ export type components = { }; /** * Image to Latents - Qwen-Image-2.1 - * @description Encodes an image into Qwen-Image-2.1 latents. The image is encoded opaque; its alpha is not read. + * @description Encodes an image into Qwen-Image-2.1 latents. + * + * For image-to-image and the canvas, the image is encoded opaque at its own size; its alpha is not read. As a + * reference for an edit, it is resized to the area the text encoder reads it at and keeps its alpha. */ QwenImage21ImageToLatentsInvocation: { /** @@ -41710,6 +41724,12 @@ export type components = { * @default 0 */ tile_size?: number; + /** + * Reference + * @description Encode as a reference image for the denoise node's reference latents: resized to ~1 megapixel, as the text encoder reads it, with its alpha kept. + * @default false + */ + reference?: boolean; /** * type * @default qwen_image_2_1_i2l @@ -41856,7 +41876,10 @@ export type components = { }; /** * Prompt - Qwen-Image-2.1 - * @description Encodes a text prompt for Qwen-Image-2.1 with its Qwen3-VL-8B encoder. + * @description Encodes a prompt for Qwen-Image-2.1 with its Qwen3-VL-8B encoder, with the reference images an edit reads. + * + * Each reference also goes to the denoise node, encoded by Image to Latents in its reference mode, in the same + * order. */ QwenImage21TextEncoderInvocation: { /** @@ -41888,6 +41911,12 @@ export type components = { * @default null */ qwen3_vl_encoder?: components["schemas"]["Qwen3VLEncoderField"] | null; + /** + * Reference Images + * @description Reference images to edit or draw from, in the order the denoise node receives their latents, at most 4. Needs a Qwen-Image-2.1 Diffusers model's own encoder. + * @default [] + */ + reference_images?: components["schemas"]["ImageField"][]; /** * type * @default qwen_image_2_1_text_encoder @@ -43346,10 +43375,10 @@ export type components = { * * Used for reference images that feed directly into the main model rather * than through a separate IP-Adapter / ControlNet model — for example - * FLUX.2 Klein, FLUX Kontext, and Qwen Image Edit. The receiving frontend - * picks the correct config type (``flux2_reference_image`` / - * ``qwen_image_reference_image`` / ``flux_kontext_reference_image``) based - * on the currently-selected main model. + * FLUX.2 Klein, FLUX Kontext, Qwen Image Edit and Qwen-Image-2.1. The + * receiving frontend picks the correct config type (``flux2_reference_image`` / + * ``qwen_image_reference_image`` / ``qwen_image_2_1_reference_image`` / + * ``flux_kontext_reference_image``) based on the currently-selected main model. */ ReferenceImageRecallParameter: { /** @@ -46578,7 +46607,7 @@ export type components = { * @description Submodel type. * @enum {string} */ - SubModelType: "unet" | "transformer" | "transformer_2" | "text_encoder" | "text_encoder_2" | "text_encoder_3" | "tokenizer" | "tokenizer_2" | "tokenizer_3" | "processor" | "pe" | "pe_tokenizer" | "vae" | "vae_decoder" | "vae_encoder" | "audio_vae" | "vocoder" | "connectors" | "latent_upsampler" | "temporal_latent_upsampler" | "scheduler" | "safety_checker"; + SubModelType: "unet" | "transformer" | "transformer_2" | "text_encoder" | "text_encoder_2" | "text_encoder_3" | "vision_tower" | "tokenizer" | "tokenizer_2" | "tokenizer_3" | "processor" | "pe" | "pe_tokenizer" | "vae" | "vae_decoder" | "vae_encoder" | "audio_vae" | "vocoder" | "connectors" | "latent_upsampler" | "temporal_latent_upsampler" | "scheduler" | "safety_checker"; /** SubmodelDefinition */ SubmodelDefinition: { /** Path Or Prefix */ diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index 434381d92df..e7891aa216b 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -2837,6 +2837,7 @@ "referenceImage": "Reference Image", "referenceImageModel": "Reference image model", "referenceImages": "Reference images", + "referenceImagesNeedFormat": "Reference images need a {{format}} {{base}} model.", "referenceImagesUnsupported": "Reference images are not supported by the selected model.", "removeConcept": "Remove concept", "removeConceptNamed": "Remove {{name}}", diff --git a/invokeai/frontend/webv2/src/features/generation/core/__fixtures__/architectureCapabilities.json b/invokeai/frontend/webv2/src/features/generation/core/__fixtures__/architectureCapabilities.json index 011566f5018..8d13ff2f462 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/__fixtures__/architectureCapabilities.json +++ b/invokeai/frontend/webv2/src/features/generation/core/__fixtures__/architectureCapabilities.json @@ -18,6 +18,7 @@ ], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -76,6 +77,7 @@ ], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -132,6 +134,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -173,6 +176,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -224,6 +228,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -268,6 +273,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -312,6 +318,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -355,6 +362,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -398,6 +406,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -441,6 +450,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -484,6 +494,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -525,6 +536,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -573,6 +585,7 @@ "control_kinds": [], "max_reference_images": 1, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -625,6 +638,7 @@ "control_kinds": [], "max_reference_images": 1, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -677,6 +691,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -718,6 +733,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -759,6 +775,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -802,6 +819,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": "edit", + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -852,8 +870,9 @@ "scheduler_set": "flow", "scheduler_applies_to_graph": false, "control_kinds": [], - "max_reference_images": 0, + "max_reference_images": 4, "reference_images_require_variant": null, + "reference_images_require_format": "diffusers", "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -893,8 +912,9 @@ "scheduler_set": "flow", "scheduler_applies_to_graph": false, "control_kinds": [], - "max_reference_images": 0, + "max_reference_images": 4, "reference_images_require_variant": null, + "reference_images_require_format": "diffusers", "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -939,6 +959,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": true, "clip_skip_max": 12, @@ -980,6 +1001,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": true, "clip_skip_max": 24, @@ -1021,6 +1043,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -1065,6 +1088,7 @@ ], "max_reference_images": 5, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": true, "clip_skip_max": null, @@ -1106,6 +1130,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -1147,6 +1172,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -1195,6 +1221,7 @@ "control_kinds": [], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": false, "regional_negative": false, "clip_skip_max": null, @@ -1245,6 +1272,7 @@ ], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, @@ -1295,6 +1323,7 @@ ], "max_reference_images": 0, "reference_images_require_variant": null, + "reference_images_require_format": null, "supports_regional_guidance": true, "regional_negative": false, "clip_skip_max": null, diff --git a/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json index 6d79322aa91..80aac9dfcd9 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json +++ b/invokeai/frontend/webv2/src/features/generation/core/__snapshots__/generateGraphNodeTypes.json @@ -265,11 +265,14 @@ ] }, "nodeTypes": [ + "collect", "core_metadata", + "image", "img_nsfw", "img_watermark", "integer", "qwen_image_2_1_denoise", + "qwen_image_2_1_i2l", "qwen_image_2_1_l2i", "qwen_image_2_1_model_loader", "qwen_image_2_1_text_encoder", @@ -1252,6 +1255,21 @@ "conditioning" ] }, + "image": { + "inputs": [], + "literalInputs": { + "image": [], + "is_intermediate": [ + true + ], + "use_cache": [ + true + ] + }, + "outputs": [ + "image" + ] + }, "img_nsfw": { "inputs": [ "image" @@ -1463,13 +1481,17 @@ }, "qwen_image_2_1_denoise": { "inputs": [ + "metadata", + "negative_conditioning", "positive_conditioning", + "reference_latents", "seed", "transformer" ], "literalInputs": { "cfg_scale": [ - 1 + 1, + 4 ], "height": [ 1024 @@ -1487,6 +1509,27 @@ 1024 ] }, + "outputs": [ + "latents", + "metadata" + ] + }, + "qwen_image_2_1_i2l": { + "inputs": [ + "image", + "vae" + ], + "literalInputs": { + "is_intermediate": [ + true + ], + "reference": [ + true + ], + "use_cache": [ + true + ] + }, "outputs": [ "latents" ] @@ -1532,7 +1575,8 @@ "qwen_image_2_1_text_encoder": { "inputs": [ "prompt", - "qwen3_vl_encoder" + "qwen3_vl_encoder", + "reference_images" ], "literalInputs": { "is_intermediate": [ diff --git a/invokeai/frontend/webv2/src/features/generation/core/architectureCapabilities.ts b/invokeai/frontend/webv2/src/features/generation/core/architectureCapabilities.ts index fdae256b518..794f1767002 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/architectureCapabilities.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/architectureCapabilities.ts @@ -26,6 +26,7 @@ export interface ArchitectureCapabilitiesRow { control_kinds: string[]; max_reference_images: number; reference_images_require_variant: string | null; + reference_images_require_format: string | null; supports_regional_guidance: boolean; regional_negative: boolean; clip_skip_max: number | null; diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts index f4f3ea64af3..80b62638d1b 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.test.ts @@ -29,12 +29,14 @@ import { getGenerationUiPolicy, getGenerationValidationReasons, getMaxReferenceImages, + getMissingReferenceImageFormat, getPromptPolicy, getSchedulerOptions, getSettingsWithModelDefaults, isReferenceImageSupported, isGenerateModelSelectable, isSupportedGenerateModel, + QWEN_IMAGE_21_STANDALONE_ENCODER_REASON, } from './baseGenerationPolicies'; import { SUPPORTED_GENERATE_BASES } from './supportedBases'; @@ -1148,6 +1150,29 @@ describe('component policies', () => { expect(isReferenceImageSupported(undefined)).toBe(false); }); + it('offers Qwen-Image-2.1 reference images only on its Diffusers pipeline, whose encoder reads images', () => { + const diffusers = createModel('qwen-image-2-1', { format: 'diffusers' }); + + expect(isReferenceImageSupported(diffusers)).toBe(true); + expect(getMaxReferenceImages(diffusers)).toBe(4); + expect(isReferenceImageSupported(createModel('qwen-image-2-1', { format: 'gguf_quantized' }))).toBe(false); + expect(isReferenceImageSupported(createModel('qwen-image-2-1', { format: 'checkpoint' }))).toBe(false); + }); + + // The UI names the missing format only when switching to it would enable references. + it('names the format a model lacks for reference images only when the format alone blocks them', () => { + expect(getMissingReferenceImageFormat(createModel('qwen-image-2-1', { format: 'gguf_quantized' }))).toBe( + 'diffusers' + ); + expect(getMissingReferenceImageFormat(createModel('qwen-image-2-1', { format: 'checkpoint' }))).toBe('diffusers'); + expect(getMissingReferenceImageFormat(createModel('qwen-image-2-1', { format: 'diffusers' }))).toBeNull(); + // Qwen-Image needs the Edit variant, not a format; CogView4 takes no references at all. + expect(getMissingReferenceImageFormat(createModel('qwen-image', { format: 'gguf_quantized' }))).toBeNull(); + expect(getMissingReferenceImageFormat(createModel('cogview4', { format: 'gguf_quantized' }))).toBeNull(); + expect(getMissingReferenceImageFormat(externalModel)).toBeNull(); + expect(getMissingReferenceImageFormat(undefined)).toBeNull(); + }); + it('derives the reference image limit from external provider capabilities', () => { expect(getMaxReferenceImages(createModel('sdxl'))).toBe(5); expect(getMaxReferenceImages(createModel('cogview4'))).toBe(0); @@ -1213,6 +1238,31 @@ describe('component policies', () => { ).toContain('Reference Image #1 is not supported by sd-3 model.'); }); + it('blocks Qwen-Image-2.1 references while a standalone encoder replaces the one that reads them', () => { + const model = createModel('qwen-image-2-1', { format: 'diffusers' }); + const encoder = { + base: 'any', + key: 'qwen3-vl-8b', + name: 'Qwen3-VL 8B', + type: 'qwen3_vl_encoder', + variant: 'qwen3_vl_8b', + } as const; + const referenceImages = [ + { + config: { image: referenceImage, type: 'qwen_image_2_1_reference_image' as const }, + id: 'ref-1', + isEnabled: true, + }, + ]; + + expect( + getGenerationValidationReasons(model, createSettings(model, { qwen3VLEncoderModel: encoder, referenceImages })) + ).toContain(QWEN_IMAGE_21_STANDALONE_ENCODER_REASON); + expect(getGenerationValidationReasons(model, createSettings(model, { referenceImages }))).not.toContain( + QWEN_IMAGE_21_STANDALONE_ENCODER_REASON + ); + }); + it('reports missing reference image model dependencies', () => { const model = createModel('sd-1'); diff --git a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts index f2fe327b4fa..149421d0d20 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/baseGenerationPolicies.ts @@ -1064,7 +1064,8 @@ const getBaseComponentSectionPolicy = ( { ...qwen3VlEncoderSlot( 'Qwen-Image-2.1 conditions on the Qwen3-VL 8B encoder, not the 4B one Krea-2 uses. Required for ' + - 'non-Diffusers Qwen-Image-2.1 models.', + "non-Diffusers Qwen-Image-2.1 models. On a Diffusers model it replaces the model's own encoder, the " + + 'only one that reads reference images.', isQwen3Vl8bEncoder ), required: (ctx) => ctx.model.format !== 'diffusers', @@ -1324,12 +1325,36 @@ export const isReferenceImageSupported = (model: GenerateModelConfig | undefined return false; } - // The backend declares which reference-image features require a specific variant. + // The backend declares which reference-image features require a specific variant or format. return ( - features.reference_images_require_variant === null || model.variant === features.reference_images_require_variant + (features.reference_images_require_variant === null || + model.variant === features.reference_images_require_variant) && + (features.reference_images_require_format === null || model.format === features.reference_images_require_format) ); }; +/** + * The model format the selected base needs for reference images, when the model's format is all that keeps them + * unavailable, so the UI can say what to switch to instead of hiding the section without a reason. + */ +export const getMissingReferenceImageFormat = (model: GenerateModelConfig | undefined): string | null => { + if (!model || model.type === 'external_image_generator' || isReferenceImageSupported(model)) { + return null; + } + + const features = getArchitectureFeatures(model.base); + + if ( + !features || + features.max_reference_images <= 0 || + (features.reference_images_require_variant !== null && model.variant !== features.reference_images_require_variant) + ) { + return null; + } + + return features.reference_images_require_format; +}; + export const getMaxReferenceImages = (model: GenerateModelConfig | undefined): number => { if (!model || !isReferenceImageSupported(model)) { return 0; @@ -1368,6 +1393,10 @@ export const getDefaultReferenceImageConfig = ( return { image, styleStrength: DEFAULT_KREA2_STYLE_STRENGTH, type: 'krea2_reference_image' }; } + if (modelBase === 'qwen-image-2-1') { + return { image, type: 'qwen_image_2_1_reference_image' }; + } + if (isFluxKontextModel(model)) { return { image, model, type: 'flux_kontext_reference_image' }; } @@ -1399,6 +1428,10 @@ const getReferenceImageConfigSupported = ( return model.type !== 'external_image_generator' && model.base === 'qwen-image' && model.variant === 'edit'; case 'krea2_reference_image': return model.type !== 'external_image_generator' && model.base === 'krea-2'; + case 'qwen_image_2_1_reference_image': + return ( + model.type !== 'external_image_generator' && model.base === 'qwen-image-2-1' && isReferenceImageSupported(model) + ); case 'flux_kontext_reference_image': return isFluxKontextModel(model); case 'flux_redux': @@ -1693,6 +1726,11 @@ const getPidValidationReasons = (model: GenerateModelConfig, settings: GenerateS return reasons; }; +/** Only the encoder of a Qwen-Image-2.1 Diffusers model keeps the vision tower that reads references. */ +export const QWEN_IMAGE_21_STANDALONE_ENCODER_REASON = + "Reference images need the Qwen-Image-2.1 model's own encoder, the only one with a vision tower. " + + 'Clear the Qwen3-VL Encoder component.'; + const getReferenceImageValidationReasons = (model: GenerateModelConfig, settings: GenerateSettings): string[] => { const reasons: string[] = []; const enabled = settings.referenceImages.filter((referenceImage) => referenceImage.isEnabled); @@ -1737,6 +1775,17 @@ const getReferenceImageValidationReasons = (model: GenerateModelConfig, settings } }); + // The encoder slot stays filled across models (Ideogram 4 and the GGUF builds share it), so a Diffusers model can + // inherit one that cannot read the references. + if ( + model.base === 'qwen-image-2-1' && + settings.qwen3VLEncoderModel && + isQwen3Vl8bEncoder(settings.qwen3VLEncoderModel) && + enabled.some((referenceImage) => referenceImage.config.type === 'qwen_image_2_1_reference_image') + ) { + reasons.push(QWEN_IMAGE_21_STANDALONE_ENCODER_REASON); + } + return reasons; }; diff --git a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts index ea3a660c815..5821f83fb2a 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/canvas/compileCanvasGraph.test.ts @@ -712,6 +712,22 @@ describe('compileCanvasGraph — inpaint per base', () => { expect(metaEdge?.destination.node_id).toBe('canvas_output'); }); + // Qwen-Image-2.1's denoise adds its cache mode to the record, so the record still passes through it once the blend + // takes over the output. + it('routes Qwen-Image-2.1 inpaint metadata through the denoise onto the final blend output', () => { + const { backendGraph } = compile(qwenImage21Model, 'inpaint'); + const metadata = getNodeByType(backendGraph, 'core_metadata'); + const metadataEdges = backendGraph.edges.filter( + (edge) => edge.source.field === 'metadata' || edge.destination.field === 'metadata' + ); + + expect(backendGraph.nodes.canvas_output?.type).toBe('invokeai_img_blend'); + expect(metadataEdges.map(({ destination, source }) => [source.node_id, destination.node_id])).toEqual([ + [metadata?.id, 'denoise_latents'], + ['denoise_latents', 'canvas_output'], + ]); + }); + it('outputs only the generated mask region with transparency when requested', () => { const { backendGraph } = compile(sd1Model, 'inpaint', { outputOnlyMaskedRegions: true }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts index e0be63ac823..679b87ffac9 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import type { ComponentModelConfig, GenerateModelConfig, + GenerateReferenceImage, GenerateSettings, LoraModelConfig, MainModelConfig, @@ -767,6 +768,65 @@ describe('compileGenerateGraph', () => { expect(getEdge(graph, 'pos_cond', 'reference_images')).toBeDefined(); expect(getEdge(graph, 'denoise_latents', 'reference_latents')).toBeDefined(); }); + + describe('Qwen-Image-2.1 reference images', () => { + const qwenImage21Model: MainModelConfig = { + base: 'qwen-image-2-1', + format: 'diffusers', + key: 'qwen21', + name: 'Qwen-Image-2.1', + type: 'main', + }; + const references = (...names: string[]): GenerateReferenceImage[] => + names.map((name) => ({ + config: { + image: { original: { image: { height: 768, image_name: name, width: 512 } } }, + type: 'qwen_image_2_1_reference_image', + }, + id: name, + isEnabled: true, + })); + + it('reads each reference in both prompts and, in the same order, as latents in denoise', () => { + const graph = compile(qwenImage21Model, { cfgScale: 4, referenceImages: references('a.png', 'b.png') }); + + const nameOf = (imageNode: string) => + (graph.nodes[imageNode]?.image as { image_name: string } | undefined)?.image_name; + // Each chain is followed back from its end: the last collect holds b, the one before it a. + const chain = (lastCollect: string) => [ + getEdge(graph, lastCollect, 'collection')?.source.node_id ?? '', + lastCollect, + ]; + + const imagesInto = getEdge(graph, 'pos_cond', 'reference_images')?.source.node_id ?? ''; + expect(getEdge(graph, 'neg_cond', 'reference_images')?.source.node_id).toBe(imagesInto); + expect(chain(imagesInto).map((collect) => nameOf(getEdge(graph, collect, 'item')?.source.node_id ?? ''))).toEqual( + ['a.png', 'b.png'] + ); + + const latentsInto = getEdge(graph, 'denoise_latents', 'reference_latents')?.source.node_id ?? ''; + const latentImages = chain(latentsInto).map((collect) => { + const i2l = getEdge(graph, collect, 'item')?.source.node_id ?? ''; + expect(graph.nodes[i2l]).toMatchObject({ reference: true, type: 'qwen_image_2_1_i2l' }); + return nameOf(getEdge(graph, i2l, 'image')?.source.node_id ?? ''); + }); + expect(latentImages).toEqual(['a.png', 'b.png']); + }); + + // The denoise records whether it reused the prompt's attention cache, which changes the image in bf16. + it('passes the image metadata through the denoise to the decoder that saves the image', () => { + const graph = compile(qwenImage21Model); + const metadata = getNodeByType(graph, 'core_metadata'); + const metadataEdges = graph.edges.filter( + (edge) => edge.source.field === 'metadata' || edge.destination.field === 'metadata' + ); + + expect(metadataEdges.map(({ destination, source }) => [source.node_id, destination.node_id])).toEqual([ + [metadata?.id, 'denoise_latents'], + ['denoise_latents', 'canvas_output'], + ]); + }); + }); }); describe('generate seeds', () => { @@ -1758,7 +1818,10 @@ describe('output filters', () => { node_id: 'canvas_unfiltered_output', }); expect(getEdge(graph, 'canvas_output', 'image')?.source).toEqual({ field: 'image', node_id: 'canvas_img_nsfw' }); - expect(getEdge(graph, 'canvas_output', 'metadata')?.source.node_id).toBe(getNodeByType(graph, 'core_metadata')?.id); + // Qwen-Image-2.1's denoise adds its cache mode to the record on the way, so the record reaches the result from it. + const metadataSource = + base === 'qwen-image-2-1' ? 'denoise_latents' : (getNodeByType(graph, 'core_metadata')?.id ?? ''); + expect(getEdge(graph, 'canvas_output', 'metadata')?.source).toEqual({ field: 'metadata', node_id: metadataSource }); expect(getEdge(graph, 'canvas_unfiltered_output', 'metadata')).toBeUndefined(); }); diff --git a/invokeai/frontend/webv2/src/features/generation/core/graph.ts b/invokeai/frontend/webv2/src/features/generation/core/graph.ts index 21abacc9494..c4b0151a212 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graph.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graph.ts @@ -1438,14 +1438,83 @@ const buildQwenImage21Graph = ( } addEdge(graph, seed, 'value', denoise, 'seed'); addEdge(graph, denoise, 'latents', output, 'latents'); - addMetadata(graph, output, settings, model, 'qwen_image_2_1_txt2img', projectSettings, { + addQwenImage21ReferenceImages(graph, settings, { + conditionings: negCond ? [posCond, negCond] : [posCond], + denoise, + modelLoader, + }); + // The denoise adds the prefix-cache mode it ran to the record, which changes the image, so the metadata passes + // through it on the way to the decoder. + addMetadata(graph, denoise, settings, model, 'qwen_image_2_1_txt2img', projectSettings, { qwen3_vl_encoder: qwen3VlEncoderModel ?? undefined, vae: vaeModel ?? undefined, }); + addEdge(graph, denoise, 'metadata', output, 'metadata'); + addReferenceImageMetadata(graph, output, settings); return graph; }; +/** + * Qwen-Image-2.1 reads each reference twice, at one size: the prompt encoder sees it through the vision tower, and + * the denoise node places its clean latents in the prompt's image slots. Both prompts are encoded with the + * references, and the latents follow in the same order. + */ +const addQwenImage21ReferenceImages = ( + graph: BackendGraphContract, + settings: GenerateSettings, + nodes: { + conditionings: BackendInvocationContract[]; + denoise: BackendInvocationContract; + modelLoader: BackendInvocationContract; + } +): void => { + const referenceImages = getEnabledReferenceImages(settings, 'qwen_image_2_1_reference_image'); + if (referenceImages.length === 0) { + return; + } + + let imageCollect: BackendInvocationContract | null = null; + let latentsCollect: BackendInvocationContract | null = null; + + for (const { config } of referenceImages) { + if (!config.image) { + continue; + } + + const imageNode = addNode(graph, { + id: createId('qwen21_ref_img'), + image: toImageField(config.image), + type: 'image', + }); + const i2l = addNode(graph, { id: createId('qwen21_ref_i2l'), reference: true, type: 'qwen_image_2_1_i2l' }); + addEdge(graph, imageNode, 'image', i2l, 'image'); + addEdge(graph, nodes.modelLoader, 'vae', i2l, 'vae'); + + const nextImageCollect = addNode(graph, { id: createId('qwen21_ref_img_collect'), type: 'collect' }); + addEdge(graph, imageNode, 'image', nextImageCollect, 'item'); + if (imageCollect) { + addEdge(graph, imageCollect, 'collection', nextImageCollect, 'collection'); + } + imageCollect = nextImageCollect; + + const nextLatentsCollect = addNode(graph, { id: createId('qwen21_ref_latents_collect'), type: 'collect' }); + addEdge(graph, i2l, 'latents', nextLatentsCollect, 'item'); + if (latentsCollect) { + addEdge(graph, latentsCollect, 'collection', nextLatentsCollect, 'collection'); + } + latentsCollect = nextLatentsCollect; + } + + if (!imageCollect || !latentsCollect) { + return; + } + for (const conditioning of nodes.conditionings) { + addEdge(graph, imageCollect, 'collection', conditioning, 'reference_images'); + } + addEdge(graph, latentsCollect, 'collection', nodes.denoise, 'reference_latents'); +}; + const buildIdeogram4Graph = ( settings: GenerateSettings, model: MainModelConfig, diff --git a/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.testing.ts b/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.testing.ts index 81e4ec21c27..885e623419b 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.testing.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/graphCoverage.testing.ts @@ -6,11 +6,18 @@ import type { GenerateComponentValueKey, SupportedGenerateBase, } from './baseGenerationPolicies'; -import type { GenerateSettings, MainModelConfig, ModelIdentifierConfig } from './types'; +import type { + GenerateReferenceImage, + GenerateReferenceImageAsset, + GenerateSettings, + MainModelConfig, + ModelIdentifierConfig, +} from './types'; import { getComponentSectionPolicy, getDefaultGenerateSettings, + isReferenceImageSupported, SUPPORTED_GENERATE_BASES, } from './baseGenerationPolicies'; import { getIsPidSupportedBase } from './pid'; @@ -210,6 +217,23 @@ export const fullyFilledSettingsFor = (base: SupportedGenerateBase, shape: Model return { filled: [...allFilled].sort(), model, settings }; }; +/** References for the families whose builders wire them through their own nodes rather than an adapter. */ +const tunedReferenceImagesFor = ( + model: MainModelConfig, + image: GenerateReferenceImageAsset +): GenerateReferenceImage[] | undefined => { + if (model.base === 'krea-2') { + return [{ config: { image, styleStrength: 1, type: 'krea2_reference_image' }, id: 'style', isEnabled: true }]; + } + + // Only the Diffusers pipeline's own encoder reads references; other shapes compile without them. + if (model.base === 'qwen-image-2-1' && isReferenceImageSupported(model)) { + return [{ config: { image, type: 'qwen_image_2_1_reference_image' }, id: 'edit', isEnabled: true }]; + } + + return undefined; +}; + /** * The optional per-family tuning and output filters, all switched on, so the backend contract also covers the nodes * and literal values a default recipe never emits. Settings a family ignores are harmless to carry. @@ -224,21 +248,14 @@ export const tunedSettingsFor = (base: SupportedGenerateBase, shape: ModelShape) projectSettings: { useCpuNoise: true, useNsfwChecker: true, useWatermark: true }, settings: { ...settings, + // CFG above 1 adds the negative prompt, which Qwen-Image-2.1 also encodes with the references. + cfgScale: base === 'qwen-image-2-1' ? 4 : settings.cfgScale, fluxDypeExponent: 1, fluxDypePreset: 'manual' as const, fluxDypeScale: 3, qwenImageEncoderQuantization: 'nf4' as const, qwenImageShift: 3, - referenceImages: - base === 'krea-2' - ? [ - { - config: { image, styleStrength: 1, type: 'krea2_reference_image' as const }, - id: 'style', - isEnabled: true, - }, - ] - : settings.referenceImages, + referenceImages: tunedReferenceImagesFor(model, image) ?? settings.referenceImages, zImageSeedVarianceEnabled: true, zImageSeedVarianceStrength: 0.2, zImageShift: 3, diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts index f5d373e07d4..0f81cc4707a 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.test.ts @@ -191,6 +191,46 @@ describe('normalizeGenerateSettings', () => { ).toBe(false); }); + // Qwen-Image-2.1 references are their own flavor, so a stored one must not fold into Qwen-Image Edit's. + it('keeps persisted Qwen-Image-2.1 reference images as their own flavor', () => { + const normalized = normalizeGenerateSettings({ + ...legacyStoredValues, + referenceImages: [ + { + id: 'edit-1', + isEnabled: false, + config: { + type: 'qwen_image_2_1_reference_image', + image: { + height: 640, + imageName: 'edit.png', + imageUrl: '/api/v1/images/i/edit.png/full', + queuedAt: '2026-01-01T00:00:00.000Z', + sourceQueueItemId: 'backend-gallery', + thumbnailUrl: '/api/v1/images/i/edit.png/thumbnail', + width: 480, + }, + }, + }, + { id: 'empty', isEnabled: true, config: { type: 'qwen_image_2_1_reference_image', image: null } }, + { id: 'broken', isEnabled: true, config: { type: 'qwen_image_2_1_reference_image', image: { width: 1 } } }, + ], + }); + + expect(normalized?.referenceImages).toEqual([ + { + config: { + image: { original: { image: { height: 640, image_name: 'edit.png', width: 480 } } }, + type: 'qwen_image_2_1_reference_image', + }, + id: 'edit-1', + isEnabled: false, + }, + { config: { image: null, type: 'qwen_image_2_1_reference_image' }, id: 'empty', isEnabled: true }, + ]); + expect(normalized && isGenerateSettings(normalized)).toBe(true); + }); + it('keeps more than five valid persisted reference images for provider-specific caps', () => { const referenceImages = Array.from({ length: 6 }, (_, index) => ({ id: `ref-${index}`, diff --git a/invokeai/frontend/webv2/src/features/generation/core/settings.ts b/invokeai/frontend/webv2/src/features/generation/core/settings.ts index c3dcc50eb61..5915285b832 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/settings.ts @@ -312,6 +312,7 @@ const REFERENCE_IMAGE_CONFIG_TYPES = new Set([ 'ip_adapter', 'krea2_reference_image', 'qwen_image_reference_image', + 'qwen_image_2_1_reference_image', ]); const isClipVisionModel = (value: unknown): value is 'ViT-H' | 'ViT-G' | 'ViT-L' => @@ -347,6 +348,7 @@ const normalizeReferenceImageConfig = (value: unknown): GenerateReferenceImageCo case 'external_reference_image': case 'flux2_reference_image': case 'qwen_image_reference_image': + case 'qwen_image_2_1_reference_image': return { image, type: value.type }; case 'flux_kontext_reference_image': return { image, model: getMainModelOrNull(value.model), type: value.type }; @@ -419,6 +421,7 @@ const cloneReferenceImageConfig = (config: GenerateReferenceImageConfig): Genera case 'flux2_reference_image': case 'krea2_reference_image': case 'qwen_image_reference_image': + case 'qwen_image_2_1_reference_image': return { ...config, image: config.image ? cloneCroppableImage(config.image) : null }; case 'flux_kontext_reference_image': return { @@ -589,6 +592,7 @@ const syncReferenceImageConfigWithModels = ( case 'flux2_reference_image': case 'krea2_reference_image': case 'qwen_image_reference_image': + case 'qwen_image_2_1_reference_image': return config; } }; diff --git a/invokeai/frontend/webv2/src/features/generation/core/types.ts b/invokeai/frontend/webv2/src/features/generation/core/types.ts index 208c3ffa964..711b8bd616d 100644 --- a/invokeai/frontend/webv2/src/features/generation/core/types.ts +++ b/invokeai/frontend/webv2/src/features/generation/core/types.ts @@ -142,6 +142,7 @@ export type GenerateReferenceImageConfig = | { type: 'qwen_image_reference_image'; image: GenerateReferenceImageAsset | null } /** Krea-2 training-free style transfer: no adapter model, a single reference, `styleStrength` 0–2 (0 bypasses). */ | { type: 'krea2_reference_image'; image: GenerateReferenceImageAsset | null; styleStrength: number } + | { type: 'qwen_image_2_1_reference_image'; image: GenerateReferenceImageAsset | null } | { type: 'external_reference_image'; image: GenerateReferenceImageAsset | null }; export interface GenerateReferenceImage { diff --git a/invokeai/frontend/webv2/src/features/generation/ui/GenerateGuidanceSection.tsx b/invokeai/frontend/webv2/src/features/generation/ui/GenerateGuidanceSection.tsx index 5bc8723152e..4043d293809 100644 --- a/invokeai/frontend/webv2/src/features/generation/ui/GenerateGuidanceSection.tsx +++ b/invokeai/frontend/webv2/src/features/generation/ui/GenerateGuidanceSection.tsx @@ -2,7 +2,10 @@ import type { GenerationModelCatalogItem as ModelConfig } from '@features/genera import type { GenerateModelConfig, GenerateSettings, LoraModelConfig } from '@features/generation/core/types'; import { Badge, Separator, Stack } from '@chakra-ui/react'; -import { isReferenceImageSupported } from '@features/generation/core/baseGenerationPolicies'; +import { + getMissingReferenceImageFormat, + isReferenceImageSupported, +} from '@features/generation/core/baseGenerationPolicies'; import { type GenerateDraft, type GenerateSettingsUpdate, @@ -85,7 +88,9 @@ export const GenerateGuidanceSection = ({ onCommit={onReferenceCommit} onCommitImmediate={onCommitImmediate} /> - {referenceImagesSupported || referenceImageCount > 0 ? : null} + {referenceImagesSupported || referenceImageCount > 0 || getMissingReferenceImageFormat(selectedModel) ? ( + + ) : null} = { diffusers: 'Diffusers' }; const IMAGE_ONLY = ['image'] as const; interface GenerateReferenceImagesContentProps { @@ -50,8 +53,9 @@ export const GenerateReferenceImagesContent = ({ selectedModel, }: GenerateReferenceImagesContentProps) => { const { t } = useTranslation(); - const { gallery } = useGenerationUi(); + const { gallery, models: modelUi } = useGenerationUi(); const isSupported = isReferenceImageSupported(selectedModel); + const missingFormat = getMissingReferenceImageFormat(selectedModel); const maxReferenceImages = getMaxReferenceImages(selectedModel); const canAdd = referenceImages.length < maxReferenceImages; const { acceptsActiveDrag, isOver, setNodeRef } = useGalleryImageDroppable({ @@ -194,8 +198,21 @@ export const GenerateReferenceImagesContent = ({ }, }); + // A model that only lacks the right format says which one would take references. + const unsupportedMessage = + selectedModel && missingFormat + ? t('widgets.generate.referenceImagesNeedFormat', { + base: modelUi.getBaseLabel(selectedModel.base), + format: MODEL_FORMAT_LABELS[missingFormat] ?? missingFormat, + }) + : t('widgets.generate.referenceImagesUnsupported'); + if (!isSupported && referenceImageCount === 0) { - return null; + return missingFormat ? ( + + {unsupportedMessage} + + ) : null; } // Unsupported-model references remain read-only with removal as recovery. @@ -203,7 +220,7 @@ export const GenerateReferenceImagesContent = ({ return ( - {t('widgets.generate.referenceImagesUnsupported')} + {unsupportedMessage}