From 855ebe2c02cd82d0ee81e95fb6f8e6aa73a06990 Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Sat, 17 Oct 2026 04:57:58 +0200 Subject: [PATCH 1/5] feat(working-memory): ship VAE working-memory coefficients as data VAE estimators return a WorkingMemoryEstimate built from fitted and analytic terms; model_on_device accepts it as working_memory=. The AutoencoderKL coefficients and their measured points move to backend/working_memory/data, checked by a data test; reservations are unchanged. --- .../contributing/new-model-integration.mdx | 4 +- .../Architecture/model-manager.mdx | 20 + .../vae/cogview4_image_to_latents.py | 2 +- .../vae/cogview4_latents_to_image.py | 2 +- .../app/invocations/vae/flux2_vae_decode.py | 2 +- .../app/invocations/vae/flux2_vae_encode.py | 2 +- .../app/invocations/vae/flux_vae_decode.py | 4 +- .../app/invocations/vae/flux_vae_encode.py | 2 +- .../vae/ideogram4_latents_to_image.py | 2 +- .../app/invocations/vae/image_to_latents.py | 2 +- .../app/invocations/vae/latents_to_image.py | 2 +- .../invocations/vae/sd3_image_to_latents.py | 2 +- .../invocations/vae/sd3_latents_to_image.py | 2 +- .../vae/z_image_image_to_latents.py | 2 +- .../vae/z_image_latents_to_image.py | 4 +- invokeai/backend/AGENTS.md | 1 + .../flux/extensions/kontext_extension.py | 2 +- invokeai/backend/flux2/ref_image_extension.py | 2 +- .../backend/model_manager/load/load_base.py | 18 +- invokeai/backend/util/vae_working_memory.py | 542 +++++++----- invokeai/backend/working_memory/__init__.py | 6 + .../backend/working_memory/coefficients.py | 191 +++++ invokeai/backend/working_memory/data/vae.json | 769 ++++++++++++++++++ invokeai/backend/working_memory/estimate.py | 52 ++ invokeai/backend/working_memory/platform.py | 45 + invokeai/backend/working_memory/spec.py | 99 +++ pyproject.toml | 2 + scripts/calibrate_flux2_working_memory.py | 37 +- scripts/calibrate_flux_vae_working_memory.py | 6 +- tests/app/invocations/conftest.py | 18 + .../invocations/test_flux2_working_memory.py | 235 ++---- tests/app/invocations/test_flux_vae_decode.py | 2 +- .../test_flux_vae_decode_tiling.py | 18 +- .../test_vae_working_memory_backend.py | 76 +- .../invocations/test_z_image_tiled_decode.py | 135 +-- .../test_z_image_working_memory.py | 12 +- .../model_manager/load/test_loaded_model.py | 64 +- .../util/test_rocm_sdpa_head_dim_guard.py | 30 +- tests/backend/working_memory/__init__.py | 0 .../test_shipped_coefficients.py | 312 +++++++ .../working_memory/test_spec_registry.py | 72 ++ 41 files changed, 2266 insertions(+), 534 deletions(-) create mode 100644 invokeai/backend/working_memory/__init__.py create mode 100644 invokeai/backend/working_memory/coefficients.py create mode 100644 invokeai/backend/working_memory/data/vae.json create mode 100644 invokeai/backend/working_memory/estimate.py create mode 100644 invokeai/backend/working_memory/platform.py create mode 100644 invokeai/backend/working_memory/spec.py create mode 100644 tests/backend/working_memory/__init__.py create mode 100644 tests/backend/working_memory/test_shipped_coefficients.py create mode 100644 tests/backend/working_memory/test_spec_registry.py diff --git a/docs/src/content/docs/contributing/new-model-integration.mdx b/docs/src/content/docs/contributing/new-model-integration.mdx index eda7821880a..04e4530b2d8 100644 --- a/docs/src/content/docs/contributing/new-model-integration.mdx +++ b/docs/src/content/docs/contributing/new-model-integration.mdx @@ -601,11 +601,11 @@ Packages are discovered automatically. A directory of node modules without an `_ ... ``` - Report progress with `context.util.sd_step_callback(state, BaseModelType.NewModel)`; it renders previews through the architecture's `LatentSpaceFacet`. The denoise node is also where these are checked: interrupts (`context.util.is_canceled()`), inpainting, img2img through `denoising_start`, and a working-memory estimate passed to `model_on_device(working_mem_bytes=...)`. + Report progress with `context.util.sd_step_callback(state, BaseModelType.NewModel)`; it renders previews through the architecture's `LatentSpaceFacet`. The denoise node is also where these are checked: interrupts (`context.util.is_canceled()`), inpainting, img2img through `denoising_start`, and a working-memory estimate passed to `model_on_device(working_mem_bytes=)`, or `working_memory=` for an estimator that declares an `EstimatorSpec`. 4. **VAE encode and decode** (`vae/new_model_image_to_latents.py`, `vae/new_model_latents_to_image.py`) - Apply the VAE's latent normalization, scaling and shift or per-channel mean and std, exactly as the reference pipeline does. Handle tiling and give a working-memory estimate. The estimate helpers live in `invokeai/backend/util/vae_working_memory.py`, with calibration scripts in `scripts/calibrate_*_working_memory.py`. Node types conventionally end in `_i2l` and `_l2i`. + Apply the VAE's latent normalization, scaling and shift or per-channel mean and std, exactly as the reference pipeline does. Handle tiling and give a working-memory estimate. The estimate helpers live in `invokeai/backend/util/vae_working_memory.py`, with calibration scripts in `scripts/calibrate_*_working_memory.py`. A new estimator declares an `EstimatorSpec`, returns a `WorkingMemoryEstimate` and takes its fitted coefficients from `invokeai/backend/working_memory/data/`, next to the measured points they were fitted on: the estimate must be at least each point and, where a fitted term decides it, at most 1.3 times it (or the point's `max_ratio`). The [model manager](/development/architecture/model-manager/) lists what such an estimator needs. `working_mem_bytes=` remains for a plain reservation. Node types conventionally end in `_i2l` and `_l2i`. 5. **LoRA loaders** (if LoRA is supported, see [Optional features](#12-optional-features)) diff --git a/docs/src/content/docs/development/Architecture/model-manager.mdx b/docs/src/content/docs/development/Architecture/model-manager.mdx index 846883395b4..2dfa5d93010 100644 --- a/docs/src/content/docs/development/Architecture/model-manager.mdx +++ b/docs/src/content/docs/development/Architecture/model-manager.mdx @@ -972,6 +972,26 @@ with loaded_model.model_on_device() as (state_dict, vae): Since not all models have state dicts, the `state_dict` return value can be None. +An operation that needs more device memory than `device_working_mem_gb` reserves it when it locks the model, so the cache offloads other models instead of running out of memory. Pass the operation's working-memory estimate: + +```py +estimate = estimate_vae_working_memory_flux2( + operation="decode", image_tensor=latents, vae=vae_info.model, device=vae_info.compute_device +) +with vae_info.model_on_device(working_memory=estimate) as (_, vae): + image = vae.decode(latents)[0] +``` + +An estimator that declares an `EstimatorSpec` returns a `WorkingMemoryEstimate` (`invokeai/backend/working_memory/estimate.py`): the reservation, the terms it adds up to, the features it was evaluated from, and the source of its coefficients, which `model_on_device()` logs at debug level. A plain byte count goes in as `working_mem_bytes=` instead; never both. The cache never reserves less than `device_working_mem_gb`. + +To write such an estimator (`invokeai/backend/util/vae_working_memory.py` has five): + +- Register an `EstimatorSpec` at module import, and list the module in `ESTIMATOR_MODULES` (`working_memory/spec.py`); the data test fails on a module that declares a spec without being listed. +- Split the estimator in two. The impure part reads the model and the device and records every runtime decision (tile, element size, score-matrix bytes) as a feature. The pure `terms(features, values)` turns features and coefficient values into terms, building fitted terms with `WorkingMemoryTerm.fitted(...)`. The public function returns `SPEC.estimate(features, device)`. +- Bump `version` whenever `terms` changes the formula. +- Ship one coefficient entry per measured platform value in `invokeai/backend/working_memory/data/`, either with its own `values` or with `values_from` another estimator's entry. Unmeasured platform values (MPS, XPU, CPU) are priced as `UNMEASURED_PRICED_AS` in `working_memory/platform.py` says. +- Ship the measured points next to them: peak reserved bytes (`reserved_peak`), one fresh process per point. `tests/backend/working_memory/` holds each estimate to at least its point, and where a fitted term decides it, to at most 1.3 times the point, or the point's tighter `max_ratio`. The `device_working_mem_gb` floor counts only where an analytic term decides. Every value an entry carries must be decided by at least one point. + ### Emitting model loading events When the `context` argument is passed to `load_model_*()`, it will retrieve the invocation event bus from the passed `InvocationContext` object to emit events on the invocation bus. The two events are "model_load_started" and "model_load_completed". Both carry the following payload: diff --git a/invokeai/app/invocations/vae/cogview4_image_to_latents.py b/invokeai/app/invocations/vae/cogview4_image_to_latents.py index 09cde601d32..dd4a3862d98 100644 --- a/invokeai/app/invocations/vae/cogview4_image_to_latents.py +++ b/invokeai/app/invocations/vae/cogview4_image_to_latents.py @@ -43,7 +43,7 @@ def vae_encode(vae_info: LoadedModel, image_tensor: torch.Tensor) -> torch.Tenso estimated_working_memory = estimate_vae_working_memory_cogview4( operation="encode", image_tensor=image_tensor, vae=vae_info.model, device=vae_info.compute_device ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): assert isinstance(vae, AutoencoderKL) vae.disable_tiling() diff --git a/invokeai/app/invocations/vae/cogview4_latents_to_image.py b/invokeai/app/invocations/vae/cogview4_latents_to_image.py index 99fb2f21261..e8b655b0ce8 100644 --- a/invokeai/app/invocations/vae/cogview4_latents_to_image.py +++ b/invokeai/app/invocations/vae/cogview4_latents_to_image.py @@ -50,7 +50,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput: ) with ( SeamlessExt.static_patch_model(vae_info.model, self.vae.seamless_axes), - vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae), + vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae), ): context.util.signal_progress("Running VAE") assert isinstance(vae, (AutoencoderKL)) diff --git a/invokeai/app/invocations/vae/flux2_vae_decode.py b/invokeai/app/invocations/vae/flux2_vae_decode.py index f0852f1880f..cb5ed93c10a 100644 --- a/invokeai/app/invocations/vae/flux2_vae_decode.py +++ b/invokeai/app/invocations/vae/flux2_vae_decode.py @@ -57,7 +57,7 @@ def _vae_decode(self, vae_info: LoadedModel, latents: torch.Tensor) -> Image.Ima operation="decode", image_tensor=latents, vae=vae_info.model, device=vae_info.compute_device ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): vae_dtype = next(iter(vae.parameters())).dtype # Use the VAE's intended compute device (CUDA/MPS, or CPU if configured cpu_only). Do NOT infer it from # current param residency: partial loading may have temporarily offloaded all weights to RAM, which would diff --git a/invokeai/app/invocations/vae/flux2_vae_encode.py b/invokeai/app/invocations/vae/flux2_vae_encode.py index 926f3a7c043..4e024158f1c 100644 --- a/invokeai/app/invocations/vae/flux2_vae_encode.py +++ b/invokeai/app/invocations/vae/flux2_vae_encode.py @@ -60,7 +60,7 @@ def _vae_encode(self, vae_info: LoadedModel, image_tensor: torch.Tensor) -> torc device=device, ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): vae_dtype = next(iter(vae.parameters())).dtype image_tensor = image_tensor.to(device=device, dtype=vae_dtype) diff --git a/invokeai/app/invocations/vae/flux_vae_decode.py b/invokeai/app/invocations/vae/flux_vae_decode.py index 170162e8077..f50c2a3272c 100644 --- a/invokeai/app/invocations/vae/flux_vae_decode.py +++ b/invokeai/app/invocations/vae/flux_vae_decode.py @@ -80,7 +80,7 @@ def _vae_decode(self, context: InvocationContext, vae_info: LoadedModel, latents if ( not use_tiling and config.auto_tiled_decode - and should_pretile_vae_decode(vae_info.compute_device, estimated_working_memory) + and should_pretile_vae_decode(vae_info.compute_device, estimated_working_memory.reservation_bytes) ): use_tiling = True tile_size = self.tile_size @@ -92,7 +92,7 @@ def _vae_decode(self, context: InvocationContext, vae_info: LoadedModel, latents device=vae_info.compute_device, ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): assert isinstance(vae, AutoencoderKL) vae_dtype = next(iter(vae.parameters())).dtype # Use the VAE's intended compute device (CUDA/MPS, or CPU if configured cpu_only). Do NOT infer it from diff --git a/invokeai/app/invocations/vae/flux_vae_encode.py b/invokeai/app/invocations/vae/flux_vae_encode.py index 5984fa737ed..a5a10900608 100644 --- a/invokeai/app/invocations/vae/flux_vae_encode.py +++ b/invokeai/app/invocations/vae/flux_vae_encode.py @@ -90,7 +90,7 @@ def vae_encode( # generator held outside would already have been drawn from by the attempt that failed, so # the retry would produce a different latent than a directly tiled run. generator = torch.Generator(device=vae_info.compute_device).manual_seed(0) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): assert isinstance(vae, AutoencoderKL) vae_dtype = next(iter(vae.parameters())).dtype # The VAE's own device, as in the decode nodes: a cpu_only VAE (or another worker's GPU) diff --git a/invokeai/app/invocations/vae/ideogram4_latents_to_image.py b/invokeai/app/invocations/vae/ideogram4_latents_to_image.py index f2e72d6e94c..0016a13ccec 100644 --- a/invokeai/app/invocations/vae/ideogram4_latents_to_image.py +++ b/invokeai/app/invocations/vae/ideogram4_latents_to_image.py @@ -81,7 +81,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput: operation="decode", image_tensor=z, vae=vae_info.model, device=device ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): context.util.signal_progress("Running VAE") vae_dtype = next(vae.parameters()).dtype TorchDevice.empty_cache() diff --git a/invokeai/app/invocations/vae/image_to_latents.py b/invokeai/app/invocations/vae/image_to_latents.py index 71d3951d79c..d599fadd862 100644 --- a/invokeai/app/invocations/vae/image_to_latents.py +++ b/invokeai/app/invocations/vae/image_to_latents.py @@ -83,7 +83,7 @@ def vae_encode( fp32=upcast, device=vae_info.compute_device, ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): assert isinstance(vae, (AutoencoderKL, AutoencoderTiny)), "VAE must be of type SD-1.5 or SDXL" orig_dtype = vae.dtype if upcast: diff --git a/invokeai/app/invocations/vae/latents_to_image.py b/invokeai/app/invocations/vae/latents_to_image.py index 4fbc87b8659..29c6a1b9db8 100644 --- a/invokeai/app/invocations/vae/latents_to_image.py +++ b/invokeai/app/invocations/vae/latents_to_image.py @@ -65,7 +65,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput: ) with ( SeamlessExt.static_patch_model(vae_info.model, self.vae.seamless_axes), - vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae), + vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae), ): context.util.signal_progress("Running VAE decoder") assert isinstance(vae, (AutoencoderKL, AutoencoderTiny)) diff --git a/invokeai/app/invocations/vae/sd3_image_to_latents.py b/invokeai/app/invocations/vae/sd3_image_to_latents.py index 5d558eeed52..fe1cb2faaa5 100644 --- a/invokeai/app/invocations/vae/sd3_image_to_latents.py +++ b/invokeai/app/invocations/vae/sd3_image_to_latents.py @@ -39,7 +39,7 @@ def vae_encode(vae_info: LoadedModel, image_tensor: torch.Tensor) -> torch.Tenso estimated_working_memory = estimate_vae_working_memory_sd3( operation="encode", image_tensor=image_tensor, vae=vae_info.model, device=vae_info.compute_device ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): assert isinstance(vae, AutoencoderKL) vae.disable_tiling() diff --git a/invokeai/app/invocations/vae/sd3_latents_to_image.py b/invokeai/app/invocations/vae/sd3_latents_to_image.py index 442ea0a83b6..87f67d73d8f 100644 --- a/invokeai/app/invocations/vae/sd3_latents_to_image.py +++ b/invokeai/app/invocations/vae/sd3_latents_to_image.py @@ -52,7 +52,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput: ) with ( SeamlessExt.static_patch_model(vae_info.model, self.vae.seamless_axes), - vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae), + vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae), ): context.util.signal_progress("Running VAE") assert isinstance(vae, (AutoencoderKL)) diff --git a/invokeai/app/invocations/vae/z_image_image_to_latents.py b/invokeai/app/invocations/vae/z_image_image_to_latents.py index de71e54ea0b..149116ae9a0 100644 --- a/invokeai/app/invocations/vae/z_image_image_to_latents.py +++ b/invokeai/app/invocations/vae/z_image_image_to_latents.py @@ -72,7 +72,7 @@ def vae_encode( device=vae_info.compute_device, ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): if not isinstance(vae, AutoencoderKL): raise TypeError( f"Expected AutoencoderKL, got {type(vae).__name__}. VAE model type changed unexpectedly after loading." diff --git a/invokeai/app/invocations/vae/z_image_latents_to_image.py b/invokeai/app/invocations/vae/z_image_latents_to_image.py index 2c5055402f3..af4c4bf07f6 100644 --- a/invokeai/app/invocations/vae/z_image_latents_to_image.py +++ b/invokeai/app/invocations/vae/z_image_latents_to_image.py @@ -74,7 +74,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput: if ( not use_tiling and config.auto_tiled_decode - and should_pretile_vae_decode(vae_info.compute_device, estimated_working_memory) + and should_pretile_vae_decode(vae_info.compute_device, estimated_working_memory.reservation_bytes) ): use_tiling = True estimated_working_memory = estimate_vae_working_memory_flux( @@ -90,7 +90,7 @@ def invoke(self, context: InvocationContext) -> ImageOutput: # FLUX VAE now honours the axes the UI has always offered for it. seamless_context = SeamlessExt.static_patch_model(vae_info.model, self.vae.seamless_axes) - with seamless_context, vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with seamless_context, vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): context.util.signal_progress("Running VAE") if not isinstance(vae, AutoencoderKL): raise TypeError( diff --git a/invokeai/backend/AGENTS.md b/invokeai/backend/AGENTS.md index 3b30a1c98c3..2f12dc1df2e 100644 --- a/invokeai/backend/AGENTS.md +++ b/invokeai/backend/AGENTS.md @@ -6,6 +6,7 @@ - Use existing device/model-management abstractions, `util/devices.py` included. Keep CPU/MPS/ROCm/XPU/multi-device support. No hardcoded CUDA, GPU zero, or dtype assumptions. - Respect session/thread device ownership. Restore temporary model patches, device state, hooks, allocations on success, cancellation, and exceptions. No request-specific mutable globals. - Define model/cache lifetimes and working-memory estimates. Bound queues, caches, downloads, batches, buffers. Consider peak memory and throughput. +- New or refitted working-memory coefficients go in `working_memory/data/` with the measured points that bound them; their estimators declare an `EstimatorSpec` and return a `WorkingMemoryEstimate`. - Keep model validation/loading safeguards, vendored exclusions, upstream provenance. No unrelated vendored rewrites or formatting. ## Performance diff --git a/invokeai/backend/flux/extensions/kontext_extension.py b/invokeai/backend/flux/extensions/kontext_extension.py index 7e801c54c04..1f31859909f 100644 --- a/invokeai/backend/flux/extensions/kontext_extension.py +++ b/invokeai/backend/flux/extensions/kontext_extension.py @@ -138,7 +138,7 @@ def _prepare_kontext(self) -> tuple[torch.Tensor, torch.Tensor]: operation="encode", image_tensor=image_tensor, vae=vae_info.model ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): assert isinstance(vae, AutoencoderKL) vae_dtype = next(iter(vae.parameters())).dtype image_tensor = image_tensor.to(device=TorchDevice.choose_torch_device(), dtype=vae_dtype) diff --git a/invokeai/backend/flux2/ref_image_extension.py b/invokeai/backend/flux2/ref_image_extension.py index 9184b15c1d2..c0e5d36b404 100644 --- a/invokeai/backend/flux2/ref_image_extension.py +++ b/invokeai/backend/flux2/ref_image_extension.py @@ -218,7 +218,7 @@ def _prepare_ref_images(self) -> tuple[torch.Tensor, torch.Tensor]: device=TorchDevice.choose_torch_device(), ) - with vae_info.model_on_device(working_mem_bytes=estimated_working_memory) as (_, vae): + with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): vae_dtype = next(iter(vae.parameters())).dtype image_tensor = image_tensor.to(device=TorchDevice.choose_torch_device(), dtype=vae_dtype) diff --git a/invokeai/backend/model_manager/load/load_base.py b/invokeai/backend/model_manager/load/load_base.py index 84a8d539516..fddf9a368dc 100644 --- a/invokeai/backend/model_manager/load/load_base.py +++ b/invokeai/backend/model_manager/load/load_base.py @@ -25,6 +25,8 @@ ModelCache, ) from invokeai.backend.model_manager.taxonomy import AnyModel, SubModelType +from invokeai.backend.util.logging import InvokeAILogger +from invokeai.backend.working_memory.estimate import WorkingMemoryEstimate class LoadedModelWithoutConfig: @@ -194,13 +196,27 @@ def __exit__(self, *args: Any, **kwargs: Any) -> None: @contextmanager def model_on_device( - self, working_mem_bytes: Optional[int] = None + self, working_mem_bytes: Optional[int] = None, *, working_memory: Optional[WorkingMemoryEstimate] = None ) -> Generator[Tuple[Optional[Dict[str, torch.Tensor]], AnyModel], None, None]: """Return a tuple consisting of the model's state dict (if it exists) and the locked model on execution device. :param working_mem_bytes: The amount of working memory to keep available on the compute device when loading the model. + :param working_memory: A working-memory estimate whose `reservation_bytes` is kept available instead; see + `invokeai.backend.working_memory`. Pass this or `working_mem_bytes`, not both. """ + if isinstance(working_mem_bytes, WorkingMemoryEstimate): + raise TypeError( + "model_on_device() got a WorkingMemoryEstimate as working_mem_bytes; pass it as working_memory=." + ) + if working_memory is not None: + if working_mem_bytes is not None: + raise ValueError("Pass either working_mem_bytes or working_memory to model_on_device(), not both.") + working_mem_bytes = working_memory.reservation_bytes + InvokeAILogger.get_logger(__name__).debug( + f"Working memory for {self._cache_record.key}: {working_memory.estimator} reserves " + f"{working_mem_bytes / 2**20:.0f} MB ({working_memory.source})" + ) self._lock_paced(working_mem_bytes) self._end_first_use_window() try: diff --git a/invokeai/backend/util/vae_working_memory.py b/invokeai/backend/util/vae_working_memory.py index 4c92894f1ee..6462dc1f41a 100644 --- a/invokeai/backend/util/vae_working_memory.py +++ b/invokeai/backend/util/vae_working_memory.py @@ -1,4 +1,6 @@ -from typing import Literal +from collections.abc import Mapping +from dataclasses import replace +from typing import Literal, cast import torch from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL @@ -13,6 +15,8 @@ from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.logging import InvokeAILogger from invokeai.backend.util.vae_tiling_scope import resolve_tile_size +from invokeai.backend.working_memory.estimate import FeatureValue, WorkingMemoryEstimate, WorkingMemoryTerm +from invokeai.backend.working_memory.spec import EstimatorSpec, register # Share of the device's memory an untiled decode may claim before the decode nodes tile it up front (see # `should_pretile_vae_decode`). High enough that common resolutions on common cards stay untiled -- tiled output is @@ -110,6 +114,96 @@ def _vae_mid_block_score_matrix_bytes( ) +# The SD1/SDXL, SD3, CogView4, FLUX.1 and FLUX.2 VAEs run the same diffusers AutoencoderKL convolution stack, and pay +# the same for it: peak reserved bytes per output pixel per element byte, one fitted coefficient per operation +# (`k_decode`, `k_encode`) and convolution backend. The values ship in `backend/working_memory/data/vae.json`, with the +# points they were fitted on and the per-card table behind them; `scripts/calibrate_flux2_working_memory.py --only vae` +# reproduces that table on any build. +# +# MIOpen's convolution workspaces are simply larger than cuDNN's -- ~1.6x for a decode, ~2.4x for an encode. This is +# not the attention term: it shows up identically on the fused path, and the implied coefficients are flat across +# resolution on both backends, so the linear model itself holds. Nor is "encoding costs half of decoding" architectural: +# it is a cuDNN property (0.49 there, 0.78 on MIOpen), which is why each operation carries its own coefficient instead +# of a ratio. Shipping the MIOpen values everywhere would add ~60% to every cuDNN decode for nothing, so the +# coefficients follow the backend of the device the VAE runs on (`platform_for`). +# +# Caveat for AMD users: `MIOPEN_FIND_MODE=2` (FAST) selects convolution algorithms heuristically instead of by +# benchmark, and measured a uniform 1.28x more memory at every resolution when the MIOpen values were first fitted. +# Invoke has since made FAST its default (`run_app.py`); the RX 9060 XT measurement was taken under it and did not show +# the 1.28x. If you see decodes exceed their reservation on another card, raise `device_working_mem_gb`. +_AUTOENCODER_KL_COEFFICIENTS = frozenset({"k_decode", "k_encode"}) +_CONV_BACKEND_KEYS = ("conv_backend",) + + +def _peak_of_two_phases( + convolution_phase: tuple[WorkingMemoryTerm, ...], score_matrix_bytes: int +) -> tuple[WorkingMemoryTerm, ...]: + """The convolution phase and the mid-block score matrix, of which only the larger is reserved. + + max, not sum: the two peak in different phases of the same forward. The mid-block sits at the 8x-downsampled + bottleneck -- first in the decoder, last in the encoder -- so the full-resolution convolution feature maps that drive + the linear term are not live while the score matrix is, and peak *reserved* is a high-water mark, not a running + total. Measured on the FLUX.2 VAE (the points and their method are in `data/vae.json`): on gfx1100 and gfx1201 + forcing `math` moves the measured peak by nothing at all up to 1024px, and the totals stay flat-linear in area + either way. On CUDA the score matrix only pokes above the convolution peak at 1536px, and then by 2.6GB against the + 21.5GB this term prices standalone, because the attention phase reuses blocks the allocator is already holding. + Summing them reserved 11.1GB for a 1024px gfx1100 decode that measures 6.7GB; taking the max reserves 6.9GB. + + Where a max model is weakest is the crossover, where the two terms are near-equal and whatever overlap exists is + no longer hidden. There is exactly one measured point like that: a 768px encode with cuDNN's linear coefficient and + a materializing kernel measures 1.80GB against a 1.35GB max. It is not reachable as a shortfall, because the cache + floors every reservation at `device_working_mem_gb` (3GB by default, see `ModelCache._get_vram_available`) and the + whole crossover region sits under that floor. On the builds that really do materialize -- gfx1100 and gfx1201 -- + the MIOpen coefficient keeps the linear term above the score term across the measured range, so the crossover does + not arise there at all. + + On a tie the convolutions are the ones reserved. + """ + convolution_bytes: float = 0 + for term in convolution_phase: + convolution_bytes += term.bytes + scores_win = score_matrix_bytes > convolution_bytes + return ( + *(replace(term, included=not scores_win) for term in convolution_phase), + WorkingMemoryTerm("mid_block_scores", score_matrix_bytes, included=scores_win), + ) + + +def _sd_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float]) -> tuple[WorkingMemoryTerm, ...]: + coefficient = f"k_{features['operation']}" + element_size = cast(int, features["element_size"]) + tile = cast(int | None, features["tile"]) + if tile is not None: + area_bytes = tile * tile * element_size + # We add 25% to the working memory estimate when tiling is enabled to account for factors like tile overlap + # and number of tiles. We could make this more precise in the future, but this should be good enough for + # most use cases. + convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * 1.25, values) + else: + area_bytes = cast(int, features["height"]) * cast(int, features["width"]) * element_size + convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes, values) + + terms: tuple[WorkingMemoryTerm, ...] = (convolutions,) + # The tiny autoencoder has no mid-block attention, so it is priced without a score matrix. + if features["vae_class"] == "AutoencoderKL": + terms = _peak_of_two_phases(terms, cast(int, features["score_matrix_bytes"])) + if features["fp32"]: + # If we are running in FP32, then we should account for the likely increase in model size (~250MB). + terms += (WorkingMemoryTerm("fp32_weights", 250 * 2**20, True),) + return terms + + +VAE_SD_SPEC = register( + EstimatorSpec( + id="vae.sd", + version=1, + coefficients=_AUTOENCODER_KL_COEFFICIENTS, + platform_keys=_CONV_BACKEND_KEYS, + terms=_sd_terms, + ) +) + + def estimate_vae_working_memory_sd15_sdxl( operation: Literal["encode", "decode"], image_tensor: torch.Tensor, @@ -117,47 +211,75 @@ def estimate_vae_working_memory_sd15_sdxl( tile_size: int | None, fp32: bool, device: torch.device | None = None, -) -> int: - """Estimate the working memory required to encode or decode the given tensor.""" - # It was found experimentally that the peak working memory scales linearly with the number of pixels and the - # element size (precision). This estimate is accurate for both SD1 and SDXL. - element_size = 4 if fp32 else 2 - - # The same AutoencoderKL convolution stack as the FLUX.1 autoencoder, so it takes the same measured constants per - # convolution backend: MIOpen needs ~1.64x what cuDNN does (see `_FLUX_VAE_SCALING_CONSTANTS`). `device` is where - # the VAE will run; it defaults to the session device. - scaling_constant = _flux_vae_scaling_constant(operation, device or TorchDevice.choose_torch_device()) +) -> WorkingMemoryEstimate: + """Estimate the working memory required to encode or decode the given tensor. + It was found experimentally that the peak working memory scales linearly with the number of pixels and the element + size (precision). This estimate is accurate for both SD1 and SDXL. `device` is where the VAE will run; it picks the + convolution backend's coefficients and defaults to the session device. + """ + element_size = 4 if fp32 else 2 + device = device if device is not None else TorchDevice.choose_torch_device() latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 + height = latent_scale_factor_for_operation * image_tensor.shape[-2] + width = latent_scale_factor_for_operation * image_tensor.shape[-1] + + if tile_size == 0: + tile_size = vae.tile_sample_min_size + assert isinstance(tile_size, int) + + features: dict[str, FeatureValue] = { + "operation": operation, + "height": height, + "width": width, + "tile": tile_size, + "element_size": element_size, + "fp32": fp32, + "vae_class": "AutoencoderKL" if isinstance(vae, AutoencoderKL) else "AutoencoderTiny", + } + if isinstance(vae, AutoencoderKL): + score_dtype = torch.float32 if fp32 else next(vae.parameters()).dtype + score_h, score_w = (tile_size, tile_size) if tile_size is not None else (height, width) + features["score_matrix_bytes"] = _vae_mid_block_score_matrix_bytes(score_h, score_w, score_dtype, device=device) - if tile_size is not None: - if tile_size == 0: - tile_size = vae.tile_sample_min_size - assert isinstance(tile_size, int) - h = tile_size - w = tile_size - working_memory = h * w * element_size * scaling_constant + return VAE_SD_SPEC.estimate(features, device) - # We add 25% to the working memory estimate when tiling is enabled to account for factors like tile overlap - # and number of tiles. We could make this more precise in the future, but this should be good enough for - # most use cases. - working_memory = working_memory * 1.25 - else: - h = latent_scale_factor_for_operation * image_tensor.shape[-2] - w = latent_scale_factor_for_operation * image_tensor.shape[-1] - working_memory = h * w * element_size * scaling_constant - if isinstance(vae, AutoencoderKL): - # max, not sum: the mid-block attention and the full-resolution convolutions peak in - # different phases of the forward (see the FLUX.2 estimator for the measurements). - score_dtype = torch.float32 if fp32 else next(vae.parameters()).dtype - working_memory = max(working_memory, _vae_mid_block_score_matrix_bytes(h, w, score_dtype, device=device)) +def _single_pass_terms( + features: Mapping[str, FeatureValue], values: Mapping[str, float] +) -> tuple[WorkingMemoryTerm, ...]: + """An untiled AutoencoderKL pass: the convolutions over the whole image, or the mid-block score matrix.""" + coefficient = f"k_{features['operation']}" + area_bytes = cast(int, features["height"]) * cast(int, features["width"]) * cast(int, features["element_size"]) + convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes, values) + return _peak_of_two_phases((convolutions,), cast(int, features["score_matrix_bytes"])) - if fp32: - # If we are running in FP32, then we should account for the likely increase in model size (~250MB). - working_memory += 250 * 2**20 - return int(working_memory) +def _single_pass_features( + operation: Literal["encode", "decode"], image_tensor: torch.Tensor, vae: AutoencoderKL, device: torch.device +) -> dict[str, FeatureValue]: + latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 + height = latent_scale_factor_for_operation * image_tensor.shape[-2] + width = latent_scale_factor_for_operation * image_tensor.shape[-1] + param = next(vae.parameters()) + return { + "operation": operation, + "height": height, + "width": width, + "element_size": param.element_size(), + "score_matrix_bytes": _vae_mid_block_score_matrix_bytes(height, width, param.dtype, device=device), + } + + +VAE_COGVIEW4_SPEC = register( + EstimatorSpec( + id="vae.cogview4", + version=1, + coefficients=_AUTOENCODER_KL_COEFFICIENTS, + platform_keys=_CONV_BACKEND_KEYS, + terms=_single_pass_terms, + ) +) def estimate_vae_working_memory_cogview4( @@ -165,25 +287,14 @@ def estimate_vae_working_memory_cogview4( image_tensor: torch.Tensor, vae: AutoencoderKL, device: torch.device | None = None, -) -> int: - """Estimate the working memory required by the invocation in bytes.""" - latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 - - h = latent_scale_factor_for_operation * image_tensor.shape[-2] - w = latent_scale_factor_for_operation * image_tensor.shape[-1] - element_size = next(vae.parameters()).element_size() - - # The same AutoencoderKL convolution stack as the FLUX.1 autoencoder, so it takes the same measured constants per - # convolution backend: MIOpen needs ~1.64x what cuDNN does (see `_FLUX_VAE_SCALING_CONSTANTS`). `device` is where - # the VAE will run; it defaults to the session device. - scaling_constant = _flux_vae_scaling_constant(operation, device or TorchDevice.choose_torch_device()) - working_memory = h * w * element_size * scaling_constant +) -> WorkingMemoryEstimate: + """Estimate the working memory required by the invocation in bytes. - # max, not sum: the mid-block attention and the full-resolution convolutions peak in different - # phases of the forward (see the FLUX.2 estimator for the measurements). - return int( - max(working_memory, _vae_mid_block_score_matrix_bytes(h, w, next(vae.parameters()).dtype, device=device)) - ) + `device` is where the VAE will run; it picks the convolution backend's coefficients and defaults to the session + device. + """ + device = device if device is not None else TorchDevice.choose_torch_device() + return VAE_COGVIEW4_SPEC.estimate(_single_pass_features(operation, image_tensor, vae, device), device) # What a tiled decode does *not* bound: the assembled image, several times over, per output pixel. @@ -217,13 +328,80 @@ def estimate_vae_working_memory_cogview4( _VAE_MOMENT_CHANNELS_PER_LATENT = 2 +def _flux_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float]) -> tuple[WorkingMemoryTerm, ...]: + operation = features["operation"] + coefficient = f"k_{operation}" + out_h = cast(int, features["height"]) + out_w = cast(int, features["width"]) + element_size = cast(int, features["element_size"]) + tile = cast(int | None, features["tile"]) + + if tile is None: + area_bytes = out_h * out_w * element_size + convolution_phase: tuple[WorkingMemoryTerm, ...] = ( + WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes, values), + ) + return _peak_of_two_phases(convolution_phase, cast(int, features["score_matrix_bytes"])) + + # `calc_tiles_min_overlap` clamps the tile to the image *per axis*, so the largest tile the + # decode builds is this, not `tile` squared. Without the clamp an 8192px tile on a + # 1024x8200 image -- tiled, because one axis exceeds it -- prices 8192x8192. + area_bytes = min(tile, out_h) * min(tile, out_w) * element_size + # A 25% margin for tile overlap and the number of tiles, mirroring the SD1/SDXL estimator. + convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * 1.25, values) + if operation == "decode": + convolution_phase = ( + convolutions, + WorkingMemoryTerm( + "assembled_image", + out_h * out_w * _IMAGE_CHANNELS * (_FLUX_VAE_TILED_IMAGE_COPIES * element_size + 1), + True, + ), + ) + else: + # A tiled encode leaves two things un-bounded: the full-resolution image it slices tiles + # from, and the moments it assembles. + # + # The moments are *not* one copy. `AutoencoderKL._tiled_encode` appends every encoded + # tile to `rows`, then builds `result_rows` from them -- blending against `rows[i-1][j]`, + # so `rows` stays live throughout -- and finally concatenates `enc` on top of both. The + # tiles overlap, so their total area is `1 / (1 - overlap_factor)**2` times the output; + # `result_rows` and `enc` are one output each. Hence the factor below, which is 3.78 at + # the 0.25 every `AutoencoderKL` ships. + live_moment_copies = 1.0 / (1.0 - cast(float, features["tile_overlap_factor"])) ** 2 + 2.0 + moments = ( + (out_h // LATENT_SCALE_FACTOR) + * (out_w // LATENT_SCALE_FACTOR) + * _VAE_MOMENT_CHANNELS_PER_LATENT + * cast(int, features["latent_channels"]) + * element_size + ) + convolution_phase = ( + convolutions, + WorkingMemoryTerm("input_image", out_h * out_w * _IMAGE_CHANNELS * element_size, True), + WorkingMemoryTerm("assembled_moments", moments * live_moment_copies, True), + ) + return _peak_of_two_phases(convolution_phase, cast(int, features["score_matrix_bytes"])) + + +VAE_FLUX_SPEC = register( + EstimatorSpec( + id="vae.flux", + version=1, + coefficients=_AUTOENCODER_KL_COEFFICIENTS, + platform_keys=_CONV_BACKEND_KEYS, + terms=_flux_terms, + ) +) + + def estimate_vae_working_memory_flux( operation: Literal["encode", "decode"], image_tensor: torch.Tensor, vae: AutoencoderKL, tile_size: int | None = None, device: torch.device | None = None, -) -> int: +) -> WorkingMemoryEstimate: """Estimate the working memory required by the invocation in bytes. `tile_size` is in output pixels and defaults to None, i.e. a single-pass decode -- the existing @@ -237,16 +415,11 @@ def estimate_vae_working_memory_flux( for the decode side. `device` is where the VAE will run (a `cpu_only` VAE runs on the CPU); it picks the convolution - backend's constant. Defaults to the session device. + backend's coefficients. Defaults to the session device. """ - latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 - element_size = next(vae.parameters()).element_size() - - # The FLUX.1 autoencoder shares the FLUX.2 VAE's convolution stack and measures the same constants on both - # backends; see `_FLUX_VAE_SCALING_CONSTANTS`. + param = next(vae.parameters()) device = device if device is not None else TorchDevice.choose_torch_device() - scaling_constant = _flux_vae_scaling_constant(operation, device) out_h = latent_scale_factor_for_operation * image_tensor.shape[-2] out_w = latent_scale_factor_for_operation * image_tensor.shape[-1] @@ -261,51 +434,27 @@ def estimate_vae_working_memory_flux( # field carries no upper bound: a 2048px tile on a 1024px image would otherwise reserve ~23GB # for a decode that needs ~4.6GB, evicting the transformer from the cache for nothing. if tile is not None and (tile < out_h or tile < out_w): - # `calc_tiles_min_overlap` clamps the tile to the image *per axis*, so the largest tile the - # decode builds is this, not `tile` squared. Without the clamp an 8192px tile on a - # 1024x8200 image -- tiled, because one axis exceeds it -- prices 8192x8192. - tile_h = min(tile, out_h) - tile_w = min(tile, out_w) - # A 25% margin for tile overlap and the number of tiles, mirroring the SD1/SDXL estimator. - working_memory = tile_h * tile_w * element_size * scaling_constant * 1.25 - if operation == "decode": - working_memory += out_h * out_w * _IMAGE_CHANNELS * (_FLUX_VAE_TILED_IMAGE_COPIES * element_size + 1) - else: - # A tiled encode leaves two things un-bounded: the full-resolution image it slices tiles - # from, and the moments it assembles. - # - # The moments are *not* one copy. `AutoencoderKL._tiled_encode` appends every encoded - # tile to `rows`, then builds `result_rows` from them -- blending against `rows[i-1][j]`, - # so `rows` stays live throughout -- and finally concatenates `enc` on top of both. The - # tiles overlap, so their total area is `1 / (1 - overlap_factor)**2` times the output; - # `result_rows` and `enc` are one output each. Hence the factor below, which is 3.78 at - # the 0.25 every `AutoencoderKL` ships. - # - # Read from the VAE rather than hard-coded: an overlap factor of 0.5 would make the - # tiles alone four times the output, and a fixed 3.78 would then under-reserve. An - # estimate consumed as `free >= estimate` has to err upwards. - config = getattr(vae, "config", None) - latent_channels = getattr(config, "latent_channels", 16) - overlap_factor = getattr(vae, "tile_overlap_factor", 0.25) - live_moment_copies = 1.0 / (1.0 - overlap_factor) ** 2 + 2.0 - moments = ( - (out_h // LATENT_SCALE_FACTOR) - * (out_w // LATENT_SCALE_FACTOR) - * _VAE_MOMENT_CHANNELS_PER_LATENT - * latent_channels - * element_size - ) - working_memory += out_h * out_w * _IMAGE_CHANNELS * element_size - working_memory += moments * live_moment_copies - score_h, score_w = tile_h, tile_w + score_h, score_w = min(tile, out_h), min(tile, out_w) else: - working_memory = out_h * out_w * element_size * scaling_constant + tile = None score_h, score_w = out_h, out_w - # max, not sum: the mid-block attention and the full-resolution convolutions peak in different - # phases of the forward (see the FLUX.2 estimator for the measurements). - dtype = next(vae.parameters()).dtype - return int(max(working_memory, _vae_mid_block_score_matrix_bytes(score_h, score_w, dtype, device=device))) + features: dict[str, FeatureValue] = { + "operation": operation, + "height": out_h, + "width": out_w, + "element_size": param.element_size(), + "tile": tile, + "score_matrix_bytes": _vae_mid_block_score_matrix_bytes(score_h, score_w, param.dtype, device=device), + } + if tile is not None and operation != "decode": + # Read from the VAE rather than hard-coded: an overlap factor of 0.5 would make the tiles + # alone four times the output, and a fixed 3.78 would then under-reserve. An estimate + # consumed as `free >= estimate` has to err upwards. + features["latent_channels"] = getattr(getattr(vae, "config", None), "latent_channels", 16) + features["tile_overlap_factor"] = getattr(vae, "tile_overlap_factor", 0.25) + + return VAE_FLUX_SPEC.estimate(features, device) # The FLUX.2 VAE runs one attention block at the bottom of the encoder and one at the top of the @@ -317,51 +466,32 @@ def estimate_vae_working_memory_flux( _FLUX2_VAE_MID_BLOCK_HEAD_DIM = 512 _FLUX2_VAE_SPATIAL_COMPRESSION = 8 -# Peak reserved bytes per output pixel per element byte, per conv backend and operation, for the FLUX.2 VAE and the -# FLUX.1 autoencoder alike. Fitted with `scripts/calibrate_flux2_working_memory.py --only vae`, which reproduces -# this table on any build: -# -# decode encode encode/decode -# cuDNN RTX 4090, torch 2.7.1 2185 1072 0.49 -# MIOpen RX 9070 XT, torch 2.10 3453 2688 0.78 -# MIOpen PRO W7900, torch 2.10 3525 2688 0.76 -# MIOpen RX 9060 XT, torch 2.12, Windows, FLUX.1 VAE 3451 2688 0.78 -# -# Two AMD generations (RDNA3 and RDNA4), the same numbers: the encode column agrees to the byte and -# the decode column to 2%. So this is MIOpen, not a per-card quirk, and the column below is fitted -# to the larger with ~2% headroom. The FLUX.1 row (bf16, 768-1536px, Invoke's ROCm SDPA guard installed, -# `MIOPEN_FIND_MODE=FAST`) lands on the same numbers: the two VAEs share their convolution stack. The -# cuDNN column is the FLUX.1 constant this estimator shipped with before. -# -# MIOpen's convolution workspaces are simply larger than cuDNN's. This is not the attention term -- -# it shows up identically on the fused path, and the implied constants are flat across resolution on -# both backends, so the linear model itself holds. And the "encoding costs half of decoding" ratio -# the other estimators in this module use turns out to be a cuDNN property rather than an -# architectural one, which is why the two operations carry their own numbers here instead of a ratio. -# -# Shipping the MIOpen numbers everywhere would add ~60% to every cuDNN decode for nothing, so the -# constant follows the backend. -# -# Caveat for AMD users: `MIOPEN_FIND_MODE=2` (FAST) selects convolution algorithms heuristically instead of -# by benchmark, and measured a uniform 1.28x more memory at every resolution when these constants were -# first fitted. Invoke has since made FAST its default (`run_app.py`); the RX 9060 XT row above was taken -# under it and did not show the 1.28x. If you see decodes exceed their reservation on another card, raise -# `device_working_mem_gb`. -_FLUX_VAE_SCALING_CONSTANTS: dict[str, dict[str, int]] = { - "cudnn": {"decode": 2200, "encode": 1100}, - "miopen": {"decode": 3600, "encode": 2750}, -} - -def _flux_vae_scaling_constant(operation: Literal["encode", "decode"], device: torch.device) -> int: - """Pick the pixel-area constant for the convolution backend this device will actually use. - - A HIP build reports ``device.type == "cuda"``, so the torch build -- not the device string -- is - what separates MIOpen from cuDNN. MPS and CPU are unmeasured and take the cuDNN column; on MPS - that pairs with a score-matrix term the probe always charges there, so the total is not thin. - """ - is_rocm = device.type == "cuda" and torch.version.hip is not None - return _FLUX_VAE_SCALING_CONSTANTS["miopen" if is_rocm else "cudnn"][operation] +def _flux2_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float]) -> tuple[WorkingMemoryTerm, ...]: + coefficient = f"k_{features['operation']}" + element_size = cast(int, features["element_size"]) + batch_size = cast(int, features["batch_size"]) + tile = cast(int | None, features["tile"]) + if tile is not None: + area_bytes = tile * tile * element_size + # Add 25% for tile overlap and the blending buffers, mirroring the SD1/SDXL estimate. + convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * 1.25 * batch_size, values) + else: + area_bytes = cast(int, features["height"]) * cast(int, features["width"]) * element_size + convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * batch_size, values) + # Both terms are per sample; the larger is reserved (see `_peak_of_two_phases` for why, and the measurements). + return _peak_of_two_phases((convolutions,), cast(int, features["score_matrix_bytes"])) + + +VAE_FLUX2_SPEC = register( + EstimatorSpec( + id="vae.flux2", + version=1, + coefficients=_AUTOENCODER_KL_COEFFICIENTS, + platform_keys=_CONV_BACKEND_KEYS, + terms=_flux2_terms, + ) +) def estimate_vae_working_memory_flux2( @@ -370,15 +500,15 @@ def estimate_vae_working_memory_flux2( vae: AutoencoderKLFlux2, tile_size: int | None = None, device: torch.device | None = None, -) -> int: +) -> WorkingMemoryEstimate: """Estimate the working memory required to encode or decode with the FLUX.2 (32-channel) VAE. Peak memory scales linearly with pixel area and element size, as it does for the FLUX.1 VAE, and - the implied constant is flat across 512-1536px on every backend measured. What is *not* constant - is the constant itself: MIOpen's convolution workspaces cost ~1.6x cuDNN's for a decode and ~2.4x - for an encode, so it is looked up per backend -- see ``_FLUX_VAE_SCALING_CONSTANTS`` for the - fitted table and the caveats. Peak *reserved* memory is what is measured throughout, the - conservative quantity that includes allocator overhead. + the implied coefficient is flat across 512-1536px on every backend measured. What is *not* constant + is the coefficient itself: MIOpen's convolution workspaces cost ~1.6x cuDNN's for a decode and ~2.4x + for an encode, so it is shipped per backend -- see `_AUTOENCODER_KL_COEFFICIENTS` for the caveats + and `backend/working_memory/data/vae.json` for the fitted values and their points. Peak *reserved* + memory is what is measured throughout, the conservative quantity that includes allocator overhead. For reference, decoding 1024x1024 peaks at ~4.3GB on cuDNN and ~6.6GB on MIOpen, and 1536x1536 at ~9.6GB -- far above the default ``device_working_mem_gb``, which is why this estimate must be @@ -394,10 +524,10 @@ def estimate_vae_working_memory_flux2( for everything else, so the estimate is right on all of them. The two terms are independent: a build can have a fused kernel and still need the larger - convolution constant (gfx1201 did, before the ROCm rule above). They also do not add -- see the - ``max`` at the end of this function for why, and for the measurements behind it. (Unlike the transformer, this attention does not go through - diffusers' attention dispatcher -- ``AttnProcessor2_0`` calls ``F.scaled_dot_product_attention`` - itself -- so torch's own answer is the whole answer here.) + convolution coefficient (gfx1201 did, before the ROCm rule above). They also do not add -- see + `_peak_of_two_phases` for why, and for the measurements behind it. (Unlike the transformer, this + attention does not go through diffusers' attention dispatcher -- ``AttnProcessor2_0`` calls + ``F.scaled_dot_product_attention`` itself -- so torch's own answer is the whole answer here.) When tiling is enabled the peak is bounded by a single tile instead of the full image (measured ~0.55GB flat at a 512px tile, from 1024px up to the 2024px reference-image cap), and the score @@ -411,53 +541,34 @@ def estimate_vae_working_memory_flux2( call sites all pass batch 1 today; the shared estimator does not assume it. """ param = next(vae.parameters()) - element_size = param.element_size() - device = device if device is not None else TorchDevice.choose_torch_device() - scaling_constant = _flux_vae_scaling_constant(operation, device) batch_size = image_tensor.shape[0] if image_tensor.dim() >= 4 else 1 + latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 + out_h = latent_scale_factor_for_operation * image_tensor.shape[-2] + out_w = latent_scale_factor_for_operation * image_tensor.shape[-1] if tile_size is not None: - # Add 25% for tile overlap and the blending buffers, mirroring the SD1/SDXL estimate. - working_memory = tile_size * tile_size * element_size * scaling_constant * 1.25 mid_block_seq_len = (tile_size // _FLUX2_VAE_SPATIAL_COMPRESSION) ** 2 else: - latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 - out_h = latent_scale_factor_for_operation * image_tensor.shape[-2] - out_w = latent_scale_factor_for_operation * image_tensor.shape[-1] - working_memory = out_h * out_w * element_size * scaling_constant mid_block_seq_len = (out_h // _FLUX2_VAE_SPATIAL_COMPRESSION) * (out_w // _FLUX2_VAE_SPATIAL_COMPRESSION) - working_memory *= batch_size - score_matrix_bytes = sdpa_score_matrix_bytes( - device=device, - dtype=param.dtype, - # The score matrix is (batch, heads, S, S); one head per sample prices the whole batch. - num_heads=_FLUX2_VAE_MID_BLOCK_HEADS * batch_size, - head_dim=_FLUX2_VAE_MID_BLOCK_HEAD_DIM, - seq_len=mid_block_seq_len, - ) - - # max, not sum: the two terms peak in different phases of the same forward. The mid-block sits at - # the 8x-downsampled bottleneck -- first in the decoder, last in the encoder -- so the full- - # resolution convolution feature maps that drive the linear term are not live while the score - # matrix is, and peak *reserved* is a high-water mark, not a running total. Measured (see the - # constants table above for the method): on gfx1100 and gfx1201 forcing `math` moves the measured - # peak by nothing at all up to 1024px, and the totals stay flat-linear in area either way. On - # CUDA the score matrix only pokes above the convolution peak at 1536px, and then by 2.6GB - # against the 21.5GB this term prices standalone, because the attention phase reuses blocks the - # allocator is already holding. Summing them reserved 11.1GB for a 1024px gfx1100 decode that - # measures 6.7GB; taking the max reserves 6.9GB. - # - # Where a max model is weakest is the crossover, where the two terms are near-equal and whatever - # overlap exists is no longer hidden. There is exactly one measured point like that: a 768px - # encode with cuDNN's linear constant and a materializing kernel measures 1.80GB against a - # 1.35GB max. It is not reachable as a shortfall, because the cache floors every reservation at - # `device_working_mem_gb` (3GB by default, see `ModelCache._get_vram_available`) and the whole - # crossover region sits under that floor. On the builds that really do materialize -- gfx1100 and - # gfx1201 -- the MIOpen constant keeps the linear term above the score term across the measured - # range, so the crossover does not arise there at all. - return int(max(working_memory, score_matrix_bytes)) + features: dict[str, FeatureValue] = { + "operation": operation, + "height": out_h, + "width": out_w, + "tile": tile_size, + "batch_size": batch_size, + "element_size": param.element_size(), + "score_matrix_bytes": sdpa_score_matrix_bytes( + device=device, + dtype=param.dtype, + # The score matrix is (batch, heads, S, S); one head per sample prices the whole batch. + num_heads=_FLUX2_VAE_MID_BLOCK_HEADS * batch_size, + head_dim=_FLUX2_VAE_MID_BLOCK_HEAD_DIM, + seq_len=mid_block_seq_len, + ), + } + return VAE_FLUX2_SPEC.estimate(features, device) def estimate_vae_working_memory_anima( @@ -954,30 +1065,27 @@ def estimate_vae_working_memory_qwen_image( return int(working_memory) +VAE_SD3_SPEC = register( + EstimatorSpec( + id="vae.sd3", + version=1, + coefficients=_AUTOENCODER_KL_COEFFICIENTS, + platform_keys=_CONV_BACKEND_KEYS, + terms=_single_pass_terms, + ) +) + + def estimate_vae_working_memory_sd3( operation: Literal["encode", "decode"], image_tensor: torch.Tensor, vae: AutoencoderKL, device: torch.device | None = None, -) -> int: - """Estimate the working memory required by the invocation in bytes.""" - # Encode operations use approximately 50% of the memory required for decode operations - - latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 - - h = latent_scale_factor_for_operation * image_tensor.shape[-2] - w = latent_scale_factor_for_operation * image_tensor.shape[-1] - element_size = next(vae.parameters()).element_size() - - # The same AutoencoderKL convolution stack as the FLUX.1 autoencoder, so it takes the same measured constants per - # convolution backend: MIOpen needs ~1.64x what cuDNN does (see `_FLUX_VAE_SCALING_CONSTANTS`). `device` is where - # the VAE will run; it defaults to the session device. - scaling_constant = _flux_vae_scaling_constant(operation, device or TorchDevice.choose_torch_device()) - - working_memory = h * w * element_size * scaling_constant +) -> WorkingMemoryEstimate: + """Estimate the working memory required by the invocation in bytes. - # max, not sum: the mid-block attention and the full-resolution convolutions peak in different - # phases of the forward (see the FLUX.2 estimator for the measurements). - return int( - max(working_memory, _vae_mid_block_score_matrix_bytes(h, w, next(vae.parameters()).dtype, device=device)) - ) + `device` is where the VAE will run; it picks the convolution backend's coefficients and defaults to the session + device. + """ + device = device if device is not None else TorchDevice.choose_torch_device() + return VAE_SD3_SPEC.estimate(_single_pass_features(operation, image_tensor, vae, device), device) diff --git a/invokeai/backend/working_memory/__init__.py b/invokeai/backend/working_memory/__init__.py new file mode 100644 index 00000000000..dd71f347f51 --- /dev/null +++ b/invokeai/backend/working_memory/__init__.py @@ -0,0 +1,6 @@ +"""Working-memory estimates as data: what an operation needs on the compute device, and how that was priced. + +An estimator turns an operation's runtime facts (`features`) and a set of fitted coefficients into a +`WorkingMemoryEstimate` that `LoadedModel.model_on_device(working_memory=...)` reserves. The coefficients ship as data +under `data/`, next to the measured points that bound them; `tests/backend/working_memory/` enforces both. +""" diff --git a/invokeai/backend/working_memory/coefficients.py b/invokeai/backend/working_memory/coefficients.py new file mode 100644 index 00000000000..d8e07b13854 --- /dev/null +++ b/invokeai/backend/working_memory/coefficients.py @@ -0,0 +1,191 @@ +"""The coefficients estimators are evaluated with, and the measured points that bound them. + +Both ship as JSON under `data/`, read through `importlib.resources` so a wheel install finds them. A coefficient entry +belongs to one estimator version on one measured platform value, and either carries its values or borrows another +entry's (`values_from`); a point records the features of an operation that was measured and its measured peak. +`tests/backend/working_memory/test_shipped_coefficients.py` resolves every entry against its `EstimatorSpec` and holds +every point to the estimate the shipped coefficients give it. +""" + +import functools +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from importlib.resources import files +from importlib.resources.abc import Traversable +from types import MappingProxyType +from typing import TYPE_CHECKING, Literal, Self + +from pydantic import BaseModel, ConfigDict, ValidationError, model_validator + +from invokeai.backend.working_memory.estimate import FeatureValue +from invokeai.backend.working_memory.platform import UNMEASURED_PRICED_AS, Platform + +if TYPE_CHECKING: + from invokeai.backend.working_memory.spec import EstimatorSpec + + +class _DataModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ValuesFrom(_DataModel): + """The entry whose values another entry takes: same platform, this estimator id and version.""" + + estimator: str + version: int + + +class CoefficientEntry(_DataModel): + """The coefficient values of one estimator version on one platform: its own, or another entry's.""" + + estimator: str + version: int + platform: dict[str, str] + values: dict[str, float] | None = None + values_from: ValuesFrom | None = None + """Borrow the values of another estimator's entry on the same platform, so that its measured points bound this + estimator too. One hop: the entry borrowed from carries its own values.""" + provenance: str + """Card, torch build, method and script the values come from, and which points they were fitted on.""" + + @model_validator(mode="after") + def _values_or_values_from(self) -> Self: + if (self.values is None) == (self.values_from is None): + raise ValueError("a coefficient entry carries exactly one of `values` and `values_from`") + return self + + +class MeasuredPoint(_DataModel): + """One measured operation. The estimate for its features must be at least `measured_bytes`, and where a fitted + term decides it, at most 1.3 times that (or `max_ratio`).""" + + estimator: str + version: int + platform: dict[str, str] + features: dict[str, FeatureValue] + measured_bytes: int + method: Literal["reserved_peak"] + """``reserved_peak``: the caching allocator's peak reserved bytes over the operation, one fresh process per + point. Reserved rather than allocated, because reserved is what the model cache's reservation has to cover.""" + device: str + provenance: str + max_ratio: float | None = None + """A tighter upper bound than the default 1.3, as a multiple of `measured_bytes`, for a point the estimate has to + stay closer to. It applies whichever term decides the estimate.""" + + +class WorkingMemoryData(_DataModel): + """One file under `data/`.""" + + schema_version: Literal[1] + coefficients: list[CoefficientEntry] + points: list[MeasuredPoint] + + +@dataclass(frozen=True) +class ResolvedCoefficients: + """The coefficient values an estimate is evaluated with, and where they came from.""" + + values: Mapping[str, float] + source: str + + +def _parse(entry: Traversable) -> WorkingMemoryData: + try: + return WorkingMemoryData.model_validate_json(entry.read_text(encoding="utf-8")) + except ValidationError as e: + raise ValueError(f"Malformed working-memory data file {entry}: {e}") from e + + +def load_data(directory: Traversable) -> Mapping[str, WorkingMemoryData]: + """Every JSON data file in `directory`, parsed, by file name. + + A directory with none raises, naming it: every estimator resolves its coefficients from there, so an install that + lost the files would otherwise fail later as a missing entry for whichever estimator ran first. + """ + entries = ( + sorted((entry for entry in directory.iterdir() if entry.name.endswith(".json")), key=lambda entry: entry.name) + if directory.is_dir() + else [] + ) + if not entries: + raise FileNotFoundError(f"No working-memory data files in {directory}; every EstimatorSpec reads them there.") + return MappingProxyType({entry.name: _parse(entry) for entry in entries}) + + +@functools.cache +def shipped_data() -> Mapping[str, WorkingMemoryData]: + """The data files shipped under `data/`. Read once per process.""" + return load_data(files("invokeai.backend.working_memory") / "data") + + +def _where(keys: Mapping[str, str]) -> str: + return ",".join(f"{key}={value}" for key, value in keys.items()) + + +def _entry( + entries: list[CoefficientEntry], estimator: str, version: int, keys: Mapping[str, str], where: str +) -> CoefficientEntry: + candidates = [entry for entry in entries if entry.estimator == estimator and entry.platform == keys] + matches = [entry for entry in candidates if entry.version == version] + if len(matches) > 1: + raise LookupError( + f"Working-memory coefficients for estimator '{estimator}' version {version} on {where} are declared " + f"{len(matches)} times." + ) + if not matches: + found = f" (found only version {sorted({entry.version for entry in candidates})})" if candidates else "" + raise LookupError( + f"No working-memory coefficients for estimator '{estimator}' version {version} on {where}{found}." + ) + return matches[0] + + +def resolve_coefficients( + data: Iterable[WorkingMemoryData], spec: "EstimatorSpec", platform: Platform, origin: str +) -> ResolvedCoefficients: + """The values `spec` takes on `platform`, and a `source` that says how they were found. + + A platform value nothing is measured for is priced as the measured value `UNMEASURED_PRICED_AS` names. The entry + for the spec's id, version and those platform values either carries the values or borrows them, one hop, from + another estimator's entry on the same platform. No other fallback: an estimate priced with coefficients fitted for + something else is the shortfall this data exists to prevent, so a missing, ambiguous, chained or mismatched entry + raises, naming what is wrong. `origin` prefixes the `source`. + """ + requested = {key: getattr(platform, key) for key in spec.platform_keys} + keys = {key: UNMEASURED_PRICED_AS.get(key, {}).get(value, value) for key, value in requested.items()} + where = _where(keys) if keys == requested else f"{_where(keys)} (priced for {_where(requested)})" + entries = [entry for file in data for entry in file.coefficients] + + entry = _entry(entries, spec.id, spec.version, keys, where) + values, lender = entry.values, None + if entry.values_from is not None: + lender = entry.values_from.estimator + lent = _entry(entries, lender, entry.values_from.version, keys, where) + if lent.values is None: + raise LookupError( + f"Working-memory coefficients for estimator '{spec.id}' on {where} borrow from '{lender}', whose " + "entry borrows in turn; borrow from the entry that carries the values." + ) + values = lent.values + assert values is not None, "a validated entry carries values or values_from" + if set(values) != spec.coefficients: + raise LookupError( + f"Working-memory coefficients for estimator '{spec.id}' on {where} provide {sorted(values)}" + f"{f' (from {lender})' if lender else ''}; the estimator reads {sorted(spec.coefficients)}." + ) + + priced = ",".join(value for key, value in keys.items() if value != requested[key]) + if priced and lender: + note = f" ({priced} values of {lender})" + elif priced: + note = f" ({priced} values)" + else: + note = f" (values of {lender})" if lender else "" + return ResolvedCoefficients(values=MappingProxyType(dict(values)), source=f"{origin}:{_where(requested)}{note}") + + +@functools.cache +def shipped_coefficients(spec: "EstimatorSpec", platform: Platform) -> ResolvedCoefficients: + """The shipped values `spec` takes on `platform`. Resolved once per spec and platform.""" + return resolve_coefficients(shipped_data().values(), spec, platform, origin="shipped") diff --git a/invokeai/backend/working_memory/data/vae.json b/invokeai/backend/working_memory/data/vae.json new file mode 100644 index 00000000000..bae9081384f --- /dev/null +++ b/invokeai/backend/working_memory/data/vae.json @@ -0,0 +1,769 @@ +{ + "schema_version": 1, + "coefficients": [ + { + "estimator": "vae.sd", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + }, + { + "estimator": "vae.sd", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + }, + { + "estimator": "vae.sd3", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD3 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + }, + { + "estimator": "vae.sd3", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD3 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + }, + { + "estimator": "vae.cogview4", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the CogView4 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + }, + { + "estimator": "vae.cogview4", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the CogView4 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + }, + { + "estimator": "vae.flux", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. The FLUX.1 point below falls inside them." + }, + { + "estimator": "vae.flux", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "values_from": { + "estimator": "vae.flux2", + "version": 1 + }, + "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. Confirmed on the FLUX.1 VAE: RX 9060 XT (gfx1200), torch 2.12, Windows, bf16, 768-1536px, Invoke's ROCm SDPA guard installed, MIOPEN_FIND_MODE=FAST: implied 3451 decode / 2688 encode." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "values": { + "k_decode": 2200, + "k_encode": 1100 + }, + "provenance": "Peak reserved bytes per output pixel per element byte of the diffusers AutoencoderKL convolution stack. Fitted on the FLUX.2 VAE (the vae.flux2 cudnn points) with scripts/calibrate_flux2_working_memory.py --only vae: RTX 4090, torch 2.7.1, bf16, untiled, fused SDPA; implied 2180/2185/2165 decode and 1072/1063/1061 encode at 512/768/1024px, flat across resolution. Rounded up to 2200 / 1100, the pair the FLUX.1 estimator already shipped." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "values": { + "k_decode": 3600, + "k_encode": 2750 + }, + "provenance": "Peak reserved bytes per output pixel per element byte of the diffusers AutoencoderKL convolution stack on MIOpen. Fitted on the FLUX.2 VAE (the vae.flux2 miopen points) with scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled: RX 9070 XT (gfx1201) implied 3453/3369/3368 decode, PRO W7900 (gfx1100) 3525 at 512px; encode 2687/2688/2688 on both cards, to the byte. Two AMD generations agreeing that closely make this MIOpen rather than a per-card quirk, so the values are the larger card's plus ~2% headroom: 3600 / 2750. MIOpen's convolution workspaces are larger than cuDNN's (~1.6x decode, ~2.4x encode), identically with fused and math attention, and its encode/decode ratio is 0.78 against cuDNN's 0.49." + } + ], + "points": [ + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1142461301, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2576980378, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4540854174, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 561566974, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1254130450, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2224793059, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7075958620, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, bf16, untiled, fused SDPA: the fused column of the max-versus-sum measurement (#9519). Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1138753536, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2531262464, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4466933760, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1328, + "width": 1328, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7493124096, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 10043260928, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 1328, + "width": 1328, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3168796672, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 561566974, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 1443889152 + }, + "measured_bytes": 1933809025, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 4563402752 + }, + "measured_bytes": 4372276707, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 4563402752 + }, + "measured_bytes": 3787087413, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 11141120000 + }, + "measured_bytes": 6360846565, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 23102226432 + }, + "measured_bytes": 12847320924, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1810328715, + "method": "reserved_peak", + "device": "RX 9070 XT (gfx1201)", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 1443889152 + }, + "measured_bytes": 3973918491, + "method": "reserved_peak", + "device": "RX 9070 XT (gfx1201)", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 4563402752 + }, + "measured_bytes": 7063073718, + "method": "reserved_peak", + "device": "RX 9070 XT (gfx1201)", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1847909679, + "method": "reserved_peak", + "device": "PRO W7900 (gfx1100)", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals.", + "max_ratio": 1.15 + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 1443889152 + }, + "measured_bytes": 4059817837, + "method": "reserved_peak", + "device": "PRO W7900 (gfx1100)", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals.", + "max_ratio": 1.15 + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 4563402752 + }, + "measured_bytes": 7197291446, + "method": "reserved_peak", + "device": "PRO W7900 (gfx1100)", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals.", + "max_ratio": 1.15 + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1408749273, + "method": "reserved_peak", + "device": "RX 9070 XT (gfx1201) and PRO W7900 (gfx1100), identical to the byte", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 1443889152 + }, + "measured_bytes": 3170759606, + "method": "reserved_peak", + "device": "RX 9070 XT (gfx1201) and PRO W7900 (gfx1100), identical to the byte", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "miopen" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 4563402752 + }, + "measured_bytes": 5637144576, + "method": "reserved_peak", + "device": "RX 9070 XT (gfx1201) and PRO W7900 (gfx1100), identical to the byte", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 2, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 8541616210, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "#9519: 1024px decode by latent batch, CUDA, bf16, fused SDPA, one fresh process per point: 4.23 / 7.96 / 11.89 GiB at batch 1 / 2 / 3. The test table wrote them as GiB x 1000 (4229, 7955, 11889); batch 1 is the 4.229 GiB 1024px decode above." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "tile": null, + "batch_size": 3, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 12765716546, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "#9519: 1024px decode by latent batch, CUDA, bf16, fused SDPA, one fresh process per point: 4.23 / 7.96 / 11.89 GiB at batch 1 / 2 / 3. The test table wrote them as GiB x 1000 (4229, 7955, 11889); batch 1 is the 4.229 GiB 1024px decode above." + }, + { + "estimator": "vae.flux2", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 2024, + "width": 2024, + "tile": 512, + "batch_size": 1, + "element_size": 2, + "score_matrix_bytes": 0 + }, + "measured_bytes": 585105408, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "#9519: reference-image encode at the 2024px reference cap with the forced 512px tile, CUDA, bf16, fused SDPA; flat at ~0.55 GB from 1024px up. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 1, + "platform": { + "conv_backend": "cudnn" + }, + "features": { + "operation": "encode", + "height": 3072, + "width": 3072, + "element_size": 2, + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 20229295964, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "Commit 326bdf1bd0: the FLUX.1 VAE as a diffusers AutoencoderKL, bf16, fused SDPA, untiled 3072px encode: 18.84 GiB. Re-measured 18.57 GiB on torch 2.13+cu130, 2026-10-17, with scripts/calibrate_flux_vae_working_memory.py." + } + ] +} diff --git a/invokeai/backend/working_memory/estimate.py b/invokeai/backend/working_memory/estimate.py new file mode 100644 index 00000000000..1f388687557 --- /dev/null +++ b/invokeai/backend/working_memory/estimate.py @@ -0,0 +1,52 @@ +from collections.abc import Mapping +from dataclasses import dataclass, field + +FeatureValue = int | float | str | bool | None +"""One runtime fact an estimator is evaluated on: a size, an element width, a tile, a byte count it measured.""" + + +@dataclass(frozen=True) +class WorkingMemoryTerm: + """One part of a working-memory estimate. + + A fitted term is proportional to a data coefficient; `fitted` builds it, so its `bytes` are exactly the + coefficient's value times `scale`. An analytic term (a score matrix, an image buffer) carries neither. `included` is + False for a term the estimate priced but does not reserve, such as the smaller of two phases that peak apart. + """ + + name: str + bytes: float + included: bool + coefficient: str | None = None + scale: float | None = None + + @classmethod + def fitted(cls, name: str, coefficient: str, scale: float, values: Mapping[str, float]) -> "WorkingMemoryTerm": + """A term of `scale` units at `values[coefficient]` bytes per unit.""" + return cls(name, values[coefficient] * scale, True, coefficient, scale) + + +@dataclass(frozen=True) +class WorkingMemoryEstimate: + """What an operation is expected to need on its compute device, and the terms that add up to it.""" + + estimator: str + """The id of the `EstimatorSpec` that produced it, e.g. ``vae.flux2``.""" + version: int + features: Mapping[str, FeatureValue] + """Everything the estimate was evaluated from, runtime decisions such as the tile and the score-matrix size + included, so it can be evaluated again from these alone.""" + terms: tuple[WorkingMemoryTerm, ...] + source: str + """Where the coefficients came from, e.g. ``shipped:conv_backend=miopen (values of vae.flux2)``.""" + reservation_bytes: int = field(init=False) + """What the model cache keeps free: the included terms' bytes, truncated. Derived from the terms, never given.""" + + def __post_init__(self) -> None: + # One term at a time in term order rather than `sum()`, which compensates float rounding since Python 3.12, + # so the reservation is exactly the running total the terms describe. + total: float = 0 + for term in self.terms: + if term.included: + total += term.bytes + object.__setattr__(self, "reservation_bytes", int(total)) diff --git a/invokeai/backend/working_memory/platform.py b/invokeai/backend/working_memory/platform.py new file mode 100644 index 00000000000..5dfc2a00997 --- /dev/null +++ b/invokeai/backend/working_memory/platform.py @@ -0,0 +1,45 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal + +import torch + +ConvBackend = Literal["cudnn", "miopen", "mps", "xpu", "cpu"] + + +@dataclass(frozen=True) +class Platform: + """The traits of the machine an operation runs on, as coefficients are keyed. Each field is a platform key an + `EstimatorSpec` can name, and holds the true value: what was measured, and what was not, is `UNMEASURED_PRICED_AS`'s + business.""" + + conv_backend: ConvBackend + + +UNMEASURED_PRICED_AS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( + { + # MPS, XPU and CPU convolutions are unmeasured and take the cuDNN coefficients. On MPS that pairs with a + # score-matrix term the SDPA probe always charges there, so the total is not thin. + "conv_backend": MappingProxyType({"mps": "cudnn", "xpu": "cudnn", "cpu": "cudnn"}), + } +) +"""Per platform key, the values no coefficients are measured for, and the measured value each is priced as. The one +place that borrowing is declared: the resolver applies it and names it in an estimate's `source`, and shipped data +carries entries for measured values only.""" + + +def platform_for(device: torch.device) -> Platform: + """The platform an operation on `device` runs on. + + A HIP build reports ``device.type == "cuda"``, so the torch build -- not the device string -- is what separates + MIOpen from cuDNN, and a `cpu_only` model on a ROCm build is not on MIOpen. Any device type other than CUDA, MPS and + XPU runs torch's CPU kernels, or none at all, and is priced as the CPU. + """ + if device.type == "cuda": + return Platform(conv_backend="miopen" if torch.version.hip is not None else "cudnn") + if device.type == "mps": + return Platform(conv_backend="mps") + if device.type == "xpu": + return Platform(conv_backend="xpu") + return Platform(conv_backend="cpu") diff --git a/invokeai/backend/working_memory/spec.py b/invokeai/backend/working_memory/spec.py new file mode 100644 index 00000000000..3f0c116a2eb --- /dev/null +++ b/invokeai/backend/working_memory/spec.py @@ -0,0 +1,99 @@ +import importlib +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final + +import torch + +from invokeai.backend.working_memory.coefficients import ResolvedCoefficients, shipped_coefficients +from invokeai.backend.working_memory.estimate import FeatureValue, WorkingMemoryEstimate, WorkingMemoryTerm +from invokeai.backend.working_memory.platform import platform_for + +TermsFunction = Callable[[Mapping[str, FeatureValue], Mapping[str, float]], tuple[WorkingMemoryTerm, ...]] +"""An estimator's pure part: its features and its coefficient values in, its terms out.""" + +ESTIMATOR_MODULES: Final = ("invokeai.backend.util.vae_working_memory",) +"""Every module that registers an `EstimatorSpec`. The registry holds only the specs of modules imported so far, so a +caller that needs all of them imports these (`import_estimator_modules`); `tests/backend/working_memory/` fails on a +module that declares a spec without being listed here.""" + + +@dataclass(frozen=True) +class EstimatorSpec: + """What a working-memory estimator declares: its id and version, the coefficients its fitted terms read, and the + platform keys those coefficients are shipped per. + + Registered at import of the module that defines the estimator. `terms` is that estimator's pure part, so an + estimate can be evaluated again from recorded features alone -- which is how the measured points under `data/` are + checked. It is behaviour rather than declaration and stays out of equality and hashing. + """ + + id: str + version: int + """Bumped whenever `terms` changes the formula, so coefficients fitted to the old one no longer resolve.""" + coefficients: frozenset[str] + platform_keys: tuple[str, ...] + terms: TermsFunction = field(compare=False, repr=False) + + def estimate(self, features: Mapping[str, FeatureValue], device: torch.device) -> WorkingMemoryEstimate: + """The estimate for an operation on `device`, priced with the shipped coefficients of the platform it runs on.""" + return self.evaluate(features, shipped_coefficients(self, platform_for(device))) + + def evaluate( + self, features: Mapping[str, FeatureValue], coefficients: ResolvedCoefficients + ) -> WorkingMemoryEstimate: + """The estimate for `features` with the given coefficients.""" + return WorkingMemoryEstimate( + estimator=self.id, + version=self.version, + features=features, + terms=self.terms(features, coefficients.values), + source=coefficients.source, + ) + + +_SPECS: Final[dict[str, EstimatorSpec]] = {} + + +def _defined_at(terms: TermsFunction) -> str: + return f"{terms.__module__}.{getattr(terms, '__qualname__', repr(terms))}" + + +def register(spec: EstimatorSpec) -> EstimatorSpec: + """Declare an estimator, once per id, from the module that defines it. + + `importlib.reload` of that module declares the same spec again, from a new `terms` function of the same module and + name: a no-op that adopts the re-executed function. Anything else under an id already taken raises -- differing + declared data, the same data from another function, or the very same function object again (two specs of one module + sharing a `terms` function and, by a copy-paste slip, an id). Each would otherwise silently replace one estimator's + formula or coefficients with another's. + + jurigged (`--dev_reload`) patches function bodies in place and re-runs only the top-level statements that changed: + an edited `terms` body takes effect with nothing re-registered, but an edited `register(...)` statement raises even + when its data is unchanged (a reformat, a comment), because it hands over the same function object. Restart the + server then. + """ + declared = _SPECS.get(spec.id) + if declared is not None: + if declared != spec: + raise ValueError(f"Working-memory estimator '{spec.id}' is already registered as {declared}.") + if declared.terms is spec.terms or _defined_at(declared.terms) != _defined_at(spec.terms): + raise ValueError( + f"Working-memory estimator '{spec.id}' is already registered by {_defined_at(declared.terms)}; " + "ids are unique per estimator." + ) + _SPECS[spec.id] = spec + return spec + + +def registered_specs() -> Mapping[str, EstimatorSpec]: + """Every spec registered so far, by id: those of the estimator modules imported in this process.""" + return MappingProxyType(_SPECS) + + +def import_estimator_modules() -> Mapping[str, EstimatorSpec]: + """Every spec there is: imports `ESTIMATOR_MODULES` first, so the answer does not depend on import order.""" + for name in ESTIMATOR_MODULES: + importlib.import_module(name) + return registered_specs() diff --git a/pyproject.toml b/pyproject.toml index 786a71e56d0..e57bc3dd037 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -548,6 +548,8 @@ version = { attr = "invokeai.version.__version__" } # absent from a wheel install and that loader raises FileNotFoundError. "invokeai.backend.minimax_h3" = ["*.json"] "invokeai.backend.t5" = ["tokenizer/*.json"] +# Fitted working-memory coefficients, read by every estimator that declares an EstimatorSpec. +"invokeai.backend.working_memory" = ["data/*.json"] "invokeai.backend.clip" = ["*.json", "tokenizer/*.json", "tokenizer/*.gz"] "invokeai.app.services.image_index" = ["*.txt"] "invokeai.app.services.workflow_records.default_workflows" = ["*.json"] diff --git a/scripts/calibrate_flux2_working_memory.py b/scripts/calibrate_flux2_working_memory.py index d7d6ca34349..c429c5b9a49 100644 --- a/scripts/calibrate_flux2_working_memory.py +++ b/scripts/calibrate_flux2_working_memory.py @@ -7,8 +7,8 @@ 1. ``SDPA_MATH_BYTES_PER_SCORE_ELEMENT`` (``backend/util/attention.py``) -- bytes per element of the score matrix, charged only where SDPA has no fused kernel and materializes it. -2. ``estimate_vae_working_memory_flux2``'s 2200 / 1100 bytes per pixel per element byte - (``backend/util/vae_working_memory.py``). +2. ``estimate_vae_working_memory_flux2``'s ``k_decode`` / ``k_encode`` bytes per pixel per element byte, + shipped per convolution backend in ``backend/working_memory/data/vae.json``. 3. ``Flux2DenoiseInvocation._estimate_working_memory``'s 0.4 MB per token, defined at the Klein 9B width (4096) and scaled linearly by the loaded variant's width. 4. The dispatch question underneath all of it: does this build's SDPA fuse these shapes, or does it @@ -67,14 +67,19 @@ from invokeai.backend.model_manager.configs.flux2_variant import flux2_hidden_size from invokeai.backend.model_manager.taxonomy import Flux2VariantType from invokeai.backend.util.attention import SDPA_MATH_BYTES_PER_SCORE_ELEMENT, sdpa_score_matrix_bytes -from invokeai.backend.util.vae_working_memory import ( - _flux_vae_scaling_constant, - estimate_vae_working_memory_flux2, -) +from invokeai.backend.util.vae_working_memory import VAE_FLUX2_SPEC, estimate_vae_working_memory_flux2 +from invokeai.backend.working_memory.coefficients import shipped_coefficients +from invokeai.backend.working_memory.platform import platform_for GIB = 1024**3 MIB = 1024**2 + +def _shipped_vae_coefficient(operation: str) -> float: + """The VAE coefficient this build's estimate uses for `operation`: the column its convolution backend selects.""" + return shipped_coefficients(VAE_FLUX2_SPEC, platform_for(torch.device("cuda"))).values[f"k_{operation}"] + + # The FLUX.2 VAE attends on the 8x-downsampled grid with a single 512-wide head. Mirrors # `_FLUX2_VAE_*` in vae_working_memory.py. VAE_SPATIAL_COMPRESSION = 8 @@ -264,7 +269,9 @@ def run() -> None: if peak is None: return row - estimate = estimate_vae_working_memory_flux2(operation=operation, image_tensor=x, vae=vae, device=device) + estimate = estimate_vae_working_memory_flux2( + operation=operation, image_tensor=x, vae=vae, device=device + ).reservation_bytes seq_len = (px // VAE_SPATIAL_COMPRESSION) ** 2 score_bytes = sdpa_score_matrix_bytes( device=device, dtype=param.dtype, num_heads=1, head_dim=VAE_MID_BLOCK_HEAD_DIM, seq_len=seq_len @@ -278,8 +285,8 @@ def run() -> None: "reserved_delta": peak, "estimate": estimate, "score_term": score_bytes, - # The whole measured peak over the pixel area, which is what the 2200/1100-style literals - # name. The score matrix is deliberately NOT backed out: the two terms peak in different + # The whole measured peak over the pixel area, which is what the `k_decode`/`k_encode` + # coefficients name. The score matrix is deliberately NOT backed out: the two terms peak in different # phases of the forward, so the estimate takes the larger of them rather than the sum, and # the convolution phase is what sets the high-water mark until the quadratic term overtakes # it past ~1280px. Where `score_term` exceeds the measured peak, this column is not the @@ -457,10 +464,10 @@ def report_sdpa(shapes: list[tuple[int, int, int]], dtype_name: str) -> list[dic def report_vae(pxs: list[int], dtype_name: str, vae_path: str | None) -> list[dict]: - decode_k = _flux_vae_scaling_constant("decode", torch.device("cuda")) - encode_k = _flux_vae_scaling_constant("encode", torch.device("cuda")) - print(f"\n=== 3. VAE linear constants (this build selects {decode_k} decode / {encode_k} encode) ===") - print("`implied_k` is the measured peak over pixel area, directly comparable to those literals;") + decode_k = _shipped_vae_coefficient("decode") + encode_k = _shipped_vae_coefficient("encode") + print(f"\n=== 3. VAE linear coefficients (this build selects {decode_k:.0f} decode / {encode_k:.0f} encode) ===") + print("`implied_k` is the measured peak over pixel area, directly comparable to those coefficients;") print("fit on the rows whose `math` column matches what this build really does (section 1).") print("`covered` is the question that matters: is the shipped estimate an upper bound here?") print("Caveat: forcing math on a build that HAS a fused kernel is not equivalent to a build that") @@ -495,7 +502,7 @@ def report_vae(pxs: list[int], dtype_name: str, vae_path: str | None) -> list[di _flag_non_monotonic(rows) for operation in ("decode", "encode"): # Compare against the column this build actually selects, not against a hard-coded number. - shipped = _flux_vae_scaling_constant(operation, torch.device("cuda")) + shipped = _shipped_vae_coefficient(operation) for force_math in (False, True): ks = [ r["implied_linear_constant"] @@ -511,7 +518,7 @@ def report_vae(pxs: list[int], dtype_name: str, vae_path: str | None) -> list[di continue mode = "math" if force_math else "fused" verdict = "OK" if shipped >= max(ks) else "SHORT" - print(f" {operation} ({mode}): implied_k max = {max(ks):.0f}, shipped = {shipped} -> {verdict}") + print(f" {operation} ({mode}): implied_k max = {max(ks):.0f}, shipped = {shipped:.0f} -> {verdict}") short = [r for r in rows if not r.get("oom") and not r["covered"]] if short: print("\n Points the shipped estimate does NOT cover:") diff --git a/scripts/calibrate_flux_vae_working_memory.py b/scripts/calibrate_flux_vae_working_memory.py index c737005939e..9c62db1c2ef 100644 --- a/scripts/calibrate_flux_vae_working_memory.py +++ b/scripts/calibrate_flux_vae_working_memory.py @@ -93,7 +93,7 @@ def _worker(vae_path: str, operation: str, h: int, w: int, dtype_name: str, tile else: x = torch.randn(1, 3, h, w, device="cuda", dtype=dtype) - estimate = estimate_vae_working_memory_flux(operation, x, vae, tile_size=tile) + estimate = estimate_vae_working_memory_flux(operation, x, vae, tile_size=tile).reservation_bytes torch.cuda.synchronize() torch.cuda.empty_cache() @@ -218,8 +218,8 @@ def main() -> None: print(f" {r['operation']} {r['h']}x{r['w']} tile={r['tile']} headroom {r['headroom']:.2f}x") print() print(" The cache under-reserves here, so it keeps other models resident and the VAE can") - print(" OOM in the last step of a generation. The constants in `vae_working_memory.py` need") - print(" a backend branch, the way `estimate_vae_working_memory_qwen_image` already has one.") + print(" OOM in the last step of a generation. The vae.flux coefficients for this convolution") + print(" backend in `invokeai/backend/working_memory/data/vae.json` need refitting.") else: worst = min(measured, key=lambda r: r["headroom"]) if measured else None if worst: diff --git a/tests/app/invocations/conftest.py b/tests/app/invocations/conftest.py index 6aed50a4190..0151a2d484f 100644 --- a/tests/app/invocations/conftest.py +++ b/tests/app/invocations/conftest.py @@ -5,6 +5,8 @@ import pytest from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL +from invokeai.backend.working_memory.estimate import WorkingMemoryEstimate, WorkingMemoryTerm + def _build_flux_shaped_vae() -> AutoencoderKL: """A real `AutoencoderKL` with the FLUX.1 VAE's *shape* and almost none of its weight. @@ -47,3 +49,19 @@ def _build_flux_shaped_vae() -> AutoencoderKL: def flux_shaped_vae() -> Callable[[], AutoencoderKL]: """Factory, not an instance: a test that needs two independent VAEs should not share one.""" return _build_flux_shaped_vae + + +def _fixed_working_memory(reservation_bytes: int) -> WorkingMemoryEstimate: + return WorkingMemoryEstimate( + estimator="test.fixed", + version=1, + features={}, + terms=(WorkingMemoryTerm("fixed", reservation_bytes, included=True),), + source="test", + ) + + +@pytest.fixture +def fixed_working_memory() -> Callable[[int], WorkingMemoryEstimate]: + """An estimate that reserves exactly the given bytes, for a node whose estimator a test patches out.""" + return _fixed_working_memory diff --git a/tests/app/invocations/test_flux2_working_memory.py b/tests/app/invocations/test_flux2_working_memory.py index 286b6b44511..6eeb2180bb1 100644 --- a/tests/app/invocations/test_flux2_working_memory.py +++ b/tests/app/invocations/test_flux2_working_memory.py @@ -7,7 +7,10 @@ a 1024x1024 generation. See https://github.com/invoke-ai/InvokeAI/issues/9500. The `MEASURED_*` tables below are peak *reserved* memory measured on CUDA in bf16 (the conservative -quantity, including allocator overhead). Every estimate must stay an upper bound on them. +quantity, including allocator overhead). Every estimate must stay an upper bound on them. The VAE's +measured points ship with its coefficients in `invokeai/backend/working_memory/data/vae.json`, where +`tests/backend/working_memory/test_shipped_coefficients.py` holds the estimate to at least each point +and, where a fitted coefficient decides it, at most 1.3x (or the point's `max_ratio`). """ from contextlib import contextmanager @@ -35,14 +38,19 @@ rocm_sdpa_uses_math_kernel, sdpa_score_matrix_bytes, ) -from invokeai.backend.util.vae_working_memory import ( - _FLUX_VAE_SCALING_CONSTANTS, - estimate_vae_working_memory_flux2, -) +from invokeai.backend.util.vae_working_memory import VAE_FLUX2_SPEC, estimate_vae_working_memory_flux2 +from invokeai.backend.working_memory.coefficients import shipped_coefficients +from invokeai.backend.working_memory.platform import Platform MB = 1024**2 GB = 1024**3 + +def _vae_k(conv_backend, operation): + """The shipped FLUX.2 VAE coefficient the estimator resolves for one convolution backend and operation.""" + return shipped_coefficients(VAE_FLUX2_SPEC, Platform(conv_backend)).values[f"k_{operation}"] + + # The estimator's default width, and the head count that follows from it. The suite's pinned # `MEASURED_DENOISE` table was taken on Klein 9B, which is that default, so those rows are unmoved # by the width scaling. @@ -272,18 +280,6 @@ def test_the_lora_margin_scales_with_the_batch(self): class TestFlux2VaeWorkingMemoryEstimate: - # (operation, pixel size, measured peak reserved MB), bf16, untiled. - MEASURED_VAE = [ - ("decode", 512, 1086), - ("decode", 768, 2414), - ("decode", 1024, 4260), - ("decode", 1328, 7146), - ("decode", 1536, 9578), - ("encode", 512, 536), - ("encode", 1024, 2122), - ("encode", 1328, 3022), - ] - def _mock_bf16_vae(self): vae = MagicMock(spec=AutoencoderKLFlux2) vae.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) # element_size == 2 @@ -293,19 +289,12 @@ def _tensor_for(self, operation, px): # decode receives 32-channel latents at pixels/8; encode receives a pixel image. return torch.zeros(1, 32, px // 8, px // 8) if operation == "decode" else torch.zeros(1, 3, px, px) - @pytest.mark.parametrize("operation, px, measured_mb", MEASURED_VAE) - def test_estimate_is_an_upper_bound_on_measured_peak(self, operation, px, measured_mb): - estimate = estimate_vae_working_memory_flux2( - operation=operation, image_tensor=self._tensor_for(operation, px), vae=self._mock_bf16_vae(), device=FUSED - ) - assert estimate >= measured_mb * MB - - @pytest.mark.parametrize("operation, expected_constant", [("decode", 2200), ("encode", 1100)]) - def test_constant_scales_pixel_area_and_element_size(self, operation, expected_constant): + @pytest.mark.parametrize("operation", ["decode", "encode"]) + def test_constant_scales_pixel_area_and_element_size(self, operation): estimate = estimate_vae_working_memory_flux2( operation=operation, image_tensor=self._tensor_for(operation, 1024), vae=self._mock_bf16_vae(), device=FUSED ) - assert estimate == 1024 * 1024 * 2 * expected_constant + assert estimate.reservation_bytes == 1024 * 1024 * 2 * _vae_k("cudnn", operation) def test_tiled_estimate_is_bounded_by_the_tile_not_the_image(self): """Reference-image encoding forces 512px tiling precisely so the peak stops following the @@ -317,79 +306,26 @@ def test_tiled_estimate_is_bounded_by_the_tile_not_the_image(self): vae=self._mock_bf16_vae(), tile_size=512, device=FUSED, - ) + ).reservation_bytes for px in (1024, 1328, 2024) ] assert len(set(estimates)) == 1 - assert estimates[0] == int(512 * 512 * 2 * 1100 * 1.25) - assert estimates[0] >= 558 * MB # measured tiled peak at 2024px + assert estimates[0] == int(512 * 512 * 2 * _vae_k("cudnn", "encode") * 1.25) # The whole point of tiling: it must shrink the reservation, not just bound the VAE. untiled = estimate_vae_working_memory_flux2( operation="encode", image_tensor=torch.zeros(1, 3, 2024, 2024), vae=self._mock_bf16_vae(), device=FUSED - ) + ).reservation_bytes assert estimates[0] < untiled / 4 class TestVaeConstantsFollowTheConvBackend: - """The pixel-area constant is not one number. MIOpen's convolution workspaces cost far more than - cuDNN's for the same decode, and the gap is not the attention term -- it is identical on the - fused path. Fitted with `scripts/calibrate_flux2_working_memory.py --only vae`, peak reserved, - each point in a fresh process: - - decode encode - cuDNN RTX 4090 2180 2185 2165 1072 1063 1061 - MIOpen RX 9070 XT 3453 3369 3368 2687 2688 2688 - - Flat across 512/768/1024px on both, so the linear model holds; only the coefficient moves. Note - the encode column: cuDNN's is half its decode, MIOpen's is four fifths, so "encoding costs half - of decoding" -- the ratio the other estimators in the module use -- is a cuDNN property, not an - architectural one. + """The pixel-area coefficient is not one number. MIOpen's convolution workspaces cost far more + than cuDNN's for the same decode, and the gap is not the attention term -- it is identical on the + fused path. Both columns, the cards they were fitted on and the points that bound them are in + `invokeai/backend/working_memory/data/vae.json`; what is pinned here is that a device selects the + column of the backend it really runs on. """ - # (backend, operation, px, measured GiB) - MEASURED = [ - ("cudnn", "decode", 512, 1.064), - ("cudnn", "decode", 768, 2.400), - ("cudnn", "decode", 1024, 4.229), - ("cudnn", "encode", 512, 0.523), - ("cudnn", "encode", 768, 1.168), - ("cudnn", "encode", 1024, 2.072), - ("miopen", "decode", 512, 1.686), - ("miopen", "decode", 768, 3.701), - ("miopen", "decode", 1024, 6.578), - # PRO W7900 (gfx1100). Costs slightly more per pixel than gfx1201 and sets the column. - ("miopen", "decode", 512, 1.721), - ("miopen", "decode", 768, 3.781), - ("miopen", "decode", 1024, 6.703), - ("miopen", "encode", 512, 1.312), - ("miopen", "encode", 768, 2.953), - ("miopen", "encode", 1024, 5.250), - ] - - @pytest.mark.parametrize("backend, operation, px, measured_gib", MEASURED) - def test_the_constant_upper_bounds_its_own_backend(self, backend, operation, px, measured_gib): - """Each column has to bound the hardware it was fitted on. Before this the cuDNN column was - used everywhere, leaving a 2.3 GiB shortfall on a 1024px MIOpen decode.""" - constant = _FLUX_VAE_SCALING_CONSTANTS[backend][operation] - estimate = px * px * 2 * constant # bf16 element size - assert estimate >= measured_gib * GB - assert estimate <= 1.3 * measured_gib * GB - - def test_the_cudnn_constant_would_not_cover_miopen(self): - """The regression this guards: a 1024px MIOpen decode needs 6.58 GiB and the cuDNN constant - reserves 4.30. That is the same class of shortfall #9500 reports, one backend over.""" - cudnn = _FLUX_VAE_SCALING_CONSTANTS["cudnn"]["decode"] - assert 1024 * 1024 * 2 * cudnn < 6.578 * GB - - def test_the_encode_ratio_is_not_architectural(self): - """cuDNN's encode is half its decode; MIOpen's is four fifths. A shared ratio cannot express - both, which is why the table carries the two operations separately.""" - ratios = { - backend: consts["encode"] / consts["decode"] for backend, consts in _FLUX_VAE_SCALING_CONSTANTS.items() - } - assert ratios["cudnn"] == pytest.approx(0.50, abs=0.02) - assert ratios["miopen"] == pytest.approx(0.76, abs=0.03) - def test_a_rocm_build_selects_the_miopen_column(self): """A HIP build reports `device.type == "cuda"`, so the torch build is the discriminator.""" vae = MagicMock(spec=AutoencoderKLFlux2) @@ -401,7 +337,7 @@ def estimate(): v.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) return estimate_vae_working_memory_flux2( operation="decode", image_tensor=latents, vae=v, device=torch.device("cuda") - ) + ).reservation_bytes # `_IS_ROCM` is pinned because it is bound at import: on a real HIP build the head-dim # guard short-circuits ahead of the probe patched here and charges a score matrix to both @@ -421,15 +357,15 @@ def estimate(): cuda = estimate() assert rocm > cuda - assert rocm / cuda == pytest.approx(3600 / 2200, rel=1e-3) + assert rocm / cuda == pytest.approx(_vae_k("miopen", "decode") / _vae_k("cudnn", "decode"), rel=1e-3) def test_a_rocm_build_composes_the_miopen_column_with_the_guard_score_matrix(self): """The state every ROCm rig is actually in, driven through the estimator. A HIP build takes the MIOpen column *and* the head-dim guard's score matrix -- the guard is unconditional for this VAE's 512-wide head -- so `max` has to pick between two terms that - are both larger than the cuDNN pair. `TestVaeTermsDoNotAdd` checks the same two terms - against the constants table, which cannot see an estimator that stops combining them: a + are both larger than the cuDNN pair. The shipped points check the same two terms against the + measurements, which cannot see an estimator that stops combining them: a plausible "the score matrix covers it anyway" refactor that drops back to the cuDNN column whenever SDPA materializes under-reserves a 1024px decode by 1.6x and passes every other test in this module. @@ -443,7 +379,7 @@ def estimate(px): image_tensor=torch.zeros(1, 32, px // 8, px // 8), vae=vae, device=torch.device("cuda"), - ) + ).reservation_bytes with ( patch("torch.version.hip", "7.1.25424"), @@ -456,9 +392,8 @@ def estimate(px): at_1024 = estimate(1024) at_1536 = estimate(1536) - miopen = _FLUX_VAE_SCALING_CONSTANTS["miopen"]["decode"] + miopen = _vae_k("miopen", "decode") assert at_1024 == 1024 * 1024 * 2 * miopen, "at 1024px the MIOpen linear term is the larger" - assert at_1024 >= 6.703 * GB, "and it still covers the measured W7900 peak it was fitted to" assert at_1536 == 36864 * 36864 * SDPA_MATH_BYTES_PER_SCORE_ELEMENT, "at 1536px the score matrix is" assert at_1536 > 1536 * 1536 * 2 * miopen @@ -470,13 +405,10 @@ class TestFlux2VaeBatchIsBudgeted: the OOM happens anyway. Measured at 1024px on CUDA/bf16, peak reserved, each point in a fresh process: 4.23GB at batch 1, - 7.96GB at batch 2, 11.89GB at batch 3. Linear, and slightly sub-linear per sample, so scaling the - single-sample estimate is an upper bound rather than a fit. + 7.96GB at batch 2, 11.89GB at batch 3 (shipped as `vae.flux2` points). Linear, and slightly + sub-linear per sample, so scaling the single-sample estimate is an upper bound rather than a fit. """ - # (batch, measured peak reserved MB) for a 1024px decode. - MEASURED_DECODE_BATCH = [(1, 4229), (2, 7955), (3, 11889)] - def _decode_estimate(self, batch, px=1024, tile_size=None, device=FUSED): vae = MagicMock(spec=AutoencoderKLFlux2) vae.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) @@ -486,13 +418,7 @@ def _decode_estimate(self, batch, px=1024, tile_size=None, device=FUSED): vae=vae, tile_size=tile_size, device=device, - ) - - @pytest.mark.parametrize("batch, measured_mb", MEASURED_DECODE_BATCH) - def test_estimate_is_an_upper_bound_on_the_measured_batch_peak(self, batch, measured_mb): - estimate = self._decode_estimate(batch) - assert estimate >= measured_mb * MB - assert estimate <= 2 * measured_mb * MB + ).reservation_bytes def test_estimate_scales_with_the_batch(self): """The regression in one assertion: before this, all three of these were equal.""" @@ -506,7 +432,7 @@ def test_a_three_dimensional_tensor_is_one_sample(self): vae.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) unbatched = estimate_vae_working_memory_flux2( operation="decode", image_tensor=torch.zeros(32, 128, 128), vae=vae, device=FUSED - ) + ).reservation_bytes assert unbatched == self._decode_estimate(1) def test_tiling_bounds_the_tile_not_the_batch(self): @@ -549,13 +475,13 @@ def _mock_vae_info(self): vae_info.model_on_device = MagicMock(return_value=cm) return vae_info - def test_decode_requests_working_memory(self): + def test_decode_requests_working_memory(self, fixed_working_memory): vae_info = self._mock_vae_info() context = MagicMock() context.models.load.return_value = vae_info context.tensors.load.return_value = torch.zeros(1, 32, 128, 128) - expected = 10 * GB + expected = fixed_working_memory(10 * GB) with patch( "invokeai.app.invocations.vae.flux2_vae_decode.estimate_vae_working_memory_flux2", return_value=expected ) as estimate: @@ -571,14 +497,14 @@ def test_decode_requests_working_memory(self): estimate.assert_called_once() assert estimate.call_args.kwargs["operation"] == "decode" - vae_info.model_on_device.assert_called_once_with(working_mem_bytes=expected) + vae_info.model_on_device.assert_called_once_with(working_memory=expected) - def test_encode_requests_working_memory(self): + def test_encode_requests_working_memory(self, fixed_working_memory): vae_info = self._mock_vae_info() context = MagicMock() context.models.load.return_value = vae_info - expected = 4 * GB + expected = fixed_working_memory(4 * GB) with ( patch( "invokeai.app.invocations.vae.flux2_vae_encode.estimate_vae_working_memory_flux2", return_value=expected @@ -598,7 +524,7 @@ def test_encode_requests_working_memory(self): estimate.assert_called_once() assert estimate.call_args.kwargs["operation"] == "encode" - vae_info.model_on_device.assert_called_once_with(working_mem_bytes=expected) + vae_info.model_on_device.assert_called_once_with(working_memory=expected) class _StopBeforeLoad(Exception): @@ -782,9 +708,9 @@ def _pinned_build(): score matrix here", but on a HIP build that same token carries two more decisions with it, and both would make this module's verdict depend on the runner: - * `_flux2_vae_scaling_constant` selects the MIOpen convolution column, 1.6-2.5x the cuDNN one - every expectation guarded by this helper is written against -- so the VAE assertions would - pass on a CUDA runner and fail on a ROCm one. + * `platform_for` selects the MIOpen convolution coefficients, 1.6-2.5x the cuDNN ones every + expectation guarded by this helper is written against -- so the VAE assertions would pass on + a CUDA runner and fail on a ROCm one. * `rocm_sdpa_uses_math_kernel` short-circuits ahead of the probe below, on a threshold a user can move with `INVOKE_ROCM_FUSED_SDPA_MAX_HEAD_DIM`. At the default it happens to agree with `_materializing_probe`; at a lower one it charges a score matrix for the denoise's 128-wide @@ -845,7 +771,7 @@ def _vae_estimate(self, operation, px, device, tile_size=None): vae=self._mock_bf16_vae(), tile_size=tile_size, device=device, - ) + ).reservation_bytes # (operation, pixel size, mid-block tokens). The VAE attends on the 8x-downsampled grid. @pytest.mark.parametrize( @@ -907,20 +833,17 @@ class TestVaeTermsDoNotAdd: decoder, last in the encoder -- so the full-resolution convolution feature maps that drive the linear term are not live while the score matrix is, and peak *reserved* is a high-water mark. - Measured, decode, peak reserved, fresh process per point: - - px cuDNN fused cuDNN forced-math linear(2200) score(17) - 1024 4.229 3.527 4.30 4.25 - 1280 6.590 5.924 6.71 10.38 - 1536 9.353 11.965 9.67 21.52 - - Forcing math measures *below* the fused path until 1536px, and even there it exceeds it by - 2.6GB against the 21.5GB the score term prices standalone -- the attention phase reuses blocks - the allocator is already holding. On gfx1100 and gfx1201, forcing math moves the measured peak - by nothing at all up to 1024px, and the totals stay flat-linear in area either way. + Measured, decode, peak reserved, fresh process per point (the cuDNN fused and forced-math points + in `invokeai/backend/working_memory/data/vae.json`): forcing math measures *below* the fused path + until 1536px, and even there it exceeds it by 2.6GB against the 21.5GB the score term prices + standalone -- the attention phase reuses blocks the allocator is already holding. On gfx1100 and + gfx1201, forcing math moves the measured peak by nothing at all up to 1024px, and the totals stay + flat-linear in area either way. Summing the terms reserved 11.1GB for a 1024px gfx1100 decode that measures 6.7GB. Taking the - max reserves 6.8GB. + max reserves 6.8GB. Where a max model is not an upper bound on its own -- the crossover, a 768px + encode with cuDNN's coefficient and a materializing kernel -- the `device_working_mem_gb` floor + is, and the shipped point there is checked against the floor. """ def _estimate(self, px, device, materializing): @@ -929,64 +852,20 @@ def _estimate(self, px, device, materializing): with _materializing() if materializing else _pinned_build(): return estimate_vae_working_memory_flux2( operation="decode", image_tensor=torch.zeros(1, 32, px // 8, px // 8), vae=vae, device=device - ) - - # (pixel size, measured GiB) for a decode on the PRO W7900. Its 512px point is what sets the - # MIOpen decode constant: implied 3525, where gfx1201 asks for only 3453. - MEASURED_W7900_DECODE = [(1024, 6.703), (768, 3.781), (512, 1.721)] - - @pytest.mark.parametrize("px, measured_gib", MEASURED_W7900_DECODE) - def test_the_max_model_bounds_the_measured_miopen_peak(self, px, measured_gib): - linear = px * px * 2 * _FLUX_VAE_SCALING_CONSTANTS["miopen"]["decode"] - score = (px // 8) ** 4 * SDPA_MATH_BYTES_PER_SCORE_ELEMENT - assert max(linear, score) >= measured_gib * GB - assert max(linear, score) <= 1.15 * measured_gib * GB - - def test_the_sum_model_would_have_over_reserved_by_two_thirds(self): - """At 1024px the sum reserves 11.1GiB for a decode that measures 6.7 -- on a 16GB card that - is the difference between the transformer staying resident and being evicted.""" - linear = 1024 * 1024 * 2 * _FLUX_VAE_SCALING_CONSTANTS["miopen"]["decode"] - score = 16384 * 16384 * SDPA_MATH_BYTES_PER_SCORE_ELEMENT - assert (linear + score) / (6.703 * GB) > 1.6 - assert max(linear, score) / (6.703 * GB) < 1.15 + ).reservation_bytes def test_the_estimate_is_the_larger_term_not_the_sum(self): """At 1024px the two are within 2% of each other on the cuDNN column, which makes this the sharpest place to tell the models apart.""" - linear = 1024 * 1024 * 2 * _FLUX_VAE_SCALING_CONSTANTS["cudnn"]["decode"] + linear = 1024 * 1024 * 2 * _vae_k("cudnn", "decode") score = 16384 * 16384 * SDPA_MATH_BYTES_PER_SCORE_ELEMENT assert self._estimate(1024, MATERIALIZING, materializing=True) == max(linear, score) assert self._estimate(1024, MATERIALIZING, materializing=True) < linear + score - # (operation, px, measured GiB) with cuDNN's linear constant and a materializing kernel -- the - # combination MPS lands in, and the only place the max model is not an upper bound on its own. - MEASURED_CUDNN_FORCED_MATH = [ - ("encode", 512, 0.523), - ("encode", 768, 1.801), - ("encode", 1024, 4.072), - ("decode", 1024, 3.527), - ("decode", 1280, 5.924), - ("decode", 1536, 11.965), - ] - - @pytest.mark.parametrize("operation, px, measured_gib", MEASURED_CUDNN_FORCED_MATH) - def test_the_working_memory_floor_covers_the_crossover(self, operation, px, measured_gib): - """A max model is weakest where the two terms are near-equal, and one measured point shows - it: a 768px encode wants 1.80GiB against a 1.35GiB max. It is not reachable as a shortfall, - because the cache floors every reservation at `device_working_mem_gb` and the whole crossover - region sits below that floor. Reproducible to three decimals across runs, so this is the - model's real shape, not noise -- which is why it is pinned rather than rounded away.""" - constant = _FLUX_VAE_SCALING_CONSTANTS["cudnn"][operation] - area = px * px if operation == "decode" else px * px - linear = area * 2 * constant - score = (px // 8) ** 4 * SDPA_MATH_BYTES_PER_SCORE_ELEMENT - floored = max(max(linear, score), 3 * GB) # what ModelCache._get_vram_available reserves - assert floored >= measured_gib * GB - def test_a_fused_build_is_unaffected(self): """The max only ever removes reservation, never adds it: with no score matrix the estimate is the linear term, exactly as before.""" - linear = 1024 * 1024 * 2 * _FLUX_VAE_SCALING_CONSTANTS["cudnn"]["decode"] + linear = 1024 * 1024 * 2 * _vae_k("cudnn", "decode") assert self._estimate(1024, FUSED, materializing=False) == linear def test_regional_prompting_adds_the_score_matrix(self): @@ -1283,7 +1162,7 @@ def estimate(): v.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) return estimate_vae_working_memory_flux2( operation="decode", image_tensor=torch.zeros(1, 32, 128, 128), vae=v, device=FUSED - ) + ).reservation_bytes with _diffusers_backend("_native_math"): forced_math = estimate() diff --git a/tests/app/invocations/test_flux_vae_decode.py b/tests/app/invocations/test_flux_vae_decode.py index 4e206cf62b2..8b93e064d3e 100644 --- a/tests/app/invocations/test_flux_vae_decode.py +++ b/tests/app/invocations/test_flux_vae_decode.py @@ -32,7 +32,7 @@ def _loaded_vae(shift_factor: float | None, scaling_factor: float = 0.3611) -> M vae_info.compute_device = torch.device("cpu") @contextmanager - def _on_device(working_mem_bytes=None): + def _on_device(working_memory=None): yield (None, vae) vae_info.model_on_device = _on_device diff --git a/tests/app/invocations/test_flux_vae_decode_tiling.py b/tests/app/invocations/test_flux_vae_decode_tiling.py index 2b6f998067e..1fca3e20687 100644 --- a/tests/app/invocations/test_flux_vae_decode_tiling.py +++ b/tests/app/invocations/test_flux_vae_decode_tiling.py @@ -107,21 +107,25 @@ def test_the_tiling_state_is_restored_afterwards(self, flux_shaped_vae): [(False, False, None), (True, False, 256), (False, True, 0)], ) def test_the_reservation_matches_the_decode_that_will_run( - self, flux_shaped_vae, tiled, force_tiled_decode, expected_tile_size + self, flux_shaped_vae, fixed_working_memory, tiled, force_tiled_decode, expected_tile_size ): """A tiled decode reserved for a single pass is the OOM the retry exists to avoid, and an untiled decode reserved for a tile evicts models it did not need to.""" path = "invokeai.app.invocations.vae.flux_vae_decode.estimate_vae_working_memory_flux" vae = flux_shaped_vae() - _, context, _ = _build_decode_mocks( + vae_info, context, _ = _build_decode_mocks( vae, torch.zeros(1, 16, 64, 64), torch.zeros(1, 3, 512, 512), force_tiled_decode=force_tiled_decode ) - with patch(path, return_value=1024) as estimate: + estimated = fixed_working_memory(1024) + with patch(path, return_value=estimated) as estimate: _build_invocation(tiled=tiled, tile_size=256 if tiled else 0).invoke(context) assert estimate.call_args.kwargs["tile_size"] == expected_tile_size + vae_info.model_on_device.assert_called_once_with(working_memory=estimated) @pytest.mark.parametrize("auto", [True, False], ids=["auto-tiled-decode-on", "auto-tiled-decode-off"]) - def test_a_decode_too_large_for_its_gpu_is_tiled_up_front_unless_switched_off(self, auto, flux_shaped_vae): + def test_a_decode_too_large_for_its_gpu_is_tiled_up_front_unless_switched_off( + self, auto, flux_shaped_vae, fixed_working_memory + ): """On Windows an allocation that does not fit is paged rather than refused, so the OOM retry never fires and the decision has to be made from the estimate alone.""" module = "invokeai.app.invocations.vae.flux_vae_decode" @@ -129,8 +133,9 @@ def test_a_decode_too_large_for_its_gpu_is_tiled_up_front_unless_switched_off(se flux_shaped_vae(), torch.zeros(1, 16, 64, 64), torch.zeros(1, 3, 512, 512) ) context.config.get.return_value.auto_tiled_decode = auto + untiled, tiled = fixed_working_memory(20 * 2**30), fixed_working_memory(2 * 2**30) with ( - patch(f"{module}.estimate_vae_working_memory_flux", side_effect=[20 * 2**30, 2 * 2**30]) as estimate, + patch(f"{module}.estimate_vae_working_memory_flux", side_effect=[untiled, tiled]) as estimate, patch(f"{module}.should_pretile_vae_decode", return_value=True) as pretile, ): _build_invocation().invoke(context) @@ -138,11 +143,12 @@ def test_a_decode_too_large_for_its_gpu_is_tiled_up_front_unless_switched_off(se if auto: pretile.assert_called_once_with(vae_info.compute_device, 20 * 2**30) assert estimate.call_args.kwargs["tile_size"] == 0 - vae_info.model_on_device.assert_called_once_with(working_mem_bytes=2 * 2**30) + vae_info.model_on_device.assert_called_once_with(working_memory=tiled) assert tiling_state[0]["use_tiling"] is True assert tiling_state[0]["tile_sample_min_size"] == DEFAULT_TILE_SAMPLE_MIN_SIZE else: pretile.assert_not_called() + vae_info.model_on_device.assert_called_once_with(working_memory=untiled) assert tiling_state[0]["use_tiling"] is False diff --git a/tests/app/invocations/test_vae_working_memory_backend.py b/tests/app/invocations/test_vae_working_memory_backend.py index 5829d061afe..9dfc4fade93 100644 --- a/tests/app/invocations/test_vae_working_memory_backend.py +++ b/tests/app/invocations/test_vae_working_memory_backend.py @@ -1,7 +1,7 @@ -"""Which convolution-backend constant the diffusers-AutoencoderKL estimators use. +"""Which convolution-backend coefficients the diffusers-AutoencoderKL estimators use, and what the SD estimator adds up. SD1/SDXL, SD3 and CogView4 run the same AutoencoderKL convolution stack as the FLUX.1 autoencoder, so they take the -same measured per-backend constants: MIOpen needs ~1.64x what cuDNN does. The change is large enough to matter -- an +same measured per-backend coefficients: MIOpen needs ~1.64x what cuDNN does. The change is large enough to matter -- an SD1.5 1024px fp32 decode reserves 9.49 GB on cuDNN and 15.36 GB on MIOpen -- and the branch is invisible on a CUDA machine, so it is pinned here rather than left to whichever card the suite runs on. """ @@ -13,10 +13,15 @@ from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL from invokeai.backend.util.vae_working_memory import ( + VAE_COGVIEW4_SPEC, + VAE_SD3_SPEC, + VAE_SD_SPEC, estimate_vae_working_memory_cogview4, estimate_vae_working_memory_sd3, estimate_vae_working_memory_sd15_sdxl, ) +from invokeai.backend.working_memory.coefficients import shipped_coefficients +from invokeai.backend.working_memory.platform import Platform def _vae(dtype: torch.dtype = torch.float16) -> MagicMock: @@ -25,39 +30,74 @@ def _vae(dtype: torch.dtype = torch.float16) -> MagicMock: return vae -def _estimate(estimator, device: torch.device, hip: str | None): +def _estimate(estimator, device: torch.device, hip: str | None) -> int: latents = torch.zeros(1, 4, 64, 64) # 512px output with ( patch("torch.version.hip", hip), # The mid-block score matrix is the other term in these estimates and has its own tests; hold it at zero so - # the assertions below are about the convolution constant alone. + # the assertions below are about the convolution coefficient alone. patch("invokeai.backend.util.vae_working_memory._vae_mid_block_score_matrix_bytes", return_value=0), ): if estimator is estimate_vae_working_memory_sd15_sdxl: - return estimator( + estimate = estimator( operation="decode", image_tensor=latents, vae=_vae(), tile_size=None, fp32=False, device=device ) - return estimator(operation="decode", image_tensor=latents, vae=_vae(), device=device) + else: + estimate = estimator(operation="decode", image_tensor=latents, vae=_vae(), device=device) + return estimate.reservation_bytes + + +def _decode_bytes(spec, conv_backend: str) -> float: + """A 512px fp16 decode at the shipped coefficient of that backend.""" + return 512 * 512 * 2 * shipped_coefficients(spec, Platform(conv_backend)).values["k_decode"] ESTIMATORS = [ - pytest.param(estimate_vae_working_memory_sd15_sdxl, id="sd15-sdxl"), - pytest.param(estimate_vae_working_memory_sd3, id="sd3"), - pytest.param(estimate_vae_working_memory_cogview4, id="cogview4"), + pytest.param(estimate_vae_working_memory_sd15_sdxl, VAE_SD_SPEC, id="sd15-sdxl"), + pytest.param(estimate_vae_working_memory_sd3, VAE_SD3_SPEC, id="sd3"), + pytest.param(estimate_vae_working_memory_cogview4, VAE_COGVIEW4_SPEC, id="cogview4"), ] -@pytest.mark.parametrize("estimator", ESTIMATORS) -def test_a_rocm_vae_reserves_the_miopen_constant(estimator): - assert _estimate(estimator, torch.device("cuda", 0), hip="7.2.0") == 512 * 512 * 2 * 3600 +@pytest.mark.parametrize("estimator, spec", ESTIMATORS) +def test_a_rocm_vae_reserves_the_miopen_coefficient(estimator, spec): + assert _estimate(estimator, torch.device("cuda", 0), hip="7.2.0") == _decode_bytes(spec, "miopen") -@pytest.mark.parametrize("estimator", ESTIMATORS) -def test_a_cuda_vae_reserves_the_cudnn_constant(estimator): - assert _estimate(estimator, torch.device("cuda", 0), hip=None) == 512 * 512 * 2 * 2200 +@pytest.mark.parametrize("estimator, spec", ESTIMATORS) +def test_a_cuda_vae_reserves_the_cudnn_coefficient(estimator, spec): + assert _estimate(estimator, torch.device("cuda", 0), hip=None) == _decode_bytes(spec, "cudnn") -@pytest.mark.parametrize("estimator", ESTIMATORS) -def test_a_cpu_only_vae_reserves_the_cudnn_constant_even_on_a_rocm_build(estimator): +@pytest.mark.parametrize("estimator, spec", ESTIMATORS) +def test_a_cpu_only_vae_reserves_the_cudnn_coefficient_even_on_a_rocm_build(estimator, spec): """A `cpu_only` VAE does not run on MIOpen, so naming its device has to change the answer.""" - assert _estimate(estimator, torch.device("cpu"), hip="7.2.0") == 512 * 512 * 2 * 2200 + assert _estimate(estimator, torch.device("cpu"), hip="7.2.0") == _decode_bytes(spec, "cudnn") + + +def _sd_fp32_decode(tile_size: int | None) -> int: + latents = torch.zeros(1, 4, 96, 128) # a 768x1024 output + with patch("invokeai.backend.util.vae_working_memory._vae_mid_block_score_matrix_bytes", return_value=0): + estimate = estimate_vae_working_memory_sd15_sdxl( + operation="decode", + image_tensor=latents, + vae=_vae(), + tile_size=tile_size, + fp32=True, + device=torch.device("cpu"), + ) + return estimate.reservation_bytes + + +FP32_WEIGHTS = 250 * 2**20 +"""What an fp32 decode adds for the upcast weights.""" + + +def test_a_tiled_fp32_sd_decode_is_one_tile_with_its_margin_plus_the_fp32_weights(): + k = shipped_coefficients(VAE_SD_SPEC, Platform("cudnn")).values["k_decode"] + assert _sd_fp32_decode(512) == int(512 * 512 * 4 * k * 1.25 + FP32_WEIGHTS) + + +def test_an_untiled_fp32_sd_decode_is_the_whole_image_plus_the_fp32_weights(): + k = shipped_coefficients(VAE_SD_SPEC, Platform("cudnn")).values["k_decode"] + assert _sd_fp32_decode(None) == int(768 * 1024 * 4 * k + FP32_WEIGHTS) diff --git a/tests/app/invocations/test_z_image_tiled_decode.py b/tests/app/invocations/test_z_image_tiled_decode.py index 2f8a46a894d..eb9d3a3a9b6 100644 --- a/tests/app/invocations/test_z_image_tiled_decode.py +++ b/tests/app/invocations/test_z_image_tiled_decode.py @@ -10,7 +10,19 @@ from invokeai.app.invocations.vae.z_image_latents_to_image import ZImageLatentsToImageInvocation from invokeai.backend.util.devices import TorchDevice from invokeai.backend.util.vae_tiling_scope import DEFAULT_TILE_SAMPLE_MIN_SIZE, MIN_TILE_SAMPLE_SIZE -from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_flux +from invokeai.backend.util.vae_working_memory import VAE_FLUX_SPEC, estimate_vae_working_memory_flux +from invokeai.backend.working_memory.coefficients import shipped_coefficients +from invokeai.backend.working_memory.platform import Platform + + +def _flux_reservation(*args, **kwargs) -> int: + """What `estimate_vae_working_memory_flux` reserves; the estimator cells are about its size.""" + return estimate_vae_working_memory_flux(*args, **kwargs).reservation_bytes + + +def _k(conv_backend: str, operation: str) -> float: + """The shipped FLUX.1 VAE coefficient the estimator resolves for one convolution backend and operation.""" + return shipped_coefficients(VAE_FLUX_SPEC, Platform(conv_backend)).values[f"k_{operation}"] def _mock_flux_vae(element_size_bytes: int = 2) -> MagicMock: @@ -36,7 +48,7 @@ def _fused_non_rocm_build(self, monkeypatch): score matrix for it. On a ROCm rig that resolves to `cuda`, where the head-dim guard charges 4.25GiB for the 1024px case below -- which the expected value here clears by 1%, so these exact-value assertions turn build-dependent the moment either constant moves. Pin the fused, non-HIP regime they are written for, which - is also the cuDNN column of `_FLUX_VAE_SCALING_CONSTANTS`; `TestClassicVaeEstimators` in + also selects the cuDNN coefficients; `TestClassicVaeEstimators` in `tests/backend/util/test_rocm_sdpa_head_dim_guard.py` owns the ROCm side of this estimator. """ import invokeai.backend.util.attention as attention @@ -46,11 +58,18 @@ def _fused_non_rocm_build(self, monkeypatch): monkeypatch.setattr(vwm.TorchDevice, "choose_torch_device", classmethod(lambda cls: torch.device("cpu"))) @pytest.mark.parametrize( - ("hip", "operation", "constant"), - [(None, "decode", 2200), (None, "encode", 1100), ("7.14.0", "decode", 3600), ("7.14.0", "encode", 2750)], + ("hip", "operation", "conv_backend"), + [ + (None, "decode", "cudnn"), + (None, "encode", "cudnn"), + ("7.14.0", "decode", "miopen"), + ("7.14.0", "encode", "miopen"), + ], ids=["cudnn-decode", "cudnn-encode", "miopen-decode", "miopen-encode"], ) - def test_the_convolution_backend_of_the_vaes_device_picks_the_constant(self, monkeypatch, hip, operation, constant): + def test_the_convolution_backend_of_the_vaes_device_picks_the_constant( + self, monkeypatch, hip, operation, conv_backend + ): """MIOpen's workspaces are larger than cuDNN's: measured 3451 decode / 2688 encode on an RX 9060 XT, the same as the FLUX.2 VAE. A HIP build reports its GPU as "cuda", so the torch build decides, for the VAE's device.""" import invokeai.backend.util.vae_working_memory as vwm @@ -59,22 +78,22 @@ def test_the_convolution_backend_of_the_vaes_device_picks_the_constant(self, mon monkeypatch.setattr(vwm, "_vae_mid_block_score_matrix_bytes", lambda *args, **kwargs: 0) tensor = torch.zeros(1, 16, 64, 64) if operation == "decode" else torch.zeros(1, 3, 512, 512) - estimate = estimate_vae_working_memory_flux( + estimate = _flux_reservation( operation=operation, image_tensor=tensor, vae=_mock_flux_vae(), device=torch.device("cuda", 0) ) - on_cpu = estimate_vae_working_memory_flux( + on_cpu = _flux_reservation( operation=operation, image_tensor=tensor, vae=_mock_flux_vae(), device=torch.device("cpu") ) - assert estimate == 512 * 512 * 2 * constant - assert on_cpu == 512 * 512 * 2 * (2200 if operation == "decode" else 1100), "a cpu_only VAE runs on cuDNN terms" + assert estimate == 512 * 512 * 2 * _k(conv_backend, operation) + assert on_cpu == 512 * 512 * 2 * _k("cudnn", operation), "a cpu_only VAE runs on cuDNN terms" def test_the_default_reproduces_the_untiled_estimate(self): """Regression guard for the six call sites that pass no tile_size at all.""" latents = torch.zeros(1, 16, 128, 128) - # 1024x1024 output px * 2 bytes * 2200 - expected = 1024 * 1024 * 2 * 2200 - actual = estimate_vae_working_memory_flux(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) + # 1024x1024 output px * 2 bytes * the cuDNN decode coefficient + expected = 1024 * 1024 * 2 * _k("cudnn", "decode") + actual = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) assert actual == expected def test_the_tiled_estimate_is_a_tile_bounded_decode_plus_the_assembled_image(self): @@ -86,13 +105,13 @@ def test_the_tiled_estimate_is_a_tile_bounded_decode_plus_the_assembled_image(se under-reservation that no other assertion here can see. """ # 512px tiles, fp16, over a 1536x1536 output. - decode_term = 512 * 512 * 2 * 2200 * 1.25 + decode_term = 512 * 512 * 2 * _k("cudnn", "decode") * 1.25 # 4 RGB buffers at the VAE's element size, plus the 8-bit one the node hands to PIL. Four # is measured, not assumed: the assembled image grows the device peak by ~1.6 copies on a # diffusers VAE and ~0.1 on the host-merging BFL one, and the node adds its in-window # clamp/scale copies on top. See the constant's comment for the measurements. image_term = 1536 * 1536 * 3 * (4 * 2 + 1) - estimate = estimate_vae_working_memory_flux( + estimate = _flux_reservation( operation="decode", image_tensor=torch.zeros(1, 16, 192, 192), vae=_mock_flux_vae(), tile_size=512 ) assert estimate == int(decode_term + image_term) @@ -101,10 +120,10 @@ def test_both_tiled_terms_scale_with_the_vaes_element_size(self): """An fp32 VAE costs twice the bytes per pixel, in the tile and in the assembled image alike. Only the 8-bit buffer is fixed, which is why this is a little under 2x.""" latents = torch.zeros(1, 16, 192, 192) - fp16 = estimate_vae_working_memory_flux( + fp16 = _flux_reservation( operation="decode", image_tensor=latents, vae=_mock_flux_vae(element_size_bytes=2), tile_size=512 ) - fp32 = estimate_vae_working_memory_flux( + fp32 = _flux_reservation( operation="decode", image_tensor=latents, vae=_mock_flux_vae(element_size_bytes=4), tile_size=512 ) assert 1.95 < fp32 / fp16 < 2.0 @@ -120,10 +139,8 @@ def test_a_tiled_estimate_stays_a_fraction_of_the_untiled_one(self, latent_hw): the clamped tile plus the same image term, which can land above the single-pass estimate.) """ latents = torch.zeros(1, 16, *latent_hw) - untiled = estimate_vae_working_memory_flux(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) - tiled = estimate_vae_working_memory_flux( - operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=512 - ) + untiled = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) + tiled = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=512) assert tiled < untiled / 2 def test_a_tile_larger_than_the_image_is_not_estimated_as_tiled(self): @@ -134,10 +151,8 @@ def test_a_tile_larger_than_the_image_is_not_estimated_as_tiled(self): decode needing ~4.6GB, evicting the transformer from the cache to make room for nothing. """ latents = torch.zeros(1, 16, 128, 128) # 1024px - untiled = estimate_vae_working_memory_flux(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) - oversized = estimate_vae_working_memory_flux( - operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=4096 - ) + untiled = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) + oversized = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=4096) assert oversized == untiled def test_a_tile_wider_than_one_axis_is_priced_at_the_clamped_tile(self): @@ -148,10 +163,8 @@ def test_a_tile_wider_than_one_axis_is_priced_at_the_clamped_tile(self): single-pass decode that tiling is there to avoid -- ~92GB against ~37GB. """ latents = torch.zeros(1, 16, 128, 1024) # 1024 x 8192 px: only the long axis exceeds the tile - tiled = estimate_vae_working_memory_flux( - operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=4096 - ) - untiled = estimate_vae_working_memory_flux(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) + tiled = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=4096) + untiled = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) assert tiled < untiled def test_a_tiled_estimate_still_covers_the_assembled_image(self): @@ -163,10 +176,10 @@ def test_a_tiled_estimate_still_covers_the_assembled_image(self): second resolution the image term is checked at, so a term that did not scale with area would fail here even if it matched at 1536px. """ - estimate = estimate_vae_working_memory_flux( + estimate = _flux_reservation( operation="decode", image_tensor=torch.zeros(1, 16, 1024, 1024), vae=_mock_flux_vae(), tile_size=512 ) - tile_term = int(512 * 512 * 2 * 2200 * 1.25) + tile_term = int(512 * 512 * 2 * _k("cudnn", "decode") * 1.25) one_full_resolution_buffer = 8192 * 8192 * 3 * 2 # The assembled output plus at least one of the post-processing copies. assert estimate >= tile_term + 2 * one_full_resolution_buffer @@ -178,8 +191,8 @@ def test_the_sentinel_does_not_read_the_size_off_the_vae(self): latents = torch.zeros(1, 16, 192, 192) vae = _mock_flux_vae() vae.tile_sample_min_size = 384 # as if a previous run had set it - estimate = estimate_vae_working_memory_flux(operation="decode", image_tensor=latents, vae=vae, tile_size=0) - assert estimate == estimate_vae_working_memory_flux( + estimate = _flux_reservation(operation="decode", image_tensor=latents, vae=vae, tile_size=0) + assert estimate == _flux_reservation( operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=DEFAULT_TILE_SAMPLE_MIN_SIZE ) @@ -190,17 +203,15 @@ def test_the_sentinel_works_on_a_vae_without_that_attribute_at_all(self): vae = MagicMock(spec=AutoencoderKL) vae.parameters.side_effect = lambda: iter([torch.zeros(1, dtype=torch.float16)]) del vae.tile_sample_min_size - estimate = estimate_vae_working_memory_flux(operation="decode", image_tensor=latents, vae=vae, tile_size=0) - assert estimate == estimate_vae_working_memory_flux( + estimate = _flux_reservation(operation="decode", image_tensor=latents, vae=vae, tile_size=0) + assert estimate == _flux_reservation( operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=DEFAULT_TILE_SAMPLE_MIN_SIZE ) def test_a_tile_below_the_cost_floor_is_estimated_at_the_floor(self): latents = torch.zeros(1, 16, 192, 192) - estimate = estimate_vae_working_memory_flux( - operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=8 - ) - assert estimate == estimate_vae_working_memory_flux( + estimate = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=8) + assert estimate == _flux_reservation( operation="decode", image_tensor=latents, vae=_mock_flux_vae(), tile_size=MIN_TILE_SAMPLE_SIZE ) @@ -221,23 +232,21 @@ def _fused_non_rocm_build(self, monkeypatch): monkeypatch.setattr("invokeai.backend.util.vae_working_memory.sdpa_score_matrix_bytes", lambda *a, **k: 0) monkeypatch.setattr(torch.version, "hip", None, raising=False) - def test_the_untiled_estimate_is_unchanged(self): - """Four call sites pass no tile and depend on this number staying what it was.""" + def test_an_untiled_encode_is_the_whole_image_at_the_encode_coefficient(self): + """What the call sites that pass no tile reserve: the full image, priced at the cuDNN encode coefficient.""" image = torch.zeros(1, 3, 1024, 1024) - assert estimate_vae_working_memory_flux("encode", image, _mock_flux_vae()) == 1024 * 1024 * 2 * 1100 + assert _flux_reservation("encode", image, _mock_flux_vae()) == 1024 * 1024 * 2 * _k("cudnn", "encode") def test_a_tiled_estimate_is_flat_in_the_image_size_where_the_untiled_one_is_not(self): vae = _mock_flux_vae() estimates = [ - estimate_vae_working_memory_flux("encode", torch.zeros(1, 3, edge, edge), vae, tile_size=512) + _flux_reservation("encode", torch.zeros(1, 3, edge, edge), vae, tile_size=512) for edge in (1024, 2048, 3072) ] # Not exactly equal: the input image and the assembled latent are not bounded by tiling. # But they are a rounding error beside the tile term, which is what "flat" has to mean. assert max(estimates) < min(estimates) * 1.2 - untiled = [ - estimate_vae_working_memory_flux("encode", torch.zeros(1, 3, edge, edge), vae) for edge in (1024, 3072) - ] + untiled = [_flux_reservation("encode", torch.zeros(1, 3, edge, edge), vae) for edge in (1024, 3072)] assert untiled[1] == untiled[0] * 9 def test_the_tiled_estimate_is_one_tile_plus_the_input_and_every_live_copy_of_the_moments(self): @@ -247,12 +256,10 @@ def test_the_tiled_estimate_is_one_tile_plus_the_input_and_every_live_copy_of_th vae = _mock_flux_vae() edge, tile = 2048, 512 live_moment_copies = 1.0 / (1.0 - 0.25) ** 2 + 2.0 - expected = tile * tile * 2 * 1100 * 1.25 + expected = tile * tile * 2 * _k("cudnn", "encode") * 1.25 expected += edge * edge * 3 * 2 expected += (edge // 8) * (edge // 8) * 2 * 16 * 2 * live_moment_copies - assert estimate_vae_working_memory_flux("encode", torch.zeros(1, 3, edge, edge), vae, tile_size=tile) == int( - expected - ) + assert _flux_reservation("encode", torch.zeros(1, 3, edge, edge), vae, tile_size=tile) == int(expected) def test_it_does_not_inherit_the_decode_residual(self): """The decode adds four element-size copies of the assembled image plus a byte buffer, for @@ -262,8 +269,8 @@ def test_it_does_not_inherit_the_decode_residual(self): image = torch.zeros(1, 3, 2048, 2048) latents = torch.zeros(1, 16, 2048 // 8, 2048 // 8) - encode = estimate_vae_working_memory_flux("encode", image, vae, tile_size=512) - decode = estimate_vae_working_memory_flux("decode", latents, vae, tile_size=512) + encode = _flux_reservation("encode", image, vae, tile_size=512) + decode = _flux_reservation("decode", latents, vae, tile_size=512) # Same tile, same output area, so the whole difference is the residual term. assert decode - encode > 2048 * 2048 * 3 * 2, "the encode is paying for the decode's image copies" @@ -273,9 +280,7 @@ def test_a_tile_larger_than_the_image_is_priced_as_the_untiled_encode(self): oversized tile would otherwise reserve for a pass that never runs.""" vae = _mock_flux_vae() image = torch.zeros(1, 3, 512, 512) - assert estimate_vae_working_memory_flux( - "encode", image, vae, tile_size=1024 - ) == estimate_vae_working_memory_flux("encode", image, vae) + assert _flux_reservation("encode", image, vae, tile_size=1024) == _flux_reservation("encode", image, vae) def _build_decode_mocks(vae, latents: torch.Tensor, decoded: torch.Tensor, force_tiled_decode: bool = False): @@ -364,15 +369,18 @@ def test_the_tiling_state_is_restored_afterwards(self, flux_shaped_vae): assert (vae.use_tiling, vae.tile_sample_min_size, vae.tile_latent_min_size) == before @pytest.mark.parametrize("auto", [True, False], ids=["auto-tiled-decode-on", "auto-tiled-decode-off"]) - def test_a_decode_too_large_for_its_gpu_is_tiled_up_front_unless_switched_off(self, auto, flux_shaped_vae): + def test_a_decode_too_large_for_its_gpu_is_tiled_up_front_unless_switched_off( + self, auto, flux_shaped_vae, fixed_working_memory + ): """Where the driver pages instead of raising an OOM, the retry below never fires: the estimate decides.""" module = "invokeai.app.invocations.vae.z_image_latents_to_image" vae_info, context, state = _build_decode_mocks( flux_shaped_vae(), torch.zeros(1, 16, 64, 64), torch.zeros(1, 3, 512, 512) ) context.config.get.return_value.auto_tiled_decode = auto + untiled, tiled = fixed_working_memory(20 * 2**30), fixed_working_memory(2 * 2**30) with ( - patch(f"{module}.estimate_vae_working_memory_flux", side_effect=[20 * 2**30, 2 * 2**30]) as estimate, + patch(f"{module}.estimate_vae_working_memory_flux", side_effect=[untiled, tiled]) as estimate, patch(f"{module}.should_pretile_vae_decode", return_value=True) as pretile, ): _build_invocation().invoke(context) @@ -380,18 +388,21 @@ def test_a_decode_too_large_for_its_gpu_is_tiled_up_front_unless_switched_off(se if auto: pretile.assert_called_once_with(vae_info.compute_device, 20 * 2**30) assert estimate.call_args.kwargs["tile_size"] == 0 - vae_info.model_on_device.assert_called_once_with(working_mem_bytes=2 * 2**30) + vae_info.model_on_device.assert_called_once_with(working_memory=tiled) assert state[0]["use_tiling"] is True else: pretile.assert_not_called() assert estimate.call_count == 1 + vae_info.model_on_device.assert_called_once_with(working_memory=untiled) assert state[0]["use_tiling"] is False - def test_requested_tiling_skips_the_untiled_estimate(self, flux_shaped_vae): + def test_requested_tiling_skips_the_untiled_estimate(self, flux_shaped_vae, fixed_working_memory): module = "invokeai.app.invocations.vae.z_image_latents_to_image" _, context, _ = _build_decode_mocks(flux_shaped_vae(), torch.zeros(1, 16, 64, 64), torch.zeros(1, 3, 512, 512)) with ( - patch(f"{module}.estimate_vae_working_memory_flux", return_value=2 * 2**30) as estimate, + patch( + f"{module}.estimate_vae_working_memory_flux", return_value=fixed_working_memory(2 * 2**30) + ) as estimate, patch(f"{module}.should_pretile_vae_decode") as pretile, ): _build_invocation(tiled=True).invoke(context) @@ -399,11 +410,13 @@ def test_requested_tiling_skips_the_untiled_estimate(self, flux_shaped_vae): pretile.assert_not_called() @pytest.mark.parametrize("tiled,expected_tile_size", [(False, None), (True, 0)]) - def test_the_estimate_is_tile_bounded_only_when_tiling(self, flux_shaped_vae, tiled, expected_tile_size): + def test_the_estimate_is_tile_bounded_only_when_tiling( + self, flux_shaped_vae, fixed_working_memory, tiled, expected_tile_size + ): path = "invokeai.app.invocations.vae.z_image_latents_to_image.estimate_vae_working_memory_flux" vae = flux_shaped_vae() _, context, _ = _build_decode_mocks(vae, torch.zeros(1, 16, 64, 64), torch.zeros(1, 3, 512, 512)) - with patch(path, return_value=1024) as estimate: + with patch(path, return_value=fixed_working_memory(1024)) as estimate: _build_invocation(tiled=tiled).invoke(context) assert estimate.call_args.kwargs["tile_size"] == expected_tile_size diff --git a/tests/app/invocations/test_z_image_working_memory.py b/tests/app/invocations/test_z_image_working_memory.py index c04e44e2b0d..655714a8c9a 100644 --- a/tests/app/invocations/test_z_image_working_memory.py +++ b/tests/app/invocations/test_z_image_working_memory.py @@ -42,7 +42,7 @@ def _vae_info(vae) -> MagicMock: class TestTheEstimateReachesTheCache: - def test_the_decode_node_reserves_what_it_estimated(self, flux_shaped_vae): + def test_the_decode_node_reserves_what_it_estimated(self, flux_shaped_vae, fixed_working_memory): vae = flux_shaped_vae() vae_info = _vae_info(vae) @@ -53,7 +53,7 @@ def test_the_decode_node_reserves_what_it_estimated(self, flux_shaped_vae): # `ImageOutput.build` validates these, so they cannot be bare MagicMocks. context.images.save.return_value = MagicMock(image_name="test.png", width=512, height=512) - expected_memory = 500 * 1024 * 1024 + expected_memory = fixed_working_memory(500 * 1024 * 1024) path = "invokeai.app.invocations.vae.z_image_latents_to_image.estimate_vae_working_memory_flux" with patch(path, return_value=expected_memory) as estimate: ZImageLatentsToImageInvocation.model_construct( @@ -64,18 +64,18 @@ def test_the_decode_node_reserves_what_it_estimated(self, flux_shaped_vae): ).invoke(context) estimate.assert_called_once() - vae_info.model_on_device.assert_called_once_with(working_mem_bytes=expected_memory) + vae_info.model_on_device.assert_called_once_with(working_memory=expected_memory) - def test_the_encode_node_reserves_what_it_estimated(self, flux_shaped_vae): + def test_the_encode_node_reserves_what_it_estimated(self, flux_shaped_vae, fixed_working_memory): vae = flux_shaped_vae() vae_info = _vae_info(vae) - expected_memory = 250 * 1024 * 1024 + expected_memory = fixed_working_memory(250 * 1024 * 1024) path = "invokeai.app.invocations.vae.z_image_image_to_latents.estimate_vae_working_memory_flux" with patch(path, return_value=expected_memory) as estimate: latents = ZImageImageToLatentsInvocation.vae_encode(vae_info, torch.zeros(1, 3, 512, 512)) estimate.assert_called_once() - vae_info.model_on_device.assert_called_once_with(working_mem_bytes=expected_memory) + vae_info.model_on_device.assert_called_once_with(working_memory=expected_memory) # The encode really ran: a 512px image through an 8x, 16-channel VAE is a 64px latent. assert latents.shape == (1, 16, 64, 64) diff --git a/tests/backend/model_manager/load/test_loaded_model.py b/tests/backend/model_manager/load/test_loaded_model.py index 04368325bde..9793e2b89e3 100644 --- a/tests/backend/model_manager/load/test_loaded_model.py +++ b/tests/backend/model_manager/load/test_loaded_model.py @@ -1,3 +1,5 @@ +import logging + import pytest import torch @@ -12,6 +14,8 @@ from invokeai.backend.model_manager.load.model_cache.torch_module_autocast.torch_module_autocast import ( apply_custom_layers_to_model, ) +from invokeai.backend.util.logging import InvokeAILogger +from invokeai.backend.working_memory.estimate import WorkingMemoryEstimate, WorkingMemoryTerm class ModelWithRequiredScale(torch.nn.Module): @@ -25,10 +29,12 @@ class FakeCache: def __init__(self): self.lock_calls = 0 self.unlock_calls = 0 + self.reserved: list[int | None] = [] def lock(self, cache_record: CacheRecord, working_mem_bytes: int | None, max_move_bytes: int | None = None) -> bool: - del cache_record, working_mem_bytes, max_move_bytes + del cache_record, max_move_bytes self.lock_calls += 1 + self.reserved.append(working_mem_bytes) return True def unlock(self, cache_record: CacheRecord) -> None: @@ -63,6 +69,62 @@ def test_model_on_device_leaves_full_load_models_unchanged(): assert all(param.device.type == "cpu" for param in model.parameters()) +def _full_load_model(cache: FakeCache) -> LoadedModelWithoutConfig: + cached_model = CachedModelOnlyFullLoad( + model=torch.nn.Linear(4, 4), compute_device=torch.device("meta"), total_bytes=1, keep_ram_copy=False + ) + return LoadedModelWithoutConfig(cache_record=CacheRecord(key="vae-key", cached_model=cached_model), cache=cache) + + +_ESTIMATE = WorkingMemoryEstimate( + estimator="vae.flux2", + version=1, + features={"operation": "decode"}, + terms=(WorkingMemoryTerm.fitted("convolutions", "k_decode", 3 * 2**29 + 0.25, {"k_decode": 2.0}),), + source="shipped:conv_backend=cudnn", +) + + +def test_a_working_memory_estimate_reserves_its_bytes_and_logs_where_they_came_from(caplog): + cache = FakeCache() + logger = "invokeai.backend.model_manager.load.load_base" + # Created before the level is lowered: creating it sets the configured level, which would undo that. + InvokeAILogger.get_logger(logger) + + with ( + caplog.at_level(logging.DEBUG, logger=logger), + _full_load_model(cache).model_on_device(working_memory=_ESTIMATE), + ): + pass + + assert cache.reserved == [3 * 2**30] + assert cache.unlock_calls == 1 + (record,) = [r for r in caplog.records if r.name == logger] + assert "vae-key" in record.message + assert "vae.flux2 reserves 3072 MB (shipped:conv_backend=cudnn)" in record.message + + +def test_an_estimate_and_a_byte_count_together_are_refused_before_locking(): + cache = FakeCache() + + with pytest.raises(ValueError, match="not both"): + with _full_load_model(cache).model_on_device(1024, working_memory=_ESTIMATE): + pass + + assert cache.lock_calls == 0 + + +def test_an_estimate_passed_as_a_byte_count_says_where_it_goes(): + """The positional slot is `working_mem_bytes`; an estimate there would fail deep inside the cache instead.""" + cache = FakeCache() + + with pytest.raises(TypeError, match="working_memory="): + with _full_load_model(cache).model_on_device(_ESTIMATE): + pass + + assert cache.lock_calls == 0 + + def test_supports_partial_loading_reflects_cached_model_type(): partial_model = CachedModelWithPartialLoad( model=torch.nn.Linear(4, 4), compute_device=torch.device("meta"), keep_ram_copy=False diff --git a/tests/backend/util/test_rocm_sdpa_head_dim_guard.py b/tests/backend/util/test_rocm_sdpa_head_dim_guard.py index e82d1ccead1..6710b3afbf3 100644 --- a/tests/backend/util/test_rocm_sdpa_head_dim_guard.py +++ b/tests/backend/util/test_rocm_sdpa_head_dim_guard.py @@ -488,7 +488,7 @@ class TestClassicVaeEstimators: def _rocm_cuda(self, monkeypatch): """The guard's ROCm switch and the convolution backend are separate questions, and this class is about the first: the assertions below carry the cuDNN linear term, so pin `torch.version.hip` (which selects MIOpen's - larger constants) away from it while the attention side stays on ROCm.""" + larger coefficients) away from it while the attention side stays on ROCm.""" import invokeai.backend.util.vae_working_memory as vwm monkeypatch.setattr(attention, "_IS_ROCM", True) @@ -508,16 +508,25 @@ def _score_bytes(px: int) -> int: tokens = (px // 8) ** 2 return tokens * tokens * attention.SDPA_MATH_BYTES_PER_SCORE_ELEMENT + @staticmethod + def _cudnn_decode_bytes(px: int) -> float: + """A bf16 decode's convolution term at the shipped cuDNN coefficient.""" + from invokeai.backend.util.vae_working_memory import VAE_SD_SPEC + from invokeai.backend.working_memory.coefficients import shipped_coefficients + from invokeai.backend.working_memory.platform import Platform + + return px * px * 2 * shipped_coefficients(VAE_SD_SPEC, Platform("cudnn")).values["k_decode"] + def test_sdxl_untiled_decode_reserves_the_score_matrix_when_it_dominates(self, monkeypatch): from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_sd15_sdxl latents = torch.zeros(1, 4, 2048 // 8, 2048 // 8) - linear = 2048 * 2048 * 2 * 2200 + linear = self._cudnn_decode_bytes(2048) with_guard = estimate_vae_working_memory_sd15_sdxl( "decode", latents, self._vae(AutoencoderKL), None, fp32=False - ) + ).reservation_bytes assert with_guard == self._score_bytes(2048) > linear monkeypatch.setattr(attention, "_IS_ROCM", False) @@ -527,7 +536,9 @@ def test_sdxl_untiled_decode_reserves_the_score_matrix_when_it_dominates(self, m real_empty = torch.empty monkeypatch.setattr(torch, "empty", lambda *a, device=None, **kw: real_empty(*a, **kw)) assert ( - estimate_vae_working_memory_sd15_sdxl("decode", latents, self._vae(AutoencoderKL), None, fp32=False) + estimate_vae_working_memory_sd15_sdxl( + "decode", latents, self._vae(AutoencoderKL), None, fp32=False + ).reservation_bytes == linear ) @@ -538,7 +549,8 @@ def test_the_other_autoencoder_kl_estimators_price_it_too(self, estimator): import invokeai.backend.util.vae_working_memory as vwm latents = torch.zeros(1, 16, 2048 // 8, 2048 // 8) - assert getattr(vwm, estimator)("decode", latents, self._vae(AutoencoderKL)) == self._score_bytes(2048) + estimate = getattr(vwm, estimator)("decode", latents, self._vae(AutoencoderKL)) + assert estimate.reservation_bytes == self._score_bytes(2048) def test_a_tiny_autoencoder_has_no_attention_to_price(self): from diffusers.models.autoencoders.autoencoder_tiny import AutoencoderTiny @@ -548,7 +560,7 @@ def test_a_tiny_autoencoder_has_no_attention_to_price(self): latents = torch.zeros(1, 4, 2048 // 8, 2048 // 8) assert estimate_vae_working_memory_sd15_sdxl( "decode", latents, self._vae(AutoencoderTiny), None, fp32=False - ) == (2048 * 2048 * 2 * 2200) + ).reservation_bytes == self._cudnn_decode_bytes(2048) def test_flux1_decode_prices_the_score_matrix_by_tile_when_tiled(self): from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL @@ -556,9 +568,11 @@ def test_flux1_decode_prices_the_score_matrix_by_tile_when_tiled(self): from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_flux latents = torch.zeros(1, 16, 2048 // 8, 2048 // 8) - untiled = estimate_vae_working_memory_flux("decode", latents, self._vae(AutoencoderKL)) + untiled = estimate_vae_working_memory_flux("decode", latents, self._vae(AutoencoderKL)).reservation_bytes assert untiled == self._score_bytes(2048) - tiled = estimate_vae_working_memory_flux("decode", latents, self._vae(AutoencoderKL), tile_size=1024) + tiled = estimate_vae_working_memory_flux( + "decode", latents, self._vae(AutoencoderKL), tile_size=1024 + ).reservation_bytes assert tiled < self._score_bytes(2048) assert tiled >= self._score_bytes(1024), "a 1024px tile still materializes its own 16384-token score matrix" diff --git a/tests/backend/working_memory/__init__.py b/tests/backend/working_memory/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/backend/working_memory/test_shipped_coefficients.py b/tests/backend/working_memory/test_shipped_coefficients.py new file mode 100644 index 00000000000..d369d03df90 --- /dev/null +++ b/tests/backend/working_memory/test_shipped_coefficients.py @@ -0,0 +1,312 @@ +"""The shipped working-memory data, checked against the estimators that read it. + +Every coefficient entry resolves for the spec it names on a measured platform value, and every value an entry carries +is decided by at least one measured point. A point is peak *reserved* bytes, one fresh process per point (see its +`provenance`), and bounds its estimate from both sides. The model cache consumes an estimate as `free >= estimate`, so +the estimate must be at least the point. Where a fitted term decides it, it must also be at most 1.3 times the point, +or the point's tighter `max_ratio`: a reservation the operation never uses is model weights evicted for nothing. Where +an analytic term (a materialized score matrix) decides it, that term bounds rather than describes, so it gets no upper +bound, and the cache's `device_working_mem_gb` floor counts toward the lower one. +""" + +import itertools +import re +from dataclasses import asdict +from pathlib import Path +from typing import get_args, get_type_hints + +import pytest +import torch +from pydantic import ValidationError + +import invokeai +from invokeai.app.services.config.config_default import InvokeAIAppConfig +from invokeai.backend.working_memory.coefficients import ( + CoefficientEntry, + MeasuredPoint, + ResolvedCoefficients, + ValuesFrom, + WorkingMemoryData, + load_data, + resolve_coefficients, + shipped_coefficients, + shipped_data, +) +from invokeai.backend.working_memory.estimate import WorkingMemoryEstimate +from invokeai.backend.working_memory.platform import UNMEASURED_PRICED_AS, Platform, platform_for +from invokeai.backend.working_memory.spec import ESTIMATOR_MODULES, EstimatorSpec, import_estimator_modules + +MAX_OVER_RESERVATION = 1.3 +"""How far above a measured point a reservation may sit where a fitted coefficient decides it, unless the point +carries a tighter `max_ratio`.""" + +WORKING_MEMORY_FLOOR = int(InvokeAIAppConfig.model_fields["device_working_mem_gb"].default * 2**30) +"""What `ModelCache._working_mem_reserve` floors every reservation at. The field default, not a config instance an +environment variable could move.""" + +# Imported deliberately rather than relying on whichever modules ran first: a spec missing from the registry is a spec +# whose data nobody checks. +SPECS = import_estimator_modules() + + +def _platforms(*, measured: bool) -> list[Platform]: + """Every platform built from measured values only, or every platform with at least one unmeasured value.""" + every = {key: get_args(hint) for key, hint in get_type_hints(Platform).items()} + measured_values = { + key: [v for v in values if v not in UNMEASURED_PRICED_AS.get(key, {})] for key, values in every.items() + } + chosen = measured_values if measured else every + platforms = [Platform(**dict(zip(chosen, values, strict=True))) for values in itertools.product(*chosen.values())] + if measured: + return platforms + return [p for p in platforms if any(value in UNMEASURED_PRICED_AS.get(key, {}) for key, value in asdict(p).items())] + + +def _platform_key(platform: dict[str, str]) -> tuple[tuple[str, str], ...]: + return tuple(sorted(platform.items())) + + +def _point_id(point: MeasuredPoint) -> str: + features = point.features + shape = f"{features.get('height')}x{features.get('width')}" + tile = f"-tile{features['tile']}" if features.get("tile") is not None else "" + batch = f"-b{features['batch_size']}" if features.get("batch_size", 1) != 1 else "" + scores = "-scores" if features.get("score_matrix_bytes") else "" + platform = ",".join(point.platform.values()) + card = point.device.split(" (")[0].replace(" ", "") + return f"{point.estimator}-{platform}-{features.get('operation')}-{shape}{tile}{batch}{scores}-{card}" + + +def _evaluate(point: MeasuredPoint) -> WorkingMemoryEstimate: + spec = SPECS[point.estimator] + assert point.version == spec.version + return spec.evaluate(point.features, shipped_coefficients(spec, Platform(**point.platform))) + + +def _fitted_decides(estimate: WorkingMemoryEstimate) -> bool: + return any(term.included and term.coefficient is not None for term in estimate.terms) + + +POINTS = [pytest.param(point, id=_point_id(point)) for data in shipped_data().values() for point in data.points] + + +def test_every_data_file_parses_and_declares_coefficients() -> None: + for name, data in shipped_data().items(): + assert data.coefficients, f"{name} declares no coefficients" + + +def test_every_coefficient_entry_resolves_for_its_spec_on_a_measured_platform_value() -> None: + """Resolving checks what an entry provides against what its spec reads, the borrowed values included. An entry for + an unmeasured value would never be read: the resolver prices that value as `UNMEASURED_PRICED_AS` says.""" + seen: set[tuple[str, int, tuple[tuple[str, str], ...]]] = set() + for name, data in shipped_data().items(): + for entry in data.coefficients: + where = f"{name}: {entry.estimator} on {entry.platform}" + spec = SPECS.get(entry.estimator) + assert spec is not None, f"{where}: no estimator registers this id" + assert entry.version == spec.version, f"{where}: version {entry.version}, the estimator is {spec.version}" + assert set(entry.platform) == set(spec.platform_keys), f"{where}: the spec keys {spec.platform_keys}" + for key, value in entry.platform.items(): + assert value not in UNMEASURED_PRICED_AS.get(key, {}), f"{where}: {key}={value} is not measured" + key = (entry.estimator, entry.version, _platform_key(entry.platform)) + assert key not in seen, f"{where}: declared twice" + seen.add(key) + shipped_coefficients(spec, Platform(**entry.platform)) + + +@pytest.mark.parametrize("spec", sorted(SPECS.values(), key=lambda spec: spec.id), ids=lambda spec: spec.id) +def test_every_estimator_has_coefficients_on_every_measured_platform_value(spec: EstimatorSpec) -> None: + for platform in _platforms(measured=True): + assert set(shipped_coefficients(spec, platform).values) == spec.coefficients + + +@pytest.mark.parametrize("spec", sorted(SPECS.values(), key=lambda spec: spec.id), ids=lambda spec: spec.id) +def test_an_unmeasured_platform_value_is_priced_as_the_measured_one_and_says_so(spec: EstimatorSpec) -> None: + for platform in _platforms(measured=False): + priced_as = Platform( + **{key: UNMEASURED_PRICED_AS.get(key, {}).get(value, value) for key, value in asdict(platform).items()} + ) + resolved = shipped_coefficients(spec, platform) + assert resolved.values == shipped_coefficients(spec, priced_as).values + assert resolved.source.startswith( + f"shipped:conv_backend={platform.conv_backend} ({priced_as.conv_backend} values" + ) + + +def test_every_carried_value_is_decided_by_a_measured_point() -> None: + """A value no point decides is unguarded: any number would pass. An estimator without points of its own takes a + measured one's values with `values_from`, and is then held to that one's points.""" + entries = { + (entry.estimator, entry.version, _platform_key(entry.platform)): entry + for data in shipped_data().values() + for entry in data.coefficients + } + decided: set[tuple[tuple[str, int, tuple[tuple[str, str], ...]], str]] = set() + for data in shipped_data().values(): + for point in data.points: + key = (point.estimator, point.version, _platform_key(point.platform)) + lender = entries[key].values_from + owner = (lender.estimator, lender.version, key[2]) if lender is not None else key + estimate = _evaluate(point) + decided |= {(owner, term.coefficient) for term in estimate.terms if term.included and term.coefficient} + + undecided = [ + f"{entry.estimator} {entry.platform} {name}" + for key, entry in entries.items() + for name in entry.values or {} + if (key, name) not in decided + ] + assert not undecided, f"no measured point decides these shipped values: {undecided}" + + +@pytest.mark.parametrize("point", POINTS) +def test_every_measured_point_is_bounded_by_its_estimate(point: MeasuredPoint) -> None: + estimate = _evaluate(point) + fitted = _fitted_decides(estimate) + + reserved = estimate.reservation_bytes if fitted else max(estimate.reservation_bytes, WORKING_MEMORY_FLOOR) + assert reserved >= point.measured_bytes, ( + f"reserves {reserved / 2**30:.3f} GiB for a measured {point.measured_bytes / 2**30:.3f} GiB" + ) + if point.max_ratio is not None or fitted: + max_ratio = point.max_ratio if point.max_ratio is not None else MAX_OVER_RESERVATION + assert estimate.reservation_bytes <= max_ratio * point.measured_bytes, ( + f"reserves {estimate.reservation_bytes / point.measured_bytes:.2f}x the measured peak, over {max_ratio}x" + ) + + +def test_a_point_bound_is_never_looser_than_the_default() -> None: + """`max_ratio` tightens the bound for a point the estimate has to stay close to; it never excuses one.""" + for name, data in shipped_data().items(): + for point in data.points: + if point.max_ratio is not None: + assert 1.0 <= point.max_ratio < MAX_OVER_RESERVATION, f"{name}: {_point_id(point)}" + + +@pytest.mark.parametrize( + "device, hip, conv_backend", + [ + (torch.device("cuda", 0), "7.2.0", "miopen"), + (torch.device("cuda", 0), None, "cudnn"), + # A HIP build reports its GPU as "cuda"; anything else on that build is not on MIOpen. + (torch.device("cpu"), "7.2.0", "cpu"), + (torch.device("mps"), "7.2.0", "mps"), + (torch.device("xpu"), None, "xpu"), + (torch.device("meta"), None, "cpu"), + ], +) +def test_the_platform_is_the_convolution_backend_the_device_really_runs(device, hip, conv_backend, monkeypatch) -> None: + monkeypatch.setattr(torch.version, "hip", hip) + assert platform_for(device) == Platform(conv_backend=conv_backend) + + +def _no_terms(features, values): + return () + + +def _spec(spec_id: str, version: int = 1) -> EstimatorSpec: + return EstimatorSpec( + id=spec_id, version=version, coefficients=frozenset({"k"}), platform_keys=("conv_backend",), terms=_no_terms + ) + + +def _entry(estimator: str, *, version: int = 1, values=None, values_from: str | None = None) -> CoefficientEntry: + return CoefficientEntry( + estimator=estimator, + version=version, + platform={"conv_backend": "cudnn"}, + values=values, + values_from=ValuesFrom(estimator=values_from, version=1) if values_from else None, + provenance="-", + ) + + +def _data(*entries: CoefficientEntry) -> list[WorkingMemoryData]: + return [WorkingMemoryData(schema_version=1, coefficients=list(entries), points=[])] + + +def test_a_missing_or_stale_entry_names_the_estimator_and_platform() -> None: + """No fallback to another platform or version: coefficients fitted for something else are the shortfall the data + exists to prevent.""" + spec = _spec("test.estimator", version=2) + + assert resolve_coefficients( + _data(_entry(spec.id, version=2, values={"k": 7.0})), spec, Platform("cudnn"), origin="test" + ) == ResolvedCoefficients(values={"k": 7.0}, source="test:conv_backend=cudnn") + with pytest.raises(LookupError, match=r"'test\.estimator' version 2 on conv_backend=miopen\."): + resolve_coefficients(_data(_entry(spec.id, version=2, values={"k": 7.0})), spec, Platform("miopen"), "test") + with pytest.raises(LookupError, match=r"'test\.estimator' version 2 on conv_backend=cudnn \(found only version"): + resolve_coefficients(_data(_entry(spec.id, version=1, values={"k": 7.0})), spec, Platform("cudnn"), "test") + with pytest.raises(LookupError, match=r"declared 2 times"): + resolve_coefficients( + _data(_entry(spec.id, version=2, values={"k": 7.0}), _entry(spec.id, version=2, values={"k": 8.0})), + spec, + Platform("cudnn"), + "test", + ) + + +def test_borrowed_values_resolve_one_hop_and_the_source_names_both_borrowings() -> None: + data = _data(_entry("test.lender", values={"k": 7.0}), _entry("test.borrower", values_from="test.lender")) + + on_cudnn = resolve_coefficients(data, _spec("test.borrower"), Platform("cudnn"), origin="test") + assert on_cudnn == ResolvedCoefficients(values={"k": 7.0}, source="test:conv_backend=cudnn (values of test.lender)") + on_xpu = resolve_coefficients(data, _spec("test.borrower"), Platform("xpu"), origin="test") + assert on_xpu == ResolvedCoefficients( + values={"k": 7.0}, source="test:conv_backend=xpu (cudnn values of test.lender)" + ) + + +@pytest.mark.parametrize( + "entries, message", + [ + pytest.param( + (_entry("a", values_from="b"), _entry("b", values_from="c"), _entry("c", values={"k": 1.0})), + "borrows in turn", + id="chain", + ), + pytest.param((_entry("a", values_from="b"), _entry("b", values_from="a")), "borrows in turn", id="cycle"), + pytest.param( + (_entry("a", values_from="b"),), r"No working-memory coefficients for estimator 'b'", id="no-lender" + ), + pytest.param( + (_entry("a", values_from="b"), _entry("b", values={"other": 1.0})), r"provide \['other'\]", id="mismatch" + ), + ], +) +def test_a_borrowing_that_is_not_one_hop_to_matching_values_is_refused(entries, message) -> None: + with pytest.raises(LookupError, match=message): + resolve_coefficients(_data(*entries), _spec("a"), Platform("cudnn"), origin="test") + + +@pytest.mark.parametrize("values, values_from", [({"k": 1.0}, "b"), (None, None)], ids=["both", "neither"]) +def test_an_entry_carries_exactly_one_of_values_and_values_from(values, values_from) -> None: + with pytest.raises(ValidationError, match="exactly one of"): + _entry("a", values=values, values_from=values_from) + + +def test_a_data_directory_without_data_files_is_named(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match=re.escape(str(tmp_path))): + load_data(tmp_path) + with pytest.raises(FileNotFoundError, match=re.escape(str(tmp_path / "missing"))): + load_data(tmp_path / "missing") + (tmp_path / "broken.json").write_text('{"schema_version": 1}', encoding="utf-8") + with pytest.raises(ValueError, match="broken.json"): + load_data(tmp_path) + + +@pytest.mark.platform_independent +def test_every_module_that_declares_an_estimator_is_listed() -> None: + """The registry fills as modules import, so a spec declared outside `ESTIMATOR_MODULES` is missing from every + check above until something else happens to import it -- and a ROCm user is the first to find its gap.""" + package = Path(invokeai.__file__).parent + declaring = sorted( + ".".join(path.relative_to(package.parent).with_suffix("").parts) + for owner in ("app", "backend") + for path in (package / owner).rglob("*.py") + if re.search(r"\bEstimatorSpec\(", path.read_text(encoding="utf-8")) + ) + assert declaring, "found no estimator modules -- the scan is broken" + unlisted = sorted(set(declaring) - set(ESTIMATOR_MODULES)) + stale = sorted(set(ESTIMATOR_MODULES) - set(declaring)) + assert not unlisted and not stale, f"add to ESTIMATOR_MODULES: {unlisted}; declares no spec: {stale}" diff --git a/tests/backend/working_memory/test_spec_registry.py b/tests/backend/working_memory/test_spec_registry.py new file mode 100644 index 00000000000..0c323aa179a --- /dev/null +++ b/tests/backend/working_memory/test_spec_registry.py @@ -0,0 +1,72 @@ +"""Registry mechanics, against throwaway specs rather than the shipped ones.""" + +import pytest + +from invokeai.backend.working_memory import spec as spec_module +from invokeai.backend.working_memory.spec import EstimatorSpec, register, registered_specs + + +@pytest.fixture(autouse=True) +def isolated_registry(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(spec_module, "_SPECS", {}) + + +def _module_terms(): + """A fresh `terms` function on every call, with one module and qualified name: what `importlib.reload` of its + defining module produces.""" + + def terms(features, values): + return () + + return terms + + +def _another_estimators_terms(features, values): + return () + + +def _spec(terms, version: int = 1) -> EstimatorSpec: + return EstimatorSpec( + id="test.estimator", + version=version, + coefficients=frozenset({"k"}), + platform_keys=("conv_backend",), + terms=terms, + ) + + +def test_re_executing_the_defining_module_adopts_its_new_terms() -> None: + """`importlib.reload`: the same declaration from the re-executed function.""" + before, after = _module_terms(), _module_terms() + register(_spec(before)) + register(_spec(after)) + + assert registered_specs()["test.estimator"].terms is after + + +def test_a_differing_declaration_under_a_taken_id_raises() -> None: + register(_spec(_module_terms())) + with pytest.raises(ValueError, match="already registered"): + register(_spec(_module_terms(), version=2)) + + assert registered_specs()["test.estimator"].version == 1, "the first one stands" + + +def test_the_same_declaration_from_another_function_raises() -> None: + """`terms` is not part of a spec's equality, so this is the case equality alone would let through.""" + first = _module_terms() + register(_spec(first)) + with pytest.raises(ValueError, match="ids are unique"): + register(_spec(_another_estimators_terms)) + + assert registered_specs()["test.estimator"].terms is first + + +def test_the_same_function_registered_twice_raises() -> None: + """Two specs of one module sharing a `terms` function -- as SD3 and CogView4 do -- under one id. `importlib.reload` + makes new function objects, so it never hands over the same one twice; jurigged re-running an edited `register` + statement does, and raises too (see `register`).""" + terms = _module_terms() + register(_spec(terms)) + with pytest.raises(ValueError, match="ids are unique"): + register(_spec(terms)) From ef1bbec220a99fd344725a50ea13867a9fa41583 Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Sat, 17 Oct 2026 15:23:39 +0200 Subject: [PATCH 2/5] fix(working-memory): reserve VAE working memory per dtype and allocator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit VAE estimators price a measured slope and base per conv backend, allocator and dtype instead of scaling one bf16 constant by element size; CogView4 gets its own lines. SD/SDXL nodes and their estimate share one run-dtype rule; on ROCm Windows an SDXL fp32 1024² decode reserves 6.2 instead of 14.6 GB and the UNet stays resident. --- .../Architecture/model-manager.mdx | 15 +- .../app/invocations/vae/image_to_latents.py | 13 +- .../app/invocations/vae/latents_to_image.py | 16 +- invokeai/app/util/torch_cuda_allocator.py | 17 +- .../load/model_cache/model_cache.py | 22 +- invokeai/backend/util/vae_working_memory.py | 278 +- .../backend/working_memory/coefficients.py | 124 +- invokeai/backend/working_memory/data/vae.json | 5971 ++++++++++++++++- invokeai/backend/working_memory/fit.py | 142 + invokeai/backend/working_memory/platform.py | 93 +- scripts/calibrate_flux2_working_memory.py | 41 +- .../invocations/test_flux2_working_memory.py | 30 +- .../test_vae_working_memory_backend.py | 380 +- .../invocations/test_z_image_tiled_decode.py | 47 +- .../test_model_cache_vram_accounting.py | 4 +- .../util/test_rocm_sdpa_head_dim_guard.py | 26 +- tests/backend/working_memory/test_fit.py | 135 + .../test_shipped_coefficients.py | 376 +- tests/conftest.py | 12 + 19 files changed, 7129 insertions(+), 613 deletions(-) create mode 100644 invokeai/backend/working_memory/fit.py create mode 100644 tests/backend/working_memory/test_fit.py diff --git a/docs/src/content/docs/development/Architecture/model-manager.mdx b/docs/src/content/docs/development/Architecture/model-manager.mdx index 2dfa5d93010..fb678b29653 100644 --- a/docs/src/content/docs/development/Architecture/model-manager.mdx +++ b/docs/src/content/docs/development/Architecture/model-manager.mdx @@ -987,10 +987,19 @@ An estimator that declares an `EstimatorSpec` returns a `WorkingMemoryEstimate` To write such an estimator (`invokeai/backend/util/vae_working_memory.py` has five): - Register an `EstimatorSpec` at module import, and list the module in `ESTIMATOR_MODULES` (`working_memory/spec.py`); the data test fails on a module that declares a spec without being listed. -- Split the estimator in two. The impure part reads the model and the device and records every runtime decision (tile, element size, score-matrix bytes) as a feature. The pure `terms(features, values)` turns features and coefficient values into terms, building fitted terms with `WorkingMemoryTerm.fitted(...)`. The public function returns `SPEC.estimate(features, device)`. +- Split the estimator in two. The impure part reads the model and the device and records every runtime decision (tile, the dtype the operation runs in, score-matrix bytes) as a feature, and refuses a dtype the spec has no line for with a `ValueError` naming the estimator, dtype and platform. The pure `terms(features, values)` turns features and coefficient values into terms, building fitted terms with `WorkingMemoryTerm.fitted(...)`. The public function returns `SPEC.estimate(features, device)`. +- Record the dtype the operation will run in, not the one the cached model holds. The SD1/SDXL nodes cast the shared VAE with `sd_vae_run_dtype`: fp16 for every GPU run that does not ask for fp32 (bf16 never runs), fp32 when asked and for a CPU decode. Their estimate prices the same function's answer, plus what the in-place cast adds to the resident weights. Uncast VAEs compute in their first floating-point parameter wider than a byte, so fp8 or packed SDNQ storage does not count. - Bump `version` whenever `terms` changes the formula. -- Ship one coefficient entry per measured platform value in `invokeai/backend/working_memory/data/`, either with its own `values` or with `values_from` another estimator's entry. Unmeasured platform values (MPS, XPU, CPU) are priced as `UNMEASURED_PRICED_AS` in `working_memory/platform.py` says. -- Ship the measured points next to them: peak reserved bytes (`reserved_peak`), one fresh process per point. `tests/backend/working_memory/` holds each estimate to at least its point, and where a fitted term decides it, to at most 1.3 times the point, or the point's tighter `max_ratio`. The `device_working_mem_gb` floor counts only where an analytic term decides. Every value an entry carries must be decided by at least one point. +- Ship one coefficient entry per measured platform in `invokeai/backend/working_memory/data/`, either with its own `values` or with `values_from` another estimator's entry, which may carry more values than the borrower reads. + - A platform is the convolution backend (`cudnn`, `miopen`, `mps`, `xpu`, `cpu`), torch's caching allocator and the OS. The allocator is torch's backend first (`cudaMallocAsync` ignores expandable segments), then `expandable` if the allocator environment variables enable expandable segments on the native allocator, else `native`; anything but a CUDA/HIP GPU is `native`. + - `MEASURED` in `working_memory/platform.py` lists the measured backend, allocator and OS combinations (an OS of `None` for every OS). `priced_as` prices every other pair as one of them, keeping the OS: a backend nothing is measured on as cuDNN with the native allocator, an allocator not measured with its backend as that backend's native allocator. There the entry that carries the values raises them to its `unmeasured_floor`, the pre-dtype slopes. + - An entry applies to the OS only when its platform names one, and then takes precedence there over the one that does not. Only MIOpen names one. With the native allocator, Windows demand is concave and up to 1.6x Linux's, so Windows has its own fit and every other OS keeps the Linux one. Expandable segments are measured on Windows only, where they are Invoke's ROCm default, so `MEASURED` lists them for Windows and every other OS prices them as the native allocator there. cuDNN is priced the same on every OS. +- Ship the measured points next to them, one fresh process per point: `reserved_peak` (the allocator's peak reserved bytes), `driver_peak` (the driver-level device usage), or `demand_peak` (the largest of driver-level, reserved and allocated peak, over two runs, which catches the memory HIP maps beyond the allocator on a process's first operation). A point names the OS only where it measured an OS-specific entry. `tests/backend/working_memory/` holds each estimate to at least its point, and where a fitted term decides an in-fit point, to at most 1.3 times the point, or the point's tighter `max_ratio`. Points no line can hold within 1.3x are listed with their own ratio in the test's `UPPER_BOUND_EXCEPTIONS`. The `device_working_mem_gb` floor counts only where an analytic term decides. +- Fit the values, never hand-tune them. The AutoencoderKL estimators price their convolutions as a line per operation and dtype: `k__` bytes per output pixel plus a base of `b__` bytes. + - `working_memory/fit.py` fits each line on the untiled single-sample in-fit points of every estimator that resolves to the entry. The line is the edge of the points' upper convex hull (with the origin, so the base is never negative) that over-reserves its worst point least, the steeper on a tie and never a falling one. It needs at least two sizes twice apart in pixels. The shipped values add a 5% margin and round up: the slope to a whole byte per pixel, the base to a whole MiB. + - A point measured in a regime its platform does not run sets `in_fit: false`: it fits nothing and gets only the lower bound (CUDA with the math attention kernel forced). The test lists every such point. + - A line no in-fit point measures is a declared stand-in, listed in the entry's `stand_ins` and never below the pre-dtype slope. MIOpen with the native allocator off Windows prices fp16 and fp32 at the pre-dtype figure, and CogView4's lines there at its expandable-segments ones. + - Every value an entry carries is a stand-in or decided by an in-fit point. The data test fails when a fitted value is not the fit of its points, naming the value to ship, and when an in-fit point measures a stand-in's line. ### Emitting model loading events diff --git a/invokeai/app/invocations/vae/image_to_latents.py b/invokeai/app/invocations/vae/image_to_latents.py index d599fadd862..1be14f136b2 100644 --- a/invokeai/app/invocations/vae/image_to_latents.py +++ b/invokeai/app/invocations/vae/image_to_latents.py @@ -27,7 +27,7 @@ from invokeai.backend.stable_diffusion.diffusers_pipeline import image_resized_to_grid_as_tensor from invokeai.backend.stable_diffusion.vae_tiling import patch_vae_tiling_params from invokeai.backend.util.devices import TorchDevice -from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_sd15_sdxl +from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_sd15_sdxl, sd_vae_run_dtype """ SDXL VAE color compensation values determined experimentally to reduce color drift. @@ -86,9 +86,10 @@ def vae_encode( with vae_info.model_on_device(working_memory=estimated_working_memory) as (_, vae): assert isinstance(vae, (AutoencoderKL, AutoencoderTiny)), "VAE must be of type SD-1.5 or SDXL" orig_dtype = vae.dtype - if upcast: - vae.to(dtype=torch.float32) - + # The dtype the estimate above priced: fp32 when upcasting, fp16 otherwise, on every device. + run_dtype = sd_vae_run_dtype("encode", upcast, vae_info.compute_device) + vae.to(dtype=run_dtype) + if run_dtype == torch.float32: use_torch_2_0_or_xformers = hasattr(vae.decoder, "mid_block") and isinstance( vae.decoder.mid_block.attentions[0].processor, ( @@ -107,10 +108,6 @@ def vae_encode( # else: # latents = latents.float() - else: - vae.to(dtype=torch.float16) - # latents = latents.half() - if tiled: vae.enable_tiling() else: diff --git a/invokeai/app/invocations/vae/latents_to_image.py b/invokeai/app/invocations/vae/latents_to_image.py index 29c6a1b9db8..bb2aafd0bfb 100644 --- a/invokeai/app/invocations/vae/latents_to_image.py +++ b/invokeai/app/invocations/vae/latents_to_image.py @@ -20,7 +20,7 @@ from invokeai.backend.stable_diffusion.extensions.seamless import SeamlessExt from invokeai.backend.stable_diffusion.vae_tiling import patch_vae_tiling_params from invokeai.backend.util.devices import TorchDevice -from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_sd15_sdxl +from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_sd15_sdxl, sd_vae_run_dtype @invocation( @@ -74,16 +74,12 @@ def invoke(self, context: InvocationContext) -> ImageOutput: # wrongly place the latents (and thus the whole decode) on the CPU (see #9373). device = vae_info.compute_device latents = latents.to(device) - # Force fp32 when running on CPU (e.g. when the VAE is configured cpu_only). fp16 conv does run on CPU with + # fp32 when asked, and on the CPU (e.g. when the VAE is configured cpu_only): fp16 conv does run on CPU with # the pinned torch, but it's much slower than fp32 there, and SD/SDXL VAE has known fp16 overflow issues - # that produce black images — fp32 avoids both. - if self.fp32 or device.type == "cpu": - # FP32 mode: convert everything to float32 for maximum precision - vae.to(dtype=torch.float32) - latents = latents.float() - else: - vae.to(dtype=torch.float16) - latents = latents.half() + # that produce black images — fp32 avoids both. fp16 otherwise. The estimate above priced the same rule. + run_dtype = sd_vae_run_dtype("decode", self.fp32, device) + vae.to(dtype=run_dtype) + latents = latents.to(dtype=run_dtype) if use_tiling: vae.enable_tiling() diff --git a/invokeai/app/util/torch_cuda_allocator.py b/invokeai/app/util/torch_cuda_allocator.py index 7e94be57900..280f5bbe752 100644 --- a/invokeai/app/util/torch_cuda_allocator.py +++ b/invokeai/app/util/torch_cuda_allocator.py @@ -6,13 +6,26 @@ import sys from pathlib import Path -# Every variable torch reads its caching-allocator configuration from. `ModelCache._expandable_segments_enabled` keeps -# its own copy of this list: the model cache imports torch, and this module must not. +# Every variable torch reads its caching-allocator configuration from. _ALLOCATOR_CONF_ENV_VARS = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_HIP_ALLOC_CONF") ROCM_WINDOWS_ALLOC_CONF = "expandable_segments:True" +def expandable_segments_enabled() -> bool: + """Whether the torch caching allocator runs in expandable-segments mode. + + The mode is configured through the allocator env vars before torch import (InvokeAI's own + `pytorch_cuda_alloc_conf` setting is plumbed into PYTORCH_CUDA_ALLOC_CONF the same way), so + parsing them is authoritative for the life of the process. It lives here, beside the variables, because this + module imports neither torch nor its callers: the model cache and the working-memory platform. + """ + for var in _ALLOCATOR_CONF_ENV_VARS: + if "expandable_segments:true" in os.environ.get(var, "").replace(" ", "").lower(): + return True + return False + + def _installed_torch_version() -> str | None: """The installed torch distribution's version, read from its metadata so torch itself is not imported.""" try: diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache.py b/invokeai/backend/model_manager/load/model_cache/model_cache.py index 3691f374425..6cf1345568e 100644 --- a/invokeai/backend/model_manager/load/model_cache/model_cache.py +++ b/invokeai/backend/model_manager/load/model_cache/model_cache.py @@ -1,6 +1,5 @@ import gc import logging -import os import queue import sys import threading @@ -15,6 +14,7 @@ import psutil import torch +from invokeai.app.util.torch_cuda_allocator import expandable_segments_enabled from invokeai.backend.model_manager.load.memory_snapshot import MemorySnapshot from invokeai.backend.model_manager.load.model_cache.cache_record import CacheRecord from invokeai.backend.model_manager.load.model_cache.cache_stats import CacheStats @@ -39,20 +39,6 @@ from invokeai.backend.util.logging import InvokeAILogger from invokeai.backend.util.prefix_logger_adapter import PrefixedLoggerAdapter - -def _expandable_segments_enabled() -> bool: - """Whether the torch caching allocator runs in expandable-segments mode. - - The mode is configured through the allocator env vars before torch import (InvokeAI's own - `pytorch_cuda_alloc_conf` setting is plumbed into PYTORCH_CUDA_ALLOC_CONF the same way), so - parsing them is authoritative for the life of the process. - """ - for var in ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_HIP_ALLOC_CONF"): - if "expandable_segments:true" in os.environ.get(var, "").replace(" ", "").lower(): - return True - return False - - # Size of a GB in bytes. GB = 2**30 @@ -2231,7 +2217,7 @@ def _release_allocator_blocks_for_reserve(self, working_mem_bytes: Optional[int] driver_free, _ = TorchDevice.cuda_mem_get_info(self._execution_device) if driver_free >= reserve: return - if _expandable_segments_enabled(): + if expandable_segments_enabled(): # Freed pages inside a segment are not counted as reclaimable there, but empty_cache() unmaps them. releasable = torch.cuda.memory_reserved(self._execution_device) - torch.cuda.memory_allocated( self._execution_device @@ -2403,7 +2389,7 @@ def _get_reclaimable_allocator_bytes(self) -> int: `_offload_unlocked_models` runs one after each offload in that mode; what it cannot do is make the credit trustworthy before it runs. """ - if _expandable_segments_enabled(): + if expandable_segments_enabled(): return 0 try: if self._execution_device.type == "cuda": @@ -2618,7 +2604,7 @@ def _offload_unlocked_models( # them -- so it is deliberately loop-local: it stops this loop offloading everything, and # the load that follows re-measures for itself (`lock`, `make_room_in_vram`) and streams # from RAM if the room did not materialize. - empty_cache_per_offload = _expandable_segments_enabled() + empty_cache_per_offload = expandable_segments_enabled() vram_bytes_freed_uncounted = 0 # TODO(ryand): Give more thought to the offloading policy used here. cache_entries_increasing_size = sorted(self._cached_models.values(), key=lambda x: x.cached_model.total_bytes()) diff --git a/invokeai/backend/util/vae_working_memory.py b/invokeai/backend/util/vae_working_memory.py index 6462dc1f41a..88e23008228 100644 --- a/invokeai/backend/util/vae_working_memory.py +++ b/invokeai/backend/util/vae_working_memory.py @@ -16,6 +16,7 @@ from invokeai.backend.util.logging import InvokeAILogger from invokeai.backend.util.vae_tiling_scope import resolve_tile_size from invokeai.backend.working_memory.estimate import FeatureValue, WorkingMemoryEstimate, WorkingMemoryTerm +from invokeai.backend.working_memory.platform import platform_for from invokeai.backend.working_memory.spec import EstimatorSpec, register # Share of the device's memory an untiled decode may claim before the decode nodes tile it up front (see @@ -77,7 +78,7 @@ def should_pretile_vae_decode( return True -# The diffusers AutoencoderKL (SD1/SDXL, SD3, CogView4) and the FLUX.1 AutoEncoder run the same +# The diffusers AutoencoderKL (SD1/SDXL, SD3) and the FLUX.1 AutoEncoder run the same # mid-block self-attention as the FLUX.2 VAE: one head over the 512-channel width, on the # 8x-downsampled grid. Where that attention runs on the math kernel (see `sdpa_score_matrix_bytes`) # its score matrix is the dominant term for a large untiled image, exactly as it is for FLUX.2; @@ -114,25 +115,85 @@ def _vae_mid_block_score_matrix_bytes( ) -# The SD1/SDXL, SD3, CogView4, FLUX.1 and FLUX.2 VAEs run the same diffusers AutoencoderKL convolution stack, and pay -# the same for it: peak reserved bytes per output pixel per element byte, one fitted coefficient per operation -# (`k_decode`, `k_encode`) and convolution backend. The values ship in `backend/working_memory/data/vae.json`, with the -# points they were fitted on and the per-card table behind them; `scripts/calibrate_flux2_working_memory.py --only vae` -# reproduces that table on any build. +# The SD1/SDXL, SD3, FLUX.1 and FLUX.2 VAEs run the same diffusers AutoencoderKL convolution stack, and pay the same +# for it: a line in the output's pixel count per operation and the dtype it runs in, `k__` +# bytes per pixel plus `b__` bytes the process pays whatever the size, shipped per convolution +# backend and allocator (`platform_for`). The values ship in `backend/working_memory/data/vae.json` with the points +# they are fitted on (`backend/working_memory/fit.py`); `scripts/calibrate_flux2_working_memory.py --only vae` +# measures a build against them. # -# MIOpen's convolution workspaces are simply larger than cuDNN's -- ~1.6x for a decode, ~2.4x for an encode. This is -# not the attention term: it shows up identically on the fused path, and the implied coefficients are flat across -# resolution on both backends, so the linear model itself holds. Nor is "encoding costs half of decoding" architectural: -# it is a cuDNN property (0.49 there, 0.78 on MIOpen), which is why each operation carries its own coefficient instead -# of a ratio. Shipping the MIOpen values everywhere would add ~60% to every cuDNN decode for nothing, so the -# coefficients follow the backend of the device the VAE runs on (`platform_for`). +# None of the three scales out of the line. The demand does not follow the element size: on MIOpen fp16 needs about a +# third of what bf16 does, and on both backends fp32 needs less per value byte than bf16. The allocator matters as +# much: `cudaMallocAsync` needs 25-35% less than the native allocator on cuDNN. And MIOpen's bf16 workspaces are +# larger than cuDNN's, ~1.5x for a decode and ~2.4x for an encode -- not the attention term, which shows up the same +# on the fused path. The base is why the per-pixel figure rises at small sizes; on HIP it includes the ~300 MiB the +# driver maps beyond the allocator's own figure on the first operation of a process. +# +# Each estimator declares the lines of the dtypes it can run, and the estimate prices the dtype the operation will run +# in, which is not always the dtype the cached VAE holds (see `sd_vae_run_dtype`). CogView4's VAE is an AutoencoderKL +# too, but a wider one (block widths up to 1024, three layers per block, no mid-block attention): it carries its own +# lines, about 1.9x the family's for a decode. # # Caveat for AMD users: `MIOPEN_FIND_MODE=2` (FAST) selects convolution algorithms heuristically instead of by # benchmark, and measured a uniform 1.28x more memory at every resolution when the MIOpen values were first fitted. -# Invoke has since made FAST its default (`run_app.py`); the RX 9060 XT measurement was taken under it and did not show -# the 1.28x. If you see decodes exceed their reservation on another card, raise `device_working_mem_gb`. -_AUTOENCODER_KL_COEFFICIENTS = frozenset({"k_decode", "k_encode"}) -_CONV_BACKEND_KEYS = ("conv_backend",) +# Invoke has since made FAST its default (`run_app.py`); the RX 9060 XT points were taken under it. If you see decodes +# exceed their reservation on another card, raise `device_working_mem_gb`. +_PLATFORM_KEYS = ("conv_backend", "allocator") +_DTYPE_NAMES: dict[torch.dtype, str] = {torch.float16: "fp16", torch.bfloat16: "bf16", torch.float32: "fp32"} +_ELEMENT_SIZES = {"fp16": 2, "bf16": 2, "fp32": 4} + + +def _convolution_coefficients(*dtypes: str) -> frozenset[str]: + """The coefficients of the convolution lines for both operations in these dtypes.""" + return frozenset( + f"{part}_{operation}_{dtype}" for part in ("k", "b") for operation in ("decode", "encode") for dtype in dtypes + ) + + +def _dtype_name(dtype: torch.dtype) -> str: + name = _DTYPE_NAMES.get(dtype) + if name is None: + raise ValueError(f"No VAE working-memory coefficients are measured for {dtype}.") + return name + + +def _compute_dtype(vae: torch.nn.Module) -> torch.dtype: + """The dtype a VAE no node casts computes in: that of its first floating-point parameter wider than a byte. + + Narrower or integer weights (fp8 storage, an SDNQ VAE's packed ones) are what a layer dequantizes from, not what + its activations are. + """ + for param in vae.parameters(): + if param.is_floating_point() and param.element_size() > 1: + return param.dtype + raise ValueError(f"{type(vae).__name__} has no floating-point parameters to compute in.") + + +def _estimate(spec: EstimatorSpec, features: Mapping[str, FeatureValue], device: torch.device) -> WorkingMemoryEstimate: + """`spec.estimate`, refusing a dtype the spec has no line for with the estimator, dtype and platform named, rather + than a bare `KeyError` from inside the terms.""" + line = f"k_{features['operation']}_{features['dtype']}" + if line not in spec.coefficients: + raise ValueError( + f"Working-memory estimator '{spec.id}' has no {features['dtype']} {features['operation']} line on " + f"{platform_for(device)}: the VAE computes in a dtype it is not measured for." + ) + return spec.estimate(features, device) + + +def _convolutions( + features: Mapping[str, FeatureValue], values: Mapping[str, float], pixels: float, samples: int = 1 +) -> tuple[WorkingMemoryTerm, ...]: + """The convolution line of this operation and dtype over `pixels` output pixels, for `samples` samples. + + The base scales with the samples too. That is an upper bound rather than a fit -- the base is paid once per process + -- and it keeps a batch at exactly that many single-sample estimates. + """ + line = f"{features['operation']}_{features['dtype']}" + return ( + WorkingMemoryTerm.fitted("convolutions", f"k_{line}", pixels * samples, values), + WorkingMemoryTerm.fitted("convolution_base", f"b_{line}", samples, values), + ) def _peak_of_two_phases( @@ -170,40 +231,60 @@ def _peak_of_two_phases( def _sd_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float]) -> tuple[WorkingMemoryTerm, ...]: - coefficient = f"k_{features['operation']}" - element_size = cast(int, features["element_size"]) tile = cast(int | None, features["tile"]) if tile is not None: - area_bytes = tile * tile * element_size # We add 25% to the working memory estimate when tiling is enabled to account for factors like tile overlap # and number of tiles. We could make this more precise in the future, but this should be good enough for # most use cases. - convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * 1.25, values) + terms = _convolutions(features, values, tile * tile * 1.25) else: - area_bytes = cast(int, features["height"]) * cast(int, features["width"]) * element_size - convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes, values) + terms = _convolutions(features, values, cast(int, features["height"]) * cast(int, features["width"])) - terms: tuple[WorkingMemoryTerm, ...] = (convolutions,) # The tiny autoencoder has no mid-block attention, so it is priced without a score matrix. if features["vae_class"] == "AutoencoderKL": terms = _peak_of_two_phases(terms, cast(int, features["score_matrix_bytes"])) - if features["fp32"]: - # If we are running in FP32, then we should account for the likely increase in model size (~250MB). - terms += (WorkingMemoryTerm("fp32_weights", 250 * 2**20, True),) + upcast_weight_bytes = cast(int, features["upcast_weight_bytes"]) + if upcast_weight_bytes: + terms += (WorkingMemoryTerm("upcast_weights", upcast_weight_bytes, True),) return terms VAE_SD_SPEC = register( EstimatorSpec( id="vae.sd", - version=1, - coefficients=_AUTOENCODER_KL_COEFFICIENTS, - platform_keys=_CONV_BACKEND_KEYS, + version=2, + coefficients=_convolution_coefficients("fp16", "fp32"), + platform_keys=_PLATFORM_KEYS, terms=_sd_terms, ) ) +def sd_vae_run_dtype(operation: Literal["encode", "decode"], fp32: bool, device: torch.device) -> torch.dtype: + """The dtype the SD1/SDXL nodes cast the shared VAE to before they run it: `l2i` and `i2l` cast with this, and + `estimate_vae_working_memory_sd15_sdxl` prices it, so the two cannot drift. + + fp32 when the node asks for it, and for a decode on the CPU; fp16 otherwise, on every GPU and for every + `precision`, so bf16 never runs. The cached VAE's own dtype says nothing about it: it is whatever the previous run + cast it to. + """ + if fp32 or (operation == "decode" and device.type == "cpu"): + return torch.float32 + return torch.float16 + + +def _upcast_weight_bytes(vae: torch.nn.Module, run_dtype: torch.dtype) -> int: + """What the node's in-place cast to `run_dtype` adds to the VAE's resident weights. + + The cast runs once the cache has placed the VAE, so the growth comes out of the reservation: 160 MiB for the 83.7M + parameters of the SD1/SDXL VAE going from fp16 to fp32, nothing for a VAE a previous run left in fp32. Measured + points exclude resident weights by construction (they record 0 here), so the fitted line does not cover it. An + upcast encode casts the decoder's mid-block and post-quant conv back, so for it this counts a little more than + grows: conservative. + """ + return sum(p.numel() * max(0, run_dtype.itemsize - p.element_size()) for p in vae.parameters()) + + def estimate_vae_working_memory_sd15_sdxl( operation: Literal["encode", "decode"], image_tensor: torch.Tensor, @@ -214,12 +295,13 @@ def estimate_vae_working_memory_sd15_sdxl( ) -> WorkingMemoryEstimate: """Estimate the working memory required to encode or decode the given tensor. - It was found experimentally that the peak working memory scales linearly with the number of pixels and the element - size (precision). This estimate is accurate for both SD1 and SDXL. `device` is where the VAE will run; it picks the - convolution backend's coefficients and defaults to the session device. + It was found experimentally that the peak working memory is a line in the number of pixels, per dtype. This + estimate is accurate for both SD1 and SDXL. It prices the dtype the node will cast the VAE to (`sd_vae_run_dtype`), + not the one the cached VAE holds. `device` is where the VAE will run; it picks the platform's coefficients and + defaults to the session device. """ - element_size = 4 if fp32 else 2 device = device if device is not None else TorchDevice.choose_torch_device() + run_dtype = sd_vae_run_dtype(operation, fp32, device) latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 height = latent_scale_factor_for_operation * image_tensor.shape[-2] width = latent_scale_factor_for_operation * image_tensor.shape[-1] @@ -233,26 +315,23 @@ def estimate_vae_working_memory_sd15_sdxl( "height": height, "width": width, "tile": tile_size, - "element_size": element_size, - "fp32": fp32, + "dtype": _dtype_name(run_dtype), "vae_class": "AutoencoderKL" if isinstance(vae, AutoencoderKL) else "AutoencoderTiny", + "upcast_weight_bytes": _upcast_weight_bytes(vae, run_dtype), } if isinstance(vae, AutoencoderKL): - score_dtype = torch.float32 if fp32 else next(vae.parameters()).dtype score_h, score_w = (tile_size, tile_size) if tile_size is not None else (height, width) - features["score_matrix_bytes"] = _vae_mid_block_score_matrix_bytes(score_h, score_w, score_dtype, device=device) + features["score_matrix_bytes"] = _vae_mid_block_score_matrix_bytes(score_h, score_w, run_dtype, device=device) - return VAE_SD_SPEC.estimate(features, device) + return _estimate(VAE_SD_SPEC, features, device) def _single_pass_terms( features: Mapping[str, FeatureValue], values: Mapping[str, float] ) -> tuple[WorkingMemoryTerm, ...]: """An untiled AutoencoderKL pass: the convolutions over the whole image, or the mid-block score matrix.""" - coefficient = f"k_{features['operation']}" - area_bytes = cast(int, features["height"]) * cast(int, features["width"]) * cast(int, features["element_size"]) - convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes, values) - return _peak_of_two_phases((convolutions,), cast(int, features["score_matrix_bytes"])) + convolutions = _convolutions(features, values, cast(int, features["height"]) * cast(int, features["width"])) + return _peak_of_two_phases(convolutions, cast(int, features["score_matrix_bytes"])) def _single_pass_features( @@ -261,23 +340,30 @@ def _single_pass_features( latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 height = latent_scale_factor_for_operation * image_tensor.shape[-2] width = latent_scale_factor_for_operation * image_tensor.shape[-1] - param = next(vae.parameters()) + dtype = _compute_dtype(vae) return { "operation": operation, "height": height, "width": width, - "element_size": param.element_size(), - "score_matrix_bytes": _vae_mid_block_score_matrix_bytes(height, width, param.dtype, device=device), + "dtype": _dtype_name(dtype), + "score_matrix_bytes": _vae_mid_block_score_matrix_bytes(height, width, dtype, device=device), } +def _cogview4_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float]) -> tuple[WorkingMemoryTerm, ...]: + """An untiled CogView4 VAE pass: its convolutions over the whole image. It has no mid-block attention + (`mid_block_add_attention: false`), so there is no score matrix to price.""" + return _convolutions(features, values, cast(int, features["height"]) * cast(int, features["width"])) + + +# The CogView4 loader forces bf16: the VAE produces black images in fp16. VAE_COGVIEW4_SPEC = register( EstimatorSpec( id="vae.cogview4", - version=1, - coefficients=_AUTOENCODER_KL_COEFFICIENTS, - platform_keys=_CONV_BACKEND_KEYS, - terms=_single_pass_terms, + version=2, + coefficients=_convolution_coefficients("bf16"), + platform_keys=_PLATFORM_KEYS, + terms=_cogview4_terms, ) ) @@ -294,7 +380,14 @@ def estimate_vae_working_memory_cogview4( device. """ device = device if device is not None else TorchDevice.choose_torch_device() - return VAE_COGVIEW4_SPEC.estimate(_single_pass_features(operation, image_tensor, vae, device), device) + latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 + features: dict[str, FeatureValue] = { + "operation": operation, + "height": latent_scale_factor_for_operation * image_tensor.shape[-2], + "width": latent_scale_factor_for_operation * image_tensor.shape[-1], + "dtype": _dtype_name(_compute_dtype(vae)), + } + return _estimate(VAE_COGVIEW4_SPEC, features, device) # What a tiled decode does *not* bound: the assembled image, several times over, per output pixel. @@ -330,28 +423,23 @@ def estimate_vae_working_memory_cogview4( def _flux_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float]) -> tuple[WorkingMemoryTerm, ...]: operation = features["operation"] - coefficient = f"k_{operation}" out_h = cast(int, features["height"]) out_w = cast(int, features["width"]) - element_size = cast(int, features["element_size"]) + element_size = _ELEMENT_SIZES[cast(str, features["dtype"])] tile = cast(int | None, features["tile"]) if tile is None: - area_bytes = out_h * out_w * element_size - convolution_phase: tuple[WorkingMemoryTerm, ...] = ( - WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes, values), - ) + convolution_phase = _convolutions(features, values, out_h * out_w) return _peak_of_two_phases(convolution_phase, cast(int, features["score_matrix_bytes"])) # `calc_tiles_min_overlap` clamps the tile to the image *per axis*, so the largest tile the # decode builds is this, not `tile` squared. Without the clamp an 8192px tile on a # 1024x8200 image -- tiled, because one axis exceeds it -- prices 8192x8192. - area_bytes = min(tile, out_h) * min(tile, out_w) * element_size # A 25% margin for tile overlap and the number of tiles, mirroring the SD1/SDXL estimator. - convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * 1.25, values) + convolutions = _convolutions(features, values, min(tile, out_h) * min(tile, out_w) * 1.25) if operation == "decode": convolution_phase = ( - convolutions, + *convolutions, WorkingMemoryTerm( "assembled_image", out_h * out_w * _IMAGE_CHANNELS * (_FLUX_VAE_TILED_IMAGE_COPIES * element_size + 1), @@ -377,19 +465,21 @@ def _flux_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float * element_size ) convolution_phase = ( - convolutions, + *convolutions, WorkingMemoryTerm("input_image", out_h * out_w * _IMAGE_CHANNELS * element_size, True), WorkingMemoryTerm("assembled_moments", moments * live_moment_copies, True), ) return _peak_of_two_phases(convolution_phase, cast(int, features["score_matrix_bytes"])) +# The FLUX.1 loaders avoid fp16, which this autoencoder is broken in, except one: an SDNQ VAE folder loads through the +# generic VAE loader and computes in the configured precision, which `auto` makes fp16 on CUDA. VAE_FLUX_SPEC = register( EstimatorSpec( id="vae.flux", - version=1, - coefficients=_AUTOENCODER_KL_COEFFICIENTS, - platform_keys=_CONV_BACKEND_KEYS, + version=2, + coefficients=_convolution_coefficients("fp16", "bf16", "fp32"), + platform_keys=_PLATFORM_KEYS, terms=_flux_terms, ) ) @@ -418,7 +508,7 @@ def estimate_vae_working_memory_flux( backend's coefficients. Defaults to the session device. """ latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 - param = next(vae.parameters()) + dtype = _compute_dtype(vae) device = device if device is not None else TorchDevice.choose_torch_device() out_h = latent_scale_factor_for_operation * image_tensor.shape[-2] @@ -443,9 +533,9 @@ def estimate_vae_working_memory_flux( "operation": operation, "height": out_h, "width": out_w, - "element_size": param.element_size(), + "dtype": _dtype_name(dtype), "tile": tile, - "score_matrix_bytes": _vae_mid_block_score_matrix_bytes(score_h, score_w, param.dtype, device=device), + "score_matrix_bytes": _vae_mid_block_score_matrix_bytes(score_h, score_w, dtype, device=device), } if tile is not None and operation != "decode": # Read from the VAE rather than hard-coded: an overlap factor of 0.5 would make the tiles @@ -454,7 +544,7 @@ def estimate_vae_working_memory_flux( features["latent_channels"] = getattr(getattr(vae, "config", None), "latent_channels", 16) features["tile_overlap_factor"] = getattr(vae, "tile_overlap_factor", 0.25) - return VAE_FLUX_SPEC.estimate(features, device) + return _estimate(VAE_FLUX_SPEC, features, device) # The FLUX.2 VAE runs one attention block at the bottom of the encoder and one at the top of the @@ -468,27 +558,25 @@ def estimate_vae_working_memory_flux( def _flux2_terms(features: Mapping[str, FeatureValue], values: Mapping[str, float]) -> tuple[WorkingMemoryTerm, ...]: - coefficient = f"k_{features['operation']}" - element_size = cast(int, features["element_size"]) batch_size = cast(int, features["batch_size"]) tile = cast(int | None, features["tile"]) if tile is not None: - area_bytes = tile * tile * element_size # Add 25% for tile overlap and the blending buffers, mirroring the SD1/SDXL estimate. - convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * 1.25 * batch_size, values) + convolutions = _convolutions(features, values, tile * tile * 1.25, samples=batch_size) else: - area_bytes = cast(int, features["height"]) * cast(int, features["width"]) * element_size - convolutions = WorkingMemoryTerm.fitted("convolutions", coefficient, area_bytes * batch_size, values) + pixels = cast(int, features["height"]) * cast(int, features["width"]) + convolutions = _convolutions(features, values, pixels, samples=batch_size) # Both terms are per sample; the larger is reserved (see `_peak_of_two_phases` for why, and the measurements). - return _peak_of_two_phases((convolutions,), cast(int, features["score_matrix_bytes"])) + return _peak_of_two_phases(convolutions, cast(int, features["score_matrix_bytes"])) +# The FLUX.2 loaders avoid fp16 (as FLUX.1's do), and none of them goes through the generic VAE loader. VAE_FLUX2_SPEC = register( EstimatorSpec( id="vae.flux2", - version=1, - coefficients=_AUTOENCODER_KL_COEFFICIENTS, - platform_keys=_CONV_BACKEND_KEYS, + version=2, + coefficients=_convolution_coefficients("bf16", "fp32"), + platform_keys=_PLATFORM_KEYS, terms=_flux2_terms, ) ) @@ -503,16 +591,16 @@ def estimate_vae_working_memory_flux2( ) -> WorkingMemoryEstimate: """Estimate the working memory required to encode or decode with the FLUX.2 (32-channel) VAE. - Peak memory scales linearly with pixel area and element size, as it does for the FLUX.1 VAE, and - the implied coefficient is flat across 512-1536px on every backend measured. What is *not* constant - is the coefficient itself: MIOpen's convolution workspaces cost ~1.6x cuDNN's for a decode and ~2.4x - for an encode, so it is shipped per backend -- see `_AUTOENCODER_KL_COEFFICIENTS` for the caveats - and `backend/working_memory/data/vae.json` for the fitted values and their points. Peak *reserved* - memory is what is measured throughout, the conservative quantity that includes allocator overhead. + Peak memory is a line in pixel area per dtype, as it is for the FLUX.1 VAE, from 512 to 2048px on + every backend measured. The line itself is not one: MIOpen's bf16 convolution workspaces cost ~1.5x + cuDNN's for a decode and ~2.4x for an encode, and the allocator and the dtype move it as much, so it + is shipped per platform and dtype -- see `_convolution_coefficients` for the caveats and + `backend/working_memory/data/vae.json` for the fitted values and their points. The points measure + peak *reserved* memory or more, the conservative quantity that includes allocator overhead. - For reference, decoding 1024x1024 peaks at ~4.3GB on cuDNN and ~6.6GB on MIOpen, and 1536x1536 at - ~9.6GB -- far above the default ``device_working_mem_gb``, which is why this estimate must be - passed to the model cache. + For reference, decoding 1024x1024 in bf16 peaks at ~4.3GB on cuDNN and ~6.6GB on MIOpen, and + 1536x1536 at ~9.6GB -- far above the default ``device_working_mem_gb``, which is why this estimate + must be passed to the model cache. That linear term holds only while ``AutoencoderKLFlux2``'s mid-block attention runs through a fused SDPA kernel, which is what CUDA does (verified: the memory-efficient kernel takes the @@ -540,7 +628,7 @@ def estimate_vae_working_memory_flux2( bound. The score matrix is shaped (batch, heads, S, S), so it scales the same way. The encode call sites all pass batch 1 today; the shared estimator does not assume it. """ - param = next(vae.parameters()) + dtype = _compute_dtype(vae) device = device if device is not None else TorchDevice.choose_torch_device() batch_size = image_tensor.shape[0] if image_tensor.dim() >= 4 else 1 latent_scale_factor_for_operation = LATENT_SCALE_FACTOR if operation == "decode" else 1 @@ -558,17 +646,17 @@ def estimate_vae_working_memory_flux2( "width": out_w, "tile": tile_size, "batch_size": batch_size, - "element_size": param.element_size(), + "dtype": _dtype_name(dtype), "score_matrix_bytes": sdpa_score_matrix_bytes( device=device, - dtype=param.dtype, + dtype=dtype, # The score matrix is (batch, heads, S, S); one head per sample prices the whole batch. num_heads=_FLUX2_VAE_MID_BLOCK_HEADS * batch_size, head_dim=_FLUX2_VAE_MID_BLOCK_HEAD_DIM, seq_len=mid_block_seq_len, ), } - return VAE_FLUX2_SPEC.estimate(features, device) + return _estimate(VAE_FLUX2_SPEC, features, device) def estimate_vae_working_memory_anima( @@ -1065,12 +1153,14 @@ def estimate_vae_working_memory_qwen_image( return int(working_memory) +# The SD3 VAE loads in the configured precision and runs uncast: `auto` gives fp16 on CUDA, and bf16 and fp32 are +# honoured. It carries the family's values in `data/vae.json`, as the one member that runs all three. VAE_SD3_SPEC = register( EstimatorSpec( id="vae.sd3", - version=1, - coefficients=_AUTOENCODER_KL_COEFFICIENTS, - platform_keys=_CONV_BACKEND_KEYS, + version=2, + coefficients=_convolution_coefficients("fp16", "bf16", "fp32"), + platform_keys=_PLATFORM_KEYS, terms=_single_pass_terms, ) ) @@ -1088,4 +1178,4 @@ def estimate_vae_working_memory_sd3( device. """ device = device if device is not None else TorchDevice.choose_torch_device() - return VAE_SD3_SPEC.estimate(_single_pass_features(operation, image_tensor, vae, device), device) + return _estimate(VAE_SD3_SPEC, _single_pass_features(operation, image_tensor, vae, device), device) diff --git a/invokeai/backend/working_memory/coefficients.py b/invokeai/backend/working_memory/coefficients.py index d8e07b13854..a341eb4bca4 100644 --- a/invokeai/backend/working_memory/coefficients.py +++ b/invokeai/backend/working_memory/coefficients.py @@ -1,15 +1,16 @@ """The coefficients estimators are evaluated with, and the measured points that bound them. Both ship as JSON under `data/`, read through `importlib.resources` so a wheel install finds them. A coefficient entry -belongs to one estimator version on one measured platform value, and either carries its values or borrows another -entry's (`values_from`); a point records the features of an operation that was measured and its measured peak. -`tests/backend/working_memory/test_shipped_coefficients.py` resolves every entry against its `EstimatorSpec` and holds -every point to the estimate the shipped coefficients give it. +belongs to one estimator version on one measured platform, and either carries its values or borrows another entry's +(`values_from`); a point records the features of an operation that was measured and its measured peak. A carried value +is the fit of the points it is resolved for (`fit.py`) or a declared stand-in (`stand_ins`). +`tests/backend/working_memory/test_shipped_coefficients.py` resolves every entry against its `EstimatorSpec`, holds +every point to the estimate the shipped coefficients give it, and every fitted value to the fit of its points. """ import functools from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from dataclasses import asdict, dataclass from importlib.resources import files from importlib.resources.abc import Traversable from types import MappingProxyType @@ -18,7 +19,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError, model_validator from invokeai.backend.working_memory.estimate import FeatureValue -from invokeai.backend.working_memory.platform import UNMEASURED_PRICED_AS, Platform +from invokeai.backend.working_memory.platform import Platform, priced_as if TYPE_CHECKING: from invokeai.backend.working_memory.spec import EstimatorSpec @@ -29,7 +30,8 @@ class _DataModel(BaseModel): class ValuesFrom(_DataModel): - """The entry whose values another entry takes: same platform, this estimator id and version.""" + """The entry whose values another entry takes: same platform, this estimator id and version. It may carry more + values than the borrower reads -- a family member that runs fewer dtypes reads fewer of them -- never fewer.""" estimator: str version: int @@ -44,7 +46,15 @@ class CoefficientEntry(_DataModel): values: dict[str, float] | None = None values_from: ValuesFrom | None = None """Borrow the values of another estimator's entry on the same platform, so that its measured points bound this - estimator too. One hop: the entry borrowed from carries its own values.""" + estimator too, and this estimator's points join the fit of those values. One hop: the entry borrowed from carries + its own values.""" + stand_ins: list[str] = [] + """Carried values that are declared conservative stand-ins rather than fits: what this platform prices a line no + in-fit point measures at. `provenance` says what they stand for; every other carried value is a fit.""" + unmeasured_floor: dict[str, float] = {} + """Declared lower bounds, not fits: what carried values are raised to where this entry prices a platform nothing + is measured on (see `priced_as`). A fit describes the platform its points were measured on, not the one it stands + in for. `provenance` says where the floors come from.""" provenance: str """Card, torch build, method and script the values come from, and which points they were fitted on.""" @@ -52,26 +62,43 @@ class CoefficientEntry(_DataModel): def _values_or_values_from(self) -> Self: if (self.values is None) == (self.values_from is None): raise ValueError("a coefficient entry carries exactly one of `values` and `values_from`") + for field, names in (("stand-ins", self.stand_ins), ("unmeasured floors", self.unmeasured_floor)): + unknown = sorted(set(names) - set(self.values or {})) + if unknown: + raise ValueError(f"{field} {unknown} are not values this entry carries") return self class MeasuredPoint(_DataModel): """One measured operation. The estimate for its features must be at least `measured_bytes`, and where a fitted - term decides it, at most 1.3 times that (or `max_ratio`).""" + term decides an in-fit point, at most 1.3 times that (or `max_ratio`). Measured with the model's weights already in + place, so `measured_bytes` is the operation's working memory alone.""" estimator: str version: int platform: dict[str, str] features: dict[str, FeatureValue] measured_bytes: int - method: Literal["reserved_peak"] - """``reserved_peak``: the caching allocator's peak reserved bytes over the operation, one fresh process per - point. Reserved rather than allocated, because reserved is what the model cache's reservation has to cover.""" + method: Literal["reserved_peak", "driver_peak", "demand_peak"] + """How `measured_bytes` was read, each as a peak over its value just before the operation, one fresh process per + point: + + - ``reserved_peak``: the caching allocator's reserved bytes. Reserved rather than allocated, because reserved is + what the model cache's reservation has to cover. + - ``driver_peak``: the process's device usage as the driver reports it (Windows ``D3DKMT`` ``CurrentUsage``), the + larger of two runs. + - ``demand_peak``: the largest of the driver-level, the reserved and the allocated peak, the larger of two runs. + The driver-level figure is what reserved misses: on HIP the first operation of a process maps ~300 MiB beyond + ``max_memory_reserved``.""" device: str provenance: str max_ratio: float | None = None """A tighter upper bound than the default 1.3, as a multiple of `measured_bytes`, for a point the estimate has to stay closer to. It applies whichever term decides the estimate.""" + in_fit: bool = True + """False for a point measured in a regime its platform does not run, kept to check another term: CUDA with the + math attention kernel forced, whose score matrix shares the forward with the convolutions. The fit leaves it out, + and it gets the lower bound only.""" class WorkingMemoryData(_DataModel): @@ -123,21 +150,30 @@ def _where(keys: Mapping[str, str]) -> str: return ",".join(f"{key}={value}" for key, value in keys.items()) -def _entry( - entries: list[CoefficientEntry], estimator: str, version: int, keys: Mapping[str, str], where: str +def find_entry( + entries: Iterable[CoefficientEntry], estimator: str, version: int, platform: Mapping[str, str], where: str ) -> CoefficientEntry: - candidates = [entry for entry in entries if entry.estimator == estimator and entry.platform == keys] + """The entry of `estimator` `version` that applies on `platform`: of those whose every platform key it matches, + the most specific one. An entry that also names the OS thereby takes precedence on that OS over the one that does + not (see `Platform`). Raises, naming `where`, if none applies or the most specific is declared twice.""" + candidates = [ + entry + for entry in entries + if entry.estimator == estimator and all(platform.get(key) == value for key, value in entry.platform.items()) + ] matches = [entry for entry in candidates if entry.version == version] - if len(matches) > 1: - raise LookupError( - f"Working-memory coefficients for estimator '{estimator}' version {version} on {where} are declared " - f"{len(matches)} times." - ) if not matches: found = f" (found only version {sorted({entry.version for entry in candidates})})" if candidates else "" raise LookupError( f"No working-memory coefficients for estimator '{estimator}' version {version} on {where}{found}." ) + most_specific = max(len(entry.platform) for entry in matches) + matches = [entry for entry in matches if len(entry.platform) == most_specific] + if len(matches) > 1: + raise LookupError( + f"Working-memory coefficients for estimator '{estimator}' version {version} on {where} are declared " + f"{len(matches)} times." + ) return matches[0] @@ -146,43 +182,51 @@ def resolve_coefficients( ) -> ResolvedCoefficients: """The values `spec` takes on `platform`, and a `source` that says how they were found. - A platform value nothing is measured for is priced as the measured value `UNMEASURED_PRICED_AS` names. The entry - for the spec's id, version and those platform values either carries the values or borrows them, one hop, from - another estimator's entry on the same platform. No other fallback: an estimate priced with coefficients fitted for - something else is the shortfall this data exists to prevent, so a missing, ambiguous, chained or mismatched entry - raises, naming what is wrong. `origin` prefixes the `source`. + A platform nothing is measured on is priced as the measured platform `priced_as` names. The entry for the spec's + id and version that applies there (`find_entry`) either carries the values or borrows them, one hop, from the + entry of another estimator that applies on the same platform. No other fallback: an estimate priced with + coefficients fitted for something else is the shortfall this data exists to prevent, so a missing, ambiguous, + chained or mismatched entry raises, naming what is wrong. `origin` prefixes the `source`. """ + measured = priced_as(platform) + full = {key: str(value) for key, value in asdict(measured).items()} + keys = {key: full[key] for key in spec.platform_keys} requested = {key: getattr(platform, key) for key in spec.platform_keys} - keys = {key: UNMEASURED_PRICED_AS.get(key, {}).get(value, value) for key, value in requested.items()} where = _where(keys) if keys == requested else f"{_where(keys)} (priced for {_where(requested)})" entries = [entry for file in data for entry in file.coefficients] - entry = _entry(entries, spec.id, spec.version, keys, where) - values, lender = entry.values, None + entry = find_entry(entries, spec.id, spec.version, full, where) + carrier, lender = entry, None if entry.values_from is not None: lender = entry.values_from.estimator - lent = _entry(entries, lender, entry.values_from.version, keys, where) - if lent.values is None: + carrier = find_entry(entries, lender, entry.values_from.version, full, where) + if carrier.values is None: raise LookupError( f"Working-memory coefficients for estimator '{spec.id}' on {where} borrow from '{lender}', whose " "entry borrows in turn; borrow from the entry that carries the values." ) - values = lent.values + values = carrier.values assert values is not None, "a validated entry carries values or values_from" - if set(values) != spec.coefficients: + # An entry's own values are exactly what its spec reads; a lender's may be more (see `ValuesFrom`). + matches = set(values) == spec.coefficients if lender is None else spec.coefficients <= set(values) + if not matches: raise LookupError( f"Working-memory coefficients for estimator '{spec.id}' on {where} provide {sorted(values)}" f"{f' (from {lender})' if lender else ''}; the estimator reads {sorted(spec.coefficients)}." ) - priced = ",".join(value for key, value in keys.items() if value != requested[key]) - if priced and lender: - note = f" ({priced} values of {lender})" - elif priced: - note = f" ({priced} values)" - else: - note = f" (values of {lender})" if lender else "" - return ResolvedCoefficients(values=MappingProxyType(dict(values)), source=f"{origin}:{_where(requested)}{note}") + # The source names the spec's keys, and a refining key (the OS) where an entry it read names one. + shown = [*spec.platform_keys, *(key for key in full if key in {*entry.platform, *carrier.platform})] + shown = list(dict.fromkeys(shown)) + read = {name: values[name] for name in sorted(spec.coefficients)} + notes = [f"values of {lender}"] if lender else [] + if measured != platform: + floors = {name: floor for name, floor in carrier.unmeasured_floor.items() if name in read} + read |= {name: max(read[name], floor) for name, floor in floors.items()} + notes.insert(0, f"priced as {_where({key: full[key] for key in shown})}{', floored' if floors else ''}") + note = f" ({', '.join(notes)})" if notes else "" + source = f"{origin}:{_where({key: str(getattr(platform, key)) for key in shown})}{note}" + return ResolvedCoefficients(values=MappingProxyType(read), source=source) @functools.cache diff --git a/invokeai/backend/working_memory/data/vae.json b/invokeai/backend/working_memory/data/vae.json index bae9081384f..c018e4cf8c5 100644 --- a/invokeai/backend/working_memory/data/vae.json +++ b/invokeai/backend/working_memory/data/vae.json @@ -3,131 +3,423 @@ "coefficients": [ { "estimator": "vae.sd", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." }, { - "estimator": "vae.sd", - "version": 1, + "estimator": "vae.sd3", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "values": { + "k_decode_fp16": 4597, + "b_decode_fp16": 83886080, + "k_decode_bf16": 4590, + "b_decode_bf16": 14680064, + "k_decode_fp32": 7702, + "b_decode_fp32": 0, + "k_encode_fp16": 2226, + "b_encode_fp16": 22020096, + "k_encode_bf16": 2249, + "b_encode_bf16": 18874368, + "k_encode_fp32": 3236, + "b_encode_fp32": 41943040 + }, + "unmeasured_floor": { + "k_decode_fp16": 4400, + "k_decode_bf16": 4400, + "k_decode_fp32": 8800, + "k_encode_fp16": 2200, + "k_encode_bf16": 2200, + "k_encode_fp32": 4400 + }, + "provenance": "The AutoencoderKL family's convolution lines (bytes per output pixel, and a base in bytes) per operation and dtype, carried by vae.sd3 as the one member that runs all three dtypes; vae.sd, vae.flux and vae.flux2 borrow the lines of theirs. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Fitted on the RTX 4090 points (torch 2.13 and, for the older FLUX.2 rows, 2.7.1) of the SDXL VAE (fp16, fp32), the FLUX.1 VAE (bf16, fp32) and the FLUX.2 VAE (bf16, fp32). Where this entry prices a platform nothing is measured on (MPS, XPU, CPU, or another allocator on this backend), each slope is floored at the figure these estimators priced every dtype at before lines per dtype: 2200 bytes per pixel per element byte for a decode, 1100 for an encode." + }, + { + "estimator": "vae.flux", + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "cudnn", + "allocator": "native" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." }, { - "estimator": "vae.sd3", - "version": 1, + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "values_from": { + "estimator": "vae.sd3", + "version": 2 + }, + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the FLUX.2 VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "values": { + "k_decode_bf16": 8696, + "b_decode_bf16": 66060288, + "k_encode_bf16": 2420, + "b_encode_bf16": 22020096 + }, + "unmeasured_floor": { + "k_decode_bf16": 4400, + "k_encode_bf16": 2200 + }, + "provenance": "The CogView4 VAE's own lines: an AutoencoderKL, but not the family's stack (block widths 128/512/1024/1024, three layers per block, no mid-block attention), and a decode costs ~1.9x the family's. bf16 only: its loader forces it. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Fitted on its RTX 4090 points (torch 2.13, native allocator). Where this entry prices a platform nothing is measured on (MPS, XPU, CPU, or another allocator on this backend), each slope is floored at the figure these estimators priced every dtype at before lines per dtype: 2200 bytes per pixel per element byte for a decode, 1100 for an encode." + }, + { + "estimator": "vae.sd", + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD3 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." }, { "estimator": "vae.sd3", - "version": 1, + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "values": { + "k_decode_fp16": 3797, + "b_decode_fp16": 0, + "k_decode_bf16": 3614, + "b_decode_bf16": 0, + "k_decode_fp32": 5964, + "b_decode_fp32": 0, + "k_encode_fp16": 1882, + "b_encode_fp16": 17825792, + "k_encode_bf16": 1882, + "b_encode_bf16": 17825792, + "k_encode_fp32": 2722, + "b_encode_fp32": 3145728 + }, + "provenance": "The AutoencoderKL family's convolution lines (bytes per output pixel, and a base in bytes) per operation and dtype, carried by vae.sd3 as the one member that runs all three dtypes; vae.sd, vae.flux and vae.flux2 borrow the lines of theirs. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Fitted on the RTX 4090 points (torch 2.13, backend:cudaMallocAsync) of the SDXL VAE (fp16, fp32), the FLUX.1 VAE (bf16, fp32) and the FLUX.2 VAE (bf16, fp32)." + }, + { + "estimator": "vae.flux", + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the SD3 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." }, { - "estimator": "vae.cogview4", - "version": 1, + "estimator": "vae.flux2", + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the CogView4 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the FLUX.2 VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." }, { "estimator": "vae.cogview4", - "version": 1, + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "values": { + "k_decode_bf16": 6765, + "b_decode_bf16": 0, + "k_encode_bf16": 1882, + "b_encode_bf16": 9437184 + }, + "provenance": "The CogView4 VAE's own lines: an AutoencoderKL, but not the family's stack (block widths 128/512/1024/1024, three layers per block, no mid-block attention), and a decode costs ~1.9x the family's. bf16 only: its loader forces it. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Fitted on its RTX 4090 points (torch 2.13, backend:cudaMallocAsync)." + }, + { + "estimator": "vae.sd", + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 + }, + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." + }, + { + "estimator": "vae.sd3", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native" }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the CogView4 VAE runs the same diffusers AutoencoderKL convolution stack. Not measured on this VAE." + "values": { + "k_decode_fp16": 7200, + "b_decode_fp16": 0, + "k_decode_bf16": 7143, + "b_decode_bf16": 68157440, + "k_decode_fp32": 14400, + "b_decode_fp32": 0, + "k_encode_fp16": 5500, + "b_encode_fp16": 0, + "k_encode_bf16": 5645, + "b_encode_bf16": 0, + "k_encode_fp32": 11000, + "b_encode_fp32": 0 + }, + "stand_ins": [ + "k_decode_fp16", + "b_decode_fp16", + "k_decode_fp32", + "b_decode_fp32", + "k_encode_fp16", + "b_encode_fp16", + "k_encode_fp32", + "b_encode_fp32" + ], + "unmeasured_floor": { + "k_decode_fp16": 7200, + "k_decode_bf16": 7200, + "k_decode_fp32": 14400, + "k_encode_fp16": 5500, + "k_encode_bf16": 5500, + "k_encode_fp32": 11000 + }, + "provenance": "The AutoencoderKL family's convolution lines (bytes per output pixel, and a base in bytes) per operation and dtype, carried by vae.sd3 as the one member that runs all three dtypes; vae.sd, vae.flux and vae.flux2 borrow the lines of theirs. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Every OS but Windows, which has its own entry. Fitted: bf16 on the FLUX.2 VAE points of Linux with the native allocator (RX 9070 XT gfx1201 and PRO W7900 gfx1100, torch 2.10). Stand-ins, no measurement here: fp16 and fp32 at 3600 decode / 2750 encode bytes per pixel per element byte, no base, the pre-dtype figure. Where this entry prices a platform nothing is measured on (MPS, XPU, CPU, or another allocator on this backend), each slope is floored at the figure these estimators priced every dtype at before lines per dtype: 3600 bytes per pixel per element byte for a decode, 2750 for an encode." }, { "estimator": "vae.flux", - "version": 1, + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native" + }, + "values_from": { + "estimator": "vae.sd3", + "version": 2 + }, + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native" + }, + "values_from": { + "estimator": "vae.sd3", + "version": 2 + }, + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the FLUX.2 VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native" + }, + "values": { + "k_decode_bf16": 13720, + "b_decode_bf16": 14680064, + "k_encode_bf16": 5500, + "b_encode_bf16": 161480704 + }, + "stand_ins": [ + "k_decode_bf16", + "b_decode_bf16", + "k_encode_bf16", + "b_encode_bf16" + ], + "unmeasured_floor": { + "k_decode_bf16": 7200, + "k_encode_bf16": 5500 + }, + "provenance": "The CogView4 VAE's own lines: an AutoencoderKL, but not the family's stack (block widths 128/512/1024/1024, three layers per block, no mid-block attention), and a decode costs ~1.9x the family's. bf16 only: its loader forces it. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Every OS but Windows, which has its own entry. Not measured here: both lines are stand-ins at its fitted MIOpen expandable-segments lines, as the family's bf16 lines measure the same under the native allocator on Linux and under expandable segments on Windows (7208 vs 7117 MiB decode at 1024px), with each slope raised to the pre-dtype figure where that is larger (the encode). Where this entry prices a platform nothing is measured on (MPS, XPU, CPU, or another allocator on this backend), each slope is floored at the figure these estimators priced every dtype at before lines per dtype: 3600 bytes per pixel per element byte for a decode, 2750 for an encode." + }, + { + "estimator": "vae.sd3", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "values": { + "k_decode_fp16": 3710, + "b_decode_fp16": 274726912, + "k_decode_bf16": 11009, + "b_decode_bf16": 0, + "k_decode_fp32": 10975, + "b_decode_fp32": 0, + "k_encode_fp16": 3703, + "b_encode_fp16": 77594624, + "k_encode_bf16": 4974, + "b_encode_bf16": 1104150528, + "k_encode_fp32": 4114, + "b_encode_fp32": 531628032 + }, + "unmeasured_floor": { + "k_decode_fp16": 7200, + "k_decode_bf16": 7200, + "k_decode_fp32": 14400, + "k_encode_fp16": 5500, + "k_encode_bf16": 5500, + "k_encode_fp32": 11000 + }, + "provenance": "The AutoencoderKL family's convolution lines (bytes per output pixel, and a base in bytes) per operation and dtype, carried by vae.sd3 as the one member that runs all three dtypes; vae.sd, vae.flux and vae.flux2 borrow the lines of theirs. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Windows: fitted on the RX 9060 XT (gfx1200, torch 2.13+rocm10) points of the SDXL VAE (fp16, fp32), the FLUX.1 VAE (bf16, fp32) and the FLUX.2 VAE (bf16, fp32) with expandable segments switched off. The demand is concave in the size, so the envelope over-reserves some points by more than 1.3x; the data test lists them. Where this entry prices a platform nothing is measured on (MPS, XPU, CPU, or another allocator on this backend), each slope is floored at the figure these estimators priced every dtype at before lines per dtype: 3600 bytes per pixel per element byte for a decode, 2750 for an encode." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "values": { + "k_decode_bf16": 17640, + "b_decode_bf16": 0, + "k_encode_bf16": 5060, + "b_encode_bf16": 1059061760 + }, + "unmeasured_floor": { + "k_decode_bf16": 7200, + "k_encode_bf16": 5500 + }, + "provenance": "The CogView4 VAE's own lines: an AutoencoderKL, but not the family's stack (block widths 128/512/1024/1024, three layers per block, no mid-block attention), and a decode costs ~1.9x the family's. bf16 only: its loader forces it. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Windows: fitted on its RX 9060 XT (gfx1200, torch 2.13+rocm10) points with expandable segments switched off. Where this entry prices a platform nothing is measured on (MPS, XPU, CPU, or another allocator on this backend), each slope is floored at the figure these estimators priced every dtype at before lines per dtype: 3600 bytes per pixel per element byte for a decode, 2750 for an encode." + }, + { + "estimator": "vae.sd", + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 + }, + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the SD1/SDXL VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." + }, + { + "estimator": "vae.sd3", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "values": { + "k_decode_fp16": 2016, + "b_decode_fp16": 506462208, + "k_decode_bf16": 6612, + "b_decode_bf16": 529530880, + "k_decode_fp32": 5988, + "b_decode_fp32": 80740352, + "k_encode_fp16": 1960, + "b_encode_fp16": 353370112, + "k_encode_bf16": 3255, + "b_encode_bf16": 483393536, + "k_encode_fp32": 3898, + "b_encode_fp32": 128974848 }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. The FLUX.1 point below falls inside them." + "provenance": "The AutoencoderKL family's convolution lines (bytes per output pixel, and a base in bytes) per operation and dtype, carried by vae.sd3 as the one member that runs all three dtypes; vae.sd, vae.flux and vae.flux2 borrow the lines of theirs. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Windows only, where expandable segments are Invoke's ROCm default: measured on gfx1200 alone. Elsewhere they are a user override nothing is measured for, priced as MIOpen with the native allocator on that OS. Fitted on the RX 9060 XT (gfx1200, torch 2.13+rocm10) points of the SDXL VAE (fp16, fp32) and the FLUX.1 VAE (bf16, fp32). vae.flux2 carries its own lines here: its fp32 decode differs from the family's by more than the margin." }, { "estimator": "vae.flux", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" }, "values_from": { - "estimator": "vae.flux2", - "version": 1 + "estimator": "vae.sd3", + "version": 2 }, - "provenance": "Borrows the vae.flux2 values, so its points bound this estimator too: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. Confirmed on the FLUX.1 VAE: RX 9060 XT (gfx1200), torch 2.12, Windows, bf16, 768-1536px, Invoke's ROCm SDPA guard installed, MIOPEN_FIND_MODE=FAST: implied 3451 decode / 2688 encode." + "provenance": "Borrows the lines of its dtypes from vae.sd3, so its points bound them and join their fit: the FLUX.1 VAE runs the same diffusers AutoencoderKL convolution stack. On Windows with MIOpen's native allocator it borrows vae.sd3's Windows entry." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" }, "values": { - "k_decode": 2200, - "k_encode": 1100 + "k_decode_bf16": 6612, + "b_decode_bf16": 529530880, + "k_decode_fp32": 5842, + "b_decode_fp32": 272629760, + "k_encode_bf16": 3255, + "b_encode_bf16": 483393536, + "k_encode_fp32": 3898, + "b_encode_fp32": 128974848 }, - "provenance": "Peak reserved bytes per output pixel per element byte of the diffusers AutoencoderKL convolution stack. Fitted on the FLUX.2 VAE (the vae.flux2 cudnn points) with scripts/calibrate_flux2_working_memory.py --only vae: RTX 4090, torch 2.7.1, bf16, untiled, fused SDPA; implied 2180/2185/2165 decode and 1072/1063/1061 encode at 512/768/1024px, flat across resolution. Rounded up to 2200 / 1100, the pair the FLUX.1 estimator already shipped." + "provenance": "The FLUX.2 VAE's own convolution lines under MIOpen with expandable segments. Windows only, where expandable segments are Invoke's ROCm default: measured on gfx1200 alone. Elsewhere they are a user override nothing is measured for, priced as MIOpen with the native allocator on that OS. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Fitted on its RX 9060 XT (gfx1200, torch 2.13+rocm10) points. Not borrowed from vae.sd3: its fp32 decode needs ~140 MiB more at 512px than the SDXL and FLUX.1 VAEs, which would lift the family's fp32 decode line 9% above theirs, past the 5% margin. Its bf16 lines and fp32 encode equal the family's." }, { - "estimator": "vae.flux2", - "version": 1, + "estimator": "vae.cogview4", + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" }, "values": { - "k_decode": 3600, - "k_encode": 2750 + "k_decode_bf16": 13720, + "b_decode_bf16": 14680064, + "k_encode_bf16": 3764, + "b_encode_bf16": 161480704 }, - "provenance": "Peak reserved bytes per output pixel per element byte of the diffusers AutoencoderKL convolution stack on MIOpen. Fitted on the FLUX.2 VAE (the vae.flux2 miopen points) with scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled: RX 9070 XT (gfx1201) implied 3453/3369/3368 decode, PRO W7900 (gfx1100) 3525 at 512px; encode 2687/2688/2688 on both cards, to the byte. Two AMD generations agreeing that closely make this MIOpen rather than a per-card quirk, so the values are the larger card's plus ~2% headroom: 3600 / 2750. MIOpen's convolution workspaces are larger than cuDNN's (~1.6x decode, ~2.4x encode), identically with fused and math attention, and its encode/decode ratio is 0.78 against cuDNN's 0.49." + "provenance": "The CogView4 VAE's own lines: an AutoencoderKL, but not the family's stack (block widths 128/512/1024/1024, three layers per block, no mid-block attention), and a decode costs ~1.9x the family's. bf16 only: its loader forces it. Fitted lines are working_memory/fit.py's: the upper envelope of the untiled single-sample in-fit points of every estimator resolving to this entry, plus 5%; tiled points only bound their estimates. Windows only, where expandable segments are Invoke's ROCm default: measured on gfx1200 alone. Elsewhere they are a user override nothing is measured for, priced as MIOpen with the native allocator on that OS. Fitted on its RX 9060 XT (gfx1200, torch 2.13+rocm10) points." } ], "points": [ { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -135,19 +427,20 @@ "width": 512, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 1142461301, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -155,19 +448,20 @@ "width": 768, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 2576980378, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -175,19 +469,20 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 4540854174, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -195,19 +490,20 @@ "width": 512, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 561566974, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -215,19 +511,20 @@ "width": 768, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 1254130450, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -235,19 +532,20 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 2224793059, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.7.1, bf16, untiled, fused SDPA (score matrix not materialized). Recorded in GiB to three decimals. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -255,19 +553,20 @@ "width": 1280, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 7075958620, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, bf16, untiled, fused SDPA: the fused column of the max-versus-sum measurement (#9519). Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, bf16, untiled, fused SDPA: the fused column of the max-versus-sum measurement (#9519). Recorded in GiB to three decimals. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -275,19 +574,20 @@ "width": 512, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 1138753536, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -295,19 +595,20 @@ "width": 768, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 2531262464, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -315,19 +616,20 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 4466933760, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -335,19 +637,20 @@ "width": 1328, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 7493124096, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -355,19 +658,20 @@ "width": 1536, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 10043260928, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -375,19 +679,20 @@ "width": 1328, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 3168796672, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement." + "provenance": "First FLUX.2 VAE calibration (#9519): CUDA, bf16, untiled, fused SDPA. Recorded in MiB. The decode rows are a separate run from the 512-1024px GiB rows; the 1536px decode is also the fused 9.353 GiB of the max-versus-sum measurement. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -395,19 +700,21 @@ "width": 512, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 285212672 + "score_matrix_bytes": 285212672, + "dtype": "bf16" }, "measured_bytes": 561566974, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB. Standalone script with no allocator configuration: the native allocator.", + "in_fit": false }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -415,19 +722,21 @@ "width": 768, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 1443889152 + "score_matrix_bytes": 1443889152, + "dtype": "bf16" }, "measured_bytes": 1933809025, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB. Standalone script with no allocator configuration: the native allocator.", + "in_fit": false }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -435,19 +744,21 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 4563402752 + "score_matrix_bytes": 4563402752, + "dtype": "bf16" }, "measured_bytes": 4372276707, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB. Standalone script with no allocator configuration: the native allocator.", + "in_fit": false }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -455,19 +766,21 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 4563402752 + "score_matrix_bytes": 4563402752, + "dtype": "bf16" }, "measured_bytes": 3787087413, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB. Standalone script with no allocator configuration: the native allocator.", + "in_fit": false }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -475,19 +788,21 @@ "width": 1280, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 11141120000 + "score_matrix_bytes": 11141120000, + "dtype": "bf16" }, "measured_bytes": 6360846565, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB. Standalone script with no allocator configuration: the native allocator.", + "in_fit": false }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -495,19 +810,21 @@ "width": 1536, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 23102226432 + "score_matrix_bytes": 23102226432, + "dtype": "bf16" }, "measured_bytes": 12847320924, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae with SDPBackend.MATH forced on CUDA -- the regime MPS lands in; bf16, untiled; score_matrix_bytes is the full math-kernel score matrix. Forcing math measures below the fused path until 1536px. The 768px encode sits at the crossover, where the larger term alone is short and the device_working_mem_gb floor covers it; reproducible to three decimals across runs. Recorded in GiB. Standalone script with no allocator configuration: the native allocator.", + "in_fit": false }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "decode", @@ -515,19 +832,20 @@ "width": 512, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 285212672 + "score_matrix_bytes": 285212672, + "dtype": "bf16" }, "measured_bytes": 1810328715, "method": "reserved_peak", "device": "RX 9070 XT (gfx1201)", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "decode", @@ -535,19 +853,20 @@ "width": 768, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 1443889152 + "score_matrix_bytes": 1443889152, + "dtype": "bf16" }, "measured_bytes": 3973918491, "method": "reserved_peak", "device": "RX 9070 XT (gfx1201)", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "decode", @@ -555,19 +874,20 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 4563402752 + "score_matrix_bytes": 4563402752, + "dtype": "bf16" }, "measured_bytes": 7063073718, "method": "reserved_peak", "device": "RX 9070 XT (gfx1201)", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "decode", @@ -575,20 +895,21 @@ "width": 512, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 285212672 + "score_matrix_bytes": 285212672, + "dtype": "bf16" }, "measured_bytes": 1847909679, "method": "reserved_peak", "device": "PRO W7900 (gfx1100)", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals.", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it.", "max_ratio": 1.15 }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "decode", @@ -596,20 +917,21 @@ "width": 768, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 1443889152 + "score_matrix_bytes": 1443889152, + "dtype": "bf16" }, "measured_bytes": 4059817837, "method": "reserved_peak", "device": "PRO W7900 (gfx1100)", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals.", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it.", "max_ratio": 1.15 }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "decode", @@ -617,20 +939,21 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 4563402752 + "score_matrix_bytes": 4563402752, + "dtype": "bf16" }, "measured_bytes": 7197291446, "method": "reserved_peak", "device": "PRO W7900 (gfx1100)", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals.", + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it.", "max_ratio": 1.15 }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "encode", @@ -638,19 +961,20 @@ "width": 512, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 285212672 + "score_matrix_bytes": 285212672, + "dtype": "bf16" }, "measured_bytes": 1408749273, "method": "reserved_peak", "device": "RX 9070 XT (gfx1201) and PRO W7900 (gfx1100), identical to the byte", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "encode", @@ -658,19 +982,20 @@ "width": 768, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 1443889152 + "score_matrix_bytes": 1443889152, + "dtype": "bf16" }, "measured_bytes": 3170759606, "method": "reserved_peak", "device": "RX 9070 XT (gfx1201) and PRO W7900 (gfx1100), identical to the byte", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "miopen" + "conv_backend": "miopen", + "allocator": "native" }, "features": { "operation": "encode", @@ -678,19 +1003,20 @@ "width": 1024, "tile": null, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 4563402752 + "score_matrix_bytes": 4563402752, + "dtype": "bf16" }, "measured_bytes": 5637144576, "method": "reserved_peak", "device": "RX 9070 XT (gfx1201) and PRO W7900 (gfx1100), identical to the byte", - "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals." + "provenance": "scripts/calibrate_flux2_working_memory.py --only vae, torch 2.10, bf16, untiled; a clean run (no MIOPEN_FIND_MODE override, no allocator GC threshold). score_matrix_bytes is the math-kernel score matrix the ROCm head-dim guard prices for the 512-wide head, unchunked; on these cards forcing math moved the peak by nothing up to 1024px. Recorded in GiB to three decimals. Linux, no allocator configuration: the native allocator, as the script ran it." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -698,19 +1024,20 @@ "width": 1024, "tile": null, "batch_size": 2, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 8541616210, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "#9519: 1024px decode by latent batch, CUDA, bf16, fused SDPA, one fresh process per point: 4.23 / 7.96 / 11.89 GiB at batch 1 / 2 / 3. The test table wrote them as GiB x 1000 (4229, 7955, 11889); batch 1 is the 4.229 GiB 1024px decode above." + "provenance": "#9519: 1024px decode by latent batch, CUDA, bf16, fused SDPA, one fresh process per point: 4.23 / 7.96 / 11.89 GiB at batch 1 / 2 / 3. The test table wrote them as GiB x 1000 (4229, 7955, 11889); batch 1 is the 4.229 GiB 1024px decode above. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "decode", @@ -718,19 +1045,20 @@ "width": 1024, "tile": null, "batch_size": 3, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 12765716546, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "#9519: 1024px decode by latent batch, CUDA, bf16, fused SDPA, one fresh process per point: 4.23 / 7.96 / 11.89 GiB at batch 1 / 2 / 3. The test table wrote them as GiB x 1000 (4229, 7955, 11889); batch 1 is the 4.229 GiB 1024px decode above." + "provenance": "#9519: 1024px decode by latent batch, CUDA, bf16, fused SDPA, one fresh process per point: 4.23 / 7.96 / 11.89 GiB at batch 1 / 2 / 3. The test table wrote them as GiB x 1000 (4229, 7955, 11889); batch 1 is the 4.229 GiB 1024px decode above. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux2", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", @@ -738,32 +1066,5247 @@ "width": 2024, "tile": 512, "batch_size": 1, - "element_size": 2, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 585105408, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "#9519: reference-image encode at the 2024px reference cap with the forced 512px tile, CUDA, bf16, fused SDPA; flat at ~0.55 GB from 1024px up. Recorded in MiB." + "provenance": "#9519: reference-image encode at the 2024px reference cap with the forced 512px tile, CUDA, bf16, fused SDPA; flat at ~0.55 GB from 1024px up. Recorded in MiB. Standalone script with no allocator configuration: the native allocator." }, { "estimator": "vae.flux", - "version": 1, + "version": 2, "platform": { - "conv_backend": "cudnn" + "conv_backend": "cudnn", + "allocator": "native" }, "features": { "operation": "encode", "height": 3072, "width": 3072, - "element_size": 2, "tile": null, - "score_matrix_bytes": 0 + "score_matrix_bytes": 0, + "dtype": "bf16" }, "measured_bytes": 20229295964, "method": "reserved_peak", "device": "RTX 4090", - "provenance": "Commit 326bdf1bd0: the FLUX.1 VAE as a diffusers AutoencoderKL, bf16, fused SDPA, untiled 3072px encode: 18.84 GiB. Re-measured 18.57 GiB on torch 2.13+cu130, 2026-10-17, with scripts/calibrate_flux_vae_working_memory.py." + "provenance": "Commit 326bdf1bd0: the FLUX.1 VAE as a diffusers AutoencoderKL, bf16, fused SDPA, untiled 3072px encode: 18.84 GiB. Re-measured 18.57 GiB on torch 2.13+cu130, 2026-10-17, with scripts/calibrate_flux_vae_working_memory.py. Standalone script with no allocator configuration: the native allocator." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1488977920, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3334471680, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5817499648, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 9105833984, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1744830464, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4326424576, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7572815872, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 11741954048, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 891289600, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2132803584, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3709861888, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5689573376, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1226833920, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2661285888, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4622123008, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7157579776, + "method": "driver_peak", + "device": "RTX 4090", + "provenance": "Isolated SDXL VAE decode (diffusers AutoencoderKL, force_upcast; random latents as l2i), one fresh process per size, the larger of two decodes: RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, fused SDPA, native allocator. Driver-level usage agreed with max_memory_reserved within 100 MiB. A game held ~3 GB on the card meanwhile; no paging measured, allocated identical. bf16 measured identical to fp16 (not shipped: this node never runs bf16). Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1140850688, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4538236928, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 10158604288, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 2048, + "width": 2048, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 18029215744, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 562036736, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2225078272, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4989124608, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 2048, + "width": 2048, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 8864661504, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 3072, + "width": 3072, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 19937624064, + "method": "reserved_peak", + "device": "RTX 4090", + "provenance": "scripts/calibrate_flux_vae_working_memory.py (2026-10-17): the FLUX.1 ae.safetensors as a diffusers AutoencoderKL, RTX 4090, torch 2.13.0+cu130, Windows, bf16, fused SDPA, untiled, native allocator (no allocator configuration). The untiled 3072px decode ran into the card's limit and is not shipped." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 985661440, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 1614807040, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2495610880, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3271557120, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 825229312, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 1433403392, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2293235712, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3040870400, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1570766848, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3355443200, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5997854720, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 9418309632, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1095761920, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2312110080, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3900702720, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6083837952, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2154823680, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3984588800, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6983516160, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 10821304320, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1197473792, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2287992832, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3546284032, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5538578432, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1570766848, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3355443200, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5997854720, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 9418309632, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1095761920, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2312110080, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3900702720, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6083837952, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2154823680, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3984588800, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6983516160, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 10821304320, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1197473792, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2287992832, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3548381184, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5538578432, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1717567488, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3376414720, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6018826240, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 9374269440, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1095761920, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2312110080, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3900702720, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6081740800, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1186988032, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2344615936, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3212836864, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 4796186624, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 998244352, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2153775104, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2526019584, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3462397952, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2686451712, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6155141120, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 10959716352, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 16733175808, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1533018112, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2495610880, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 4433379328, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6924795904, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2176843776, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5135925248, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 10993270784, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 16787701760, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2293235712, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3206545408, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5639241728, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 8812232704, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2684354560, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6155141120, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 10959716352, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 16733175808, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1533018112, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2495610880, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 4433379328, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6922698752, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2178940928, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5135925248, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 10993270784, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 16789798912, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2293235712, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 3206545408, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 5641338880, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 8810135552, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 2684354560, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6155141120, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 10959716352, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 16689135616, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 285212672 + }, + "measured_bytes": 1533018112, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 2493513728, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 4431282176, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 1073741824 + }, + "measured_bytes": 6922698752, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1157627904, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2592079872, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4552916992, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 10173284352, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 576716800, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1268776960, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2241855488, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5022679040, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1675624448, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4257218560, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7503609856, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 16770924544, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 847249408, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1843396608, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3261071360, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7308574720, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1157627904, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2592079872, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4555014144, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 10175381504, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 576716800, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1270874112, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2239758336, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5005901824, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1675624448, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4259315712, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7501512704, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 16770924544, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 845152256, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1843396608, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3261071360, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7306477568, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1159725056, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2592079872, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4555014144, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 10057940992, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 578813952, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1270874112, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2241855488, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5005901824, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1675624448, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4257218560, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7501512704, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 16791896064, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 847249408, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1841299456, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3258974208, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 7306477568, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 822083584, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2063597568, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3640655872, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 8069840896, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 486539264, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1056964608, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1895825408, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp16", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4244635648, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1442840576, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3265265664, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5748293632, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 12996050944, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 681574400, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1522532352, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2694840320, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.sd", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "vae_class": "AutoencoderKL", + "upcast_weight_bytes": 0, + "score_matrix_bytes": 0 + }, + "measured_bytes": 6083837952, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init SDXL VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 788529152, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2030043136, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3607101440, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 8036286464, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 486539264, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1056964608, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1895825408, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4244635648, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1442840576, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3265265664, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5714739200, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 12962496512, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 681574400, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1520435200, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2694840320, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "score_matrix_bytes": 0 + }, + "measured_bytes": 6083837952, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.1 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 788529152, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2030043136, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3607101440, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 8069840896, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 486539264, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1073741824, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1895825408, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "bf16", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 4244635648, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1409286144, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 3265265664, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 5781848064, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 12962496512, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 681574400, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 1520435200, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 2694840320, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.flux2", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "fp32", + "tile": null, + "batch_size": 1, + "score_matrix_bytes": 0 + }, + "measured_bytes": 6117392384, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init FLUX.2 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 2233466880, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 4947181568, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 8690597888, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "bf16" + }, + "measured_bytes": 19386073088, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 624951296, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 1379926016, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 2436890624, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "native" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "bf16" + }, + "measured_bytes": 5454692352, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, native allocator, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 1644167168, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 3800039424, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 6685720576, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "decode", + "height": 1536, + "width": 1536, + "dtype": "bf16" + }, + "measured_bytes": 14738784256, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 478150656, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 1048576000, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 1887436800, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "cudnn", + "allocator": "cudaMallocAsync" + }, + "features": { + "operation": "encode", + "height": 1536, + "width": 1536, + "dtype": "bf16" + }, + "measured_bytes": 4236247040, + "method": "demand_peak", + "device": "RTX 4090", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RTX 4090, torch 2.13.0+cu130, Windows, driver 32.0.16.1047, PYTORCH_CUDA_ALLOC_CONF=backend:cudaMallocAsync, fused SDPA. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 3439329280, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (10040 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 7717519360, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (10040 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 13715374080, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (10040 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 1092616192, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (10040 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 2267021312, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (10040 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 3755999232, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (10040 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "expandable", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "bf16" + }, + "measured_bytes": 5874122752, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True (torch reports the native backend), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (10040 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 4399824896, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (16650 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 9909043200, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (16650 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "decode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 17616076800, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (16650 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 512, + "width": 512, + "dtype": "bf16" + }, + "measured_bytes": 2271215616, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (16650 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 768, + "width": 768, + "dtype": "bf16" + }, + "measured_bytes": 3206545408, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (16650 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1024, + "width": 1024, + "dtype": "bf16" + }, + "measured_bytes": 5695864832, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (16650 MiB)." + }, + { + "estimator": "vae.cogview4", + "version": 2, + "platform": { + "conv_backend": "miopen", + "allocator": "native", + "os": "windows" + }, + "features": { + "operation": "encode", + "height": 1280, + "width": 1280, + "dtype": "bf16" + }, + "measured_bytes": 8902410240, + "method": "demand_peak", + "device": "RX 9060 XT (gfx1200)", + "provenance": "Scratch probe vae_dtype_probe.py (2026-10-17): RX 9060 XT (gfx1200), torch 2.13.0+rocm10.0.0, Windows, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:False (the native allocator), MIOPEN_FIND_MODE=FAST, Invoke's ROCm SDPA guard installed, so score_matrix_bytes is its chunked math term. Windows-only: the native allocator's demand there is concave (first-run driver spikes at small sizes, a second run reserving up to 1.6x the first at large ones) and up to 1.6x Linux's. Random-init CogView4 VAE from the released config (memory does not depend on weight values), untiled, one fresh process per point, two runs, no allocator retries. Recorded in MiB. The 1280px decode is not shipped: it needs ~20 GB on this 16 GB card, which Windows pages into system memory instead of failing, so it read low (16650 MiB)." } ] } diff --git a/invokeai/backend/working_memory/fit.py b/invokeai/backend/working_memory/fit.py new file mode 100644 index 00000000000..94ac5da208c --- /dev/null +++ b/invokeai/backend/working_memory/fit.py @@ -0,0 +1,142 @@ +"""The shipped fit: the values under `data/` reproduced from their measured points. + +The AutoencoderKL estimators price their convolutions as a line in the output's pixel count, one per operation and +dtype: `k__` bytes per pixel plus `b__` bytes, the base a process pays whatever the +size. Each carried pair is the upper envelope of the points it is resolved for (see `upper_envelope`), plus +`SHIPPED_MARGIN`. A point joins the fit of the entry its estimator resolves to on its platform -- the lender's, where +the estimator borrows -- when it is a single-sample, untiled operation of that pair (`in_fit` unless measured out of +regime). Tiled points bound their estimates but fit nothing: the tile term reuses the untiled pair. + +`tests/backend/working_memory/test_shipped_coefficients.py` holds every carried value that is not a declared stand-in +to `fitted_values` of the shipped points, so a new point that moves a fit fails there with the value to ship. +""" + +import math +from collections.abc import Iterable, Mapping, Sequence +from fractions import Fraction +from typing import Final + +from invokeai.backend.working_memory.coefficients import CoefficientEntry, MeasuredPoint, WorkingMemoryData, find_entry + +SHIPPED_MARGIN: Final = Fraction(1, 20) +"""What the shipped slope and intercept add over the envelope of their points: 5%, for the runs and cards the points +do not cover.""" + +MIN_AREA_RATIO: Final = 2 +"""How far apart the sizes a line is fitted on must lie, as a ratio of pixel counts: closer points pin the intercept +no better than their noise.""" + +_MIB: Final = 2**20 + +EntryKey = tuple[str, int, tuple[tuple[str, str], ...]] +"""An entry's estimator id, version and platform keys: what a point is resolved by.""" + + +def entry_key(entry: CoefficientEntry | MeasuredPoint) -> EntryKey: + return (entry.estimator, entry.version, tuple(sorted(entry.platform.items()))) + + +def upper_envelope(points: Sequence[tuple[int, int]]) -> tuple[Fraction, Fraction]: + """The line ``y = k * x + b`` on or above every ``(x, y)`` point, with ``b >= 0``, that over-reserves least. + + The candidates are the edges of the upper convex hull of the points and the origin: each such line lies on or + above every point, and the origin keeps its intercept non-negative. Of those, the one whose largest ratio to a + point is smallest wins, the steeper one on a tie, since it is the more conservative past the largest point. + Raises unless the points span two sizes at least `MIN_AREA_RATIO` apart. + """ + if not points or any(x <= 0 or y <= 0 for x, y in points): + raise ValueError(f"an envelope needs points with positive sizes and bytes, got {list(points)}") + sizes = sorted({x for x, _ in points}) + if len(sizes) < 2 or sizes[-1] < MIN_AREA_RATIO * sizes[0]: + raise ValueError( + f"an envelope needs two sizes at least {MIN_AREA_RATIO}x apart in pixels, got {sizes}; measure more sizes" + ) + + highest = {x: max(y for px, y in points if px == x) for x in sizes} + hull: list[tuple[int, int]] = [(0, 0)] + for point in highest.items(): + # Drop the last vertex while it is not above the chord from the one before it to this point. + while len(hull) >= 2: + (x0, y0), (x1, y1) = hull[-2], hull[-1] + if (x1 - x0) * (point[1] - y0) - (y1 - y0) * (point[0] - x0) < 0: + break + hull.pop() + hull.append(point) + + best: tuple[Fraction, Fraction, Fraction] | None = None + for (x0, y0), (x1, y1) in zip(hull, hull[1:], strict=False): + k = Fraction(y1 - y0, x1 - x0) + b = y0 - k * x0 + if k < 0: + continue + worst = max((k * x + b) / y for x, y in points) + if best is None or (worst, -k) < (best[0], -best[1]): + best = (worst, k, b) + assert best is not None, "the edge from the origin always rises" + return best[1], best[2] + + +def shipped_pair(points: Sequence[tuple[int, int]]) -> tuple[int, int]: + """The slope and intercept to ship for these points: the envelope plus `SHIPPED_MARGIN`, the slope rounded up to + a whole byte per pixel and the intercept to a whole MiB.""" + k, b = upper_envelope(points) + margin = 1 + SHIPPED_MARGIN + return math.ceil(k * margin), math.ceil(b * margin / _MIB) * _MIB + + +def _pairs(entry: CoefficientEntry) -> set[str]: + """The `_` suffixes an entry carries both halves of a convolution line for.""" + names = set(entry.values or {}) + return {name[2:] for name in names if name.startswith("k_") and f"b_{name[2:]}" in names} + + +def fit_points(data: Iterable[WorkingMemoryData]) -> Mapping[tuple[EntryKey, str], list[tuple[int, int]]]: + """Per carried entry and line, the ``(pixels, bytes)`` points that line is fitted on. + + A point belongs to the entry its own estimator's entry resolves to on its platform, one `values_from` hop, and to + the line its operation and dtype name. Lines no point measures are absent. + """ + files = list(data) + entries = [entry for file in files for entry in file.coefficients] + groups: dict[tuple[EntryKey, str], list[tuple[int, int]]] = {} + for point in (point for file in files for point in file.points): + features = point.features + if not point.in_fit or features.get("tile") is not None or features.get("batch_size", 1) != 1: + continue + owner = carrier_of(entries, point) + suffix = f"{features.get('operation')}_{features.get('dtype')}" + if suffix in _pairs(owner): + pixels = int(features["height"]) * int(features["width"]) # type: ignore[arg-type] + groups.setdefault((entry_key(owner), suffix), []).append((pixels, point.measured_bytes)) + return groups + + +def carrier_of(entries: Sequence[CoefficientEntry], point: MeasuredPoint) -> CoefficientEntry: + """The entry that carries the values `point` is evaluated with: its estimator's entry on its platform, or the one + that entry borrows from. A point names the OS only where it measured an OS-specific entry's platform.""" + where = f"{point.estimator} point on {point.platform}" + entry = find_entry(entries, point.estimator, point.version, point.platform, where) + if entry.values_from is None: + return entry + return find_entry(entries, entry.values_from.estimator, entry.values_from.version, point.platform, where) + + +def fitted_values(data: Iterable[WorkingMemoryData]) -> Mapping[EntryKey, Mapping[str, int]]: + """Per entry that carries values, what `shipped_pair` gives each line it does not declare a stand-in. + + Raises, naming the entry and line, where the points cannot fit that line. + """ + files = list(data) + groups = fit_points(files) + fitted: dict[EntryKey, dict[str, int]] = {} + for entry in (entry for file in files for entry in file.coefficients if entry.values is not None): + key = entry_key(entry) + for suffix in sorted(_pairs(entry)): + if {f"k_{suffix}", f"b_{suffix}"} <= set(entry.stand_ins): + continue + try: + k, b = shipped_pair(groups.get((key, suffix), [])) + except ValueError as e: + raise ValueError(f"{entry.estimator} on {entry.platform}, {suffix}: {e}") from e + fitted.setdefault(key, {}).update({f"k_{suffix}": k, f"b_{suffix}": b}) + return fitted diff --git a/invokeai/backend/working_memory/platform.py b/invokeai/backend/working_memory/platform.py index 5dfc2a00997..8ceafbd9626 100644 --- a/invokeai/backend/working_memory/platform.py +++ b/invokeai/backend/working_memory/platform.py @@ -1,32 +1,90 @@ -from collections.abc import Mapping -from dataclasses import dataclass -from types import MappingProxyType +import sys +from dataclasses import astuple, dataclass, replace from typing import Final, Literal import torch +from invokeai.app.util.torch_cuda_allocator import expandable_segments_enabled + ConvBackend = Literal["cudnn", "miopen", "mps", "xpu", "cpu"] +Allocator = Literal["native", "cudaMallocAsync", "expandable"] +"""torch's caching allocator: the native one, the native one in expandable-segments mode, or the driver's +``cudaMallocAsync`` pool (``hipMallocAsync`` on a HIP build, which torch reports under the same name).""" +OperatingSystem = Literal["windows", "linux", "other"] @dataclass(frozen=True) class Platform: - """The traits of the machine an operation runs on, as coefficients are keyed. Each field is a platform key an - `EstimatorSpec` can name, and holds the true value: what was measured, and what was not, is `UNMEASURED_PRICED_AS`'s - business.""" + """The traits of the machine an operation runs on, as coefficients are keyed. Each field holds the true value: + what was measured, and what was not, is `priced_as`'s business. + + `conv_backend` and `allocator` are the keys an `EstimatorSpec` names. `os` refines them only where an entry says + so: an entry whose platform names an `os` applies there and takes precedence over the entry that does not, so the + operating system enters the pricing of exactly the platforms measured on one OS or measured to differ by it, and + of no other. Both are MIOpen: its native allocator demands up to 1.6x more on Windows than on Linux, and expandable + segments, Invoke's Windows default, are measured only on Windows. + """ conv_backend: ConvBackend + allocator: Allocator + os: OperatingSystem -UNMEASURED_PRICED_AS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( +MEASURED: Final = frozenset( { - # MPS, XPU and CPU convolutions are unmeasured and take the cuDNN coefficients. On MPS that pairs with a - # score-matrix term the SDPA probe always charges there, so the total is not thin. - "conv_backend": MappingProxyType({"mps": "cudnn", "xpu": "cudnn", "cpu": "cudnn"}), + ("cudnn", "native", None), + ("cudnn", "cudaMallocAsync", None), + ("miopen", "native", None), + ("miopen", "expandable", "windows"), } ) -"""Per platform key, the values no coefficients are measured for, and the measured value each is priced as. The one -place that borrowing is declared: the resolver applies it and names it in an estimate's `source`, and shipped data -carries entries for measured values only.""" +"""The (convolution backend, allocator, OS) combinations coefficients are measured on, None for every OS. Shipped data +carries entries for these only.""" + + +def is_measured(platform: Platform) -> bool: + return {(platform.conv_backend, platform.allocator, None), astuple(platform)} & MEASURED != set() + + +def priced_as(platform: Platform) -> Platform: + """The measured platform an operation on `platform` is priced as: `platform` itself where it is measured. The OS + stays: it only selects among the entries of the measured pair. + + The one place that borrowing is declared; the resolver applies it and names it in an estimate's `source`. + + - A convolution backend nothing is measured on (MPS, XPU, CPU) takes cuDNN under the native allocator. On MPS that + pairs with a score-matrix term the SDPA probe always charges there, so the total is not thin. + - An allocator not measured with its backend on this OS takes that backend's native allocator: cuDNN in + expandable-segments mode, MIOpen under ``hipMallocAsync``, and MIOpen with expandable segments off Windows (a + user override there; Linux defaults to native). On cuDNN, native is the larger of the two measured allocators + (``cudaMallocAsync`` needs 25-35% less); on MIOpen, native is the larger too, on Windows by up to 1.6x. + """ + if is_measured(platform): + return platform + if platform.conv_backend in ("mps", "xpu", "cpu"): + return replace(platform, conv_backend="cudnn", allocator="native") + return replace(platform, allocator="native") + + +def _allocator_for(device: torch.device) -> Allocator: + """The caching allocator an operation on `device` allocates through: `native` for anything but a CUDA/HIP GPU. + + The backend is fixed before torch initializes CUDA, so torch's answer holds for the process; it is asked first, + because the driver's pool ignores expandable segments. Expandable segments are a mode of the native allocator, read + from the environment the model cache reads them from. A ``cuda`` device on a build without one (a test pinning the + device) has no backend to ask. + """ + if device.type != "cuda": + return "native" + if torch.cuda.is_available() and torch.cuda.get_allocator_backend() == "cudaMallocAsync": + return "cudaMallocAsync" + return "expandable" if expandable_segments_enabled() else "native" + + +def _host_os() -> OperatingSystem: + if sys.platform == "win32": + return "windows" + return "linux" if sys.platform.startswith("linux") else "other" def platform_for(device: torch.device) -> Platform: @@ -36,10 +94,11 @@ def platform_for(device: torch.device) -> Platform: MIOpen from cuDNN, and a `cpu_only` model on a ROCm build is not on MIOpen. Any device type other than CUDA, MPS and XPU runs torch's CPU kernels, or none at all, and is priced as the CPU. """ + allocator, os = _allocator_for(device), _host_os() if device.type == "cuda": - return Platform(conv_backend="miopen" if torch.version.hip is not None else "cudnn") + return Platform("miopen" if torch.version.hip is not None else "cudnn", allocator, os) if device.type == "mps": - return Platform(conv_backend="mps") + return Platform("mps", allocator, os) if device.type == "xpu": - return Platform(conv_backend="xpu") - return Platform(conv_backend="cpu") + return Platform("xpu", allocator, os) + return Platform("cpu", allocator, os) diff --git a/scripts/calibrate_flux2_working_memory.py b/scripts/calibrate_flux2_working_memory.py index c429c5b9a49..783fe00d04f 100644 --- a/scripts/calibrate_flux2_working_memory.py +++ b/scripts/calibrate_flux2_working_memory.py @@ -7,8 +7,9 @@ 1. ``SDPA_MATH_BYTES_PER_SCORE_ELEMENT`` (``backend/util/attention.py``) -- bytes per element of the score matrix, charged only where SDPA has no fused kernel and materializes it. -2. ``estimate_vae_working_memory_flux2``'s ``k_decode`` / ``k_encode`` bytes per pixel per element byte, - shipped per convolution backend in ``backend/working_memory/data/vae.json``. +2. ``estimate_vae_working_memory_flux2``'s convolution lines, ``k__`` bytes per pixel plus a + ``b__`` base, shipped per convolution backend and allocator in + ``backend/working_memory/data/vae.json``. 3. ``Flux2DenoiseInvocation._estimate_working_memory``'s 0.4 MB per token, defined at the Klein 9B width (4096) and scaled linearly by the loaded variant's width. 4. The dispatch question underneath all of it: does this build's SDPA fuse these shapes, or does it @@ -75,9 +76,12 @@ MIB = 1024**2 -def _shipped_vae_coefficient(operation: str) -> float: - """The VAE coefficient this build's estimate uses for `operation`: the column its convolution backend selects.""" - return shipped_coefficients(VAE_FLUX2_SPEC, platform_for(torch.device("cuda"))).values[f"k_{operation}"] +def _shipped_vae_line(operation: str, dtype_name: str) -> tuple[float, float]: + """The VAE line this build's estimate uses for `operation` in `dtype_name`: bytes per pixel and base bytes, as the + platform (convolution backend, allocator) selects them.""" + dtype = {"float16": "fp16", "bfloat16": "bf16", "float32": "fp32"}[dtype_name] + values = shipped_coefficients(VAE_FLUX2_SPEC, platform_for(torch.device("cuda"))).values + return values[f"k_{operation}_{dtype}"], values[f"b_{operation}_{dtype}"] # The FLUX.2 VAE attends on the 8x-downsampled grid with a single 512-wide head. Mirrors @@ -285,13 +289,13 @@ def run() -> None: "reserved_delta": peak, "estimate": estimate, "score_term": score_bytes, - # The whole measured peak over the pixel area, which is what the `k_decode`/`k_encode` - # coefficients name. The score matrix is deliberately NOT backed out: the two terms peak in different + # The whole measured peak over the pixel area: the line's bytes per pixel plus its base spread over + # that area. The score matrix is deliberately NOT backed out: the two terms peak in different # phases of the forward, so the estimate takes the larger of them rather than the sum, and # the convolution phase is what sets the high-water mark until the quadratic term overtakes # it past ~1280px. Where `score_term` exceeds the measured peak, this column is not the # quantity being fitted -- the score model is, and it is bounding rather than describing. - "implied_linear_constant": peak / (px * px * element_size), + "implied_linear_constant": peak / (px * px), "covered": peak <= estimate, } @@ -464,10 +468,12 @@ def report_sdpa(shapes: list[tuple[int, int, int]], dtype_name: str) -> list[dic def report_vae(pxs: list[int], dtype_name: str, vae_path: str | None) -> list[dict]: - decode_k = _shipped_vae_coefficient("decode") - encode_k = _shipped_vae_coefficient("encode") - print(f"\n=== 3. VAE linear coefficients (this build selects {decode_k:.0f} decode / {encode_k:.0f} encode) ===") - print("`implied_k` is the measured peak over pixel area, directly comparable to those coefficients;") + (decode_k, decode_b), (encode_k, encode_b) = (_shipped_vae_line(op, dtype_name) for op in ("decode", "encode")) + print( + f"\n=== 3. VAE lines (this build selects {decode_k:.0f} B/px + {decode_b / MIB:.0f} MiB decode, " + f"{encode_k:.0f} B/px + {encode_b / MIB:.0f} MiB encode) ===" + ) + print("`implied_k` is the measured peak over pixel area, comparable to those bytes per pixel plus the base;") print("fit on the rows whose `math` column matches what this build really does (section 1).") print("`covered` is the question that matters: is the shipped estimate an upper bound here?") print("Caveat: forcing math on a build that HAS a fused kernel is not equivalent to a build that") @@ -502,10 +508,10 @@ def report_vae(pxs: list[int], dtype_name: str, vae_path: str | None) -> list[di _flag_non_monotonic(rows) for operation in ("decode", "encode"): # Compare against the column this build actually selects, not against a hard-coded number. - shipped = _shipped_vae_coefficient(operation) + shipped_k, shipped_b = _shipped_vae_line(operation, dtype_name) for force_math in (False, True): ks = [ - r["implied_linear_constant"] + r["implied_linear_constant"] - shipped_b / (r["px"] * r["px"]) for r in rows if r["operation"] == operation and r["force_math"] is force_math @@ -517,8 +523,11 @@ def report_vae(pxs: list[int], dtype_name: str, vae_path: str | None) -> list[di if not ks: continue mode = "math" if force_math else "fused" - verdict = "OK" if shipped >= max(ks) else "SHORT" - print(f" {operation} ({mode}): implied_k max = {max(ks):.0f}, shipped = {shipped:.0f} -> {verdict}") + verdict = "OK" if shipped_k >= max(ks) else "SHORT" + print( + f" {operation} ({mode}): implied_k max past the base = {max(ks):.0f}, shipped = {shipped_k:.0f} " + f"-> {verdict}" + ) short = [r for r in rows if not r.get("oom") and not r["covered"]] if short: print("\n Points the shipped estimate does NOT cover:") diff --git a/tests/app/invocations/test_flux2_working_memory.py b/tests/app/invocations/test_flux2_working_memory.py index 6eeb2180bb1..80ada811c4b 100644 --- a/tests/app/invocations/test_flux2_working_memory.py +++ b/tests/app/invocations/test_flux2_working_memory.py @@ -46,9 +46,10 @@ GB = 1024**3 -def _vae_k(conv_backend, operation): - """The shipped FLUX.2 VAE coefficient the estimator resolves for one convolution backend and operation.""" - return shipped_coefficients(VAE_FLUX2_SPEC, Platform(conv_backend)).values[f"k_{operation}"] +def _vae_line(conv_backend, operation, pixels, allocator="native", os="linux"): + """What the shipped FLUX.2 VAE line reserves in bf16 over `pixels` output pixels on one platform.""" + values = shipped_coefficients(VAE_FLUX2_SPEC, Platform(conv_backend, allocator, os)).values + return int(values[f"k_{operation}_bf16"] * pixels + values[f"b_{operation}_bf16"]) # The estimator's default width, and the head count that follows from it. The suite's pinned @@ -282,7 +283,7 @@ def test_the_lora_margin_scales_with_the_batch(self): class TestFlux2VaeWorkingMemoryEstimate: def _mock_bf16_vae(self): vae = MagicMock(spec=AutoencoderKLFlux2) - vae.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) # element_size == 2 + vae.parameters.return_value = iter([torch.zeros(1, dtype=torch.bfloat16)]) return vae def _tensor_for(self, operation, px): @@ -290,11 +291,11 @@ def _tensor_for(self, operation, px): return torch.zeros(1, 32, px // 8, px // 8) if operation == "decode" else torch.zeros(1, 3, px, px) @pytest.mark.parametrize("operation", ["decode", "encode"]) - def test_constant_scales_pixel_area_and_element_size(self, operation): + def test_the_estimate_is_the_bf16_line_over_the_pixel_area(self, operation): estimate = estimate_vae_working_memory_flux2( operation=operation, image_tensor=self._tensor_for(operation, 1024), vae=self._mock_bf16_vae(), device=FUSED ) - assert estimate.reservation_bytes == 1024 * 1024 * 2 * _vae_k("cudnn", operation) + assert estimate.reservation_bytes == _vae_line("cudnn", operation, 1024 * 1024) def test_tiled_estimate_is_bounded_by_the_tile_not_the_image(self): """Reference-image encoding forces 512px tiling precisely so the peak stops following the @@ -310,7 +311,7 @@ def test_tiled_estimate_is_bounded_by_the_tile_not_the_image(self): for px in (1024, 1328, 2024) ] assert len(set(estimates)) == 1 - assert estimates[0] == int(512 * 512 * 2 * _vae_k("cudnn", "encode") * 1.25) + assert estimates[0] == _vae_line("cudnn", "encode", 512 * 512 * 1.25) # The whole point of tiling: it must shrink the reservation, not just bound the VAE. untiled = estimate_vae_working_memory_flux2( operation="encode", image_tensor=torch.zeros(1, 3, 2024, 2024), vae=self._mock_bf16_vae(), device=FUSED @@ -357,7 +358,8 @@ def estimate(): cuda = estimate() assert rocm > cuda - assert rocm / cuda == pytest.approx(_vae_k("miopen", "decode") / _vae_k("cudnn", "decode"), rel=1e-3) + assert rocm == _vae_line("miopen", "decode", 1024 * 1024) + assert cuda == _vae_line("cudnn", "decode", 1024 * 1024) def test_a_rocm_build_composes_the_miopen_column_with_the_guard_score_matrix(self): """The state every ROCm rig is actually in, driven through the estimator. @@ -383,6 +385,9 @@ def estimate(px): with ( patch("torch.version.hip", "7.1.25424"), + # Expandable segments on Windows, the ROCm default there, where the MIOpen line is the smaller one. + patch("invokeai.backend.working_memory.platform.expandable_segments_enabled", return_value=True), + patch("invokeai.backend.working_memory.platform._host_os", return_value="windows"), patch("invokeai.backend.util.attention._IS_ROCM", True), # Pinned against `INVOKE_ROCM_FUSED_SDPA_MAX_HEAD_DIM`, which a user can set high # enough to disarm the guard; any threshold between the denoise's 128-wide head and @@ -392,10 +397,9 @@ def estimate(px): at_1024 = estimate(1024) at_1536 = estimate(1536) - miopen = _vae_k("miopen", "decode") - assert at_1024 == 1024 * 1024 * 2 * miopen, "at 1024px the MIOpen linear term is the larger" + assert at_1024 == _vae_line("miopen", "decode", 1024 * 1024, "expandable", "windows"), "the line is larger" assert at_1536 == 36864 * 36864 * SDPA_MATH_BYTES_PER_SCORE_ELEMENT, "at 1536px the score matrix is" - assert at_1536 > 1536 * 1536 * 2 * miopen + assert at_1536 > _vae_line("miopen", "decode", 1536 * 1536, "expandable", "windows") class TestFlux2VaeBatchIsBudgeted: @@ -857,7 +861,7 @@ def _estimate(self, px, device, materializing): def test_the_estimate_is_the_larger_term_not_the_sum(self): """At 1024px the two are within 2% of each other on the cuDNN column, which makes this the sharpest place to tell the models apart.""" - linear = 1024 * 1024 * 2 * _vae_k("cudnn", "decode") + linear = _vae_line("cudnn", "decode", 1024 * 1024) score = 16384 * 16384 * SDPA_MATH_BYTES_PER_SCORE_ELEMENT assert self._estimate(1024, MATERIALIZING, materializing=True) == max(linear, score) assert self._estimate(1024, MATERIALIZING, materializing=True) < linear + score @@ -865,7 +869,7 @@ def test_the_estimate_is_the_larger_term_not_the_sum(self): def test_a_fused_build_is_unaffected(self): """The max only ever removes reservation, never adds it: with no score matrix the estimate is the linear term, exactly as before.""" - linear = 1024 * 1024 * 2 * _vae_k("cudnn", "decode") + linear = _vae_line("cudnn", "decode", 1024 * 1024) assert self._estimate(1024, FUSED, materializing=False) == linear def test_regional_prompting_adds_the_score_matrix(self): diff --git a/tests/app/invocations/test_vae_working_memory_backend.py b/tests/app/invocations/test_vae_working_memory_backend.py index 9dfc4fade93..63f8f092335 100644 --- a/tests/app/invocations/test_vae_working_memory_backend.py +++ b/tests/app/invocations/test_vae_working_memory_backend.py @@ -1,9 +1,9 @@ -"""Which convolution-backend coefficients the diffusers-AutoencoderKL estimators use, and what the SD estimator adds up. +"""Which convolution line the diffusers-AutoencoderKL estimators price, and what the SD estimator adds up. -SD1/SDXL, SD3 and CogView4 run the same AutoencoderKL convolution stack as the FLUX.1 autoencoder, so they take the -same measured per-backend coefficients: MIOpen needs ~1.64x what cuDNN does. The change is large enough to matter -- an -SD1.5 1024px fp32 decode reserves 9.49 GB on cuDNN and 15.36 GB on MIOpen -- and the branch is invisible on a CUDA -machine, so it is pinned here rather than left to whichever card the suite runs on. +SD1/SDXL, SD3, FLUX.1 and FLUX.2 share measured lines per operation, dtype and platform (convolution backend, +allocator, and on MIOpen's native allocator the OS); CogView4 carries its own. The selection is invisible on whichever +machine runs the suite -- on the RX 9060 XT an fp16 SDXL decode costs a third of a bf16 one, and the SD nodes run fp16 +whatever dtype the cached VAE holds -- so it is pinned here rather than left to the card. """ from unittest.mock import MagicMock, patch @@ -11,93 +11,355 @@ import pytest import torch from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL +from diffusers.models.autoencoders.autoencoder_kl_flux2 import AutoencoderKLFlux2 +from invokeai.app.invocations.vae.image_to_latents import ImageToLatentsInvocation +from invokeai.app.invocations.vae.latents_to_image import LatentsToImageInvocation from invokeai.backend.util.vae_working_memory import ( VAE_COGVIEW4_SPEC, + VAE_FLUX2_SPEC, + VAE_FLUX_SPEC, VAE_SD3_SPEC, VAE_SD_SPEC, estimate_vae_working_memory_cogview4, + estimate_vae_working_memory_flux, + estimate_vae_working_memory_flux2, estimate_vae_working_memory_sd3, estimate_vae_working_memory_sd15_sdxl, + sd_vae_run_dtype, ) -from invokeai.backend.working_memory.coefficients import shipped_coefficients -from invokeai.backend.working_memory.platform import Platform +from invokeai.backend.working_memory import platform as platform_module +from invokeai.backend.working_memory.coefficients import shipped_coefficients, shipped_data +from invokeai.backend.working_memory.platform import Platform, priced_as +PRE_DTYPE_BYTES_PER_PIXEL_PER_ELEMENT_BYTE = { + "cudnn": {"decode": 2200, "encode": 1100}, + "miopen": {"decode": 3600, "encode": 2750}, +} +"""What these estimators reserved before lines per dtype, per convolution backend, with no base: the floor no stand-in +and nothing unmeasured may price a slope below. Written out rather than read from the data, so a lowered floor fails +here. It covers the convolution lines; the one place v2 can reserve less than v1 there is the SD fp32 upcast, which +v1 guessed at 250 MiB and v2 prices as what the cast adds (160 MiB from fp16, nothing from fp32).""" +ELEMENT_SIZES = {"fp16": 2, "bf16": 2, "fp32": 4} -def _vae(dtype: torch.dtype = torch.float16) -> MagicMock: - vae = MagicMock(spec=AutoencoderKL) - vae.parameters.side_effect = lambda: iter([torch.zeros(1, dtype=dtype)]) +MEASURED = [ + Platform("cudnn", "native", "linux"), + Platform("cudnn", "cudaMallocAsync", "linux"), + Platform("miopen", "native", "linux"), + Platform("miopen", "native", "windows"), + Platform("miopen", "expandable", "windows"), +] + + +def _vae(cls=AutoencoderKL, dtype: torch.dtype = torch.float16, numel: int = 1, params=None) -> MagicMock: + vae = MagicMock(spec=cls) + vae.parameters.side_effect = lambda: iter(params if params is not None else [torch.zeros(numel, dtype=dtype)]) + vae.config.latent_channels = 16 return vae -def _estimate(estimator, device: torch.device, hip: str | None) -> int: - latents = torch.zeros(1, 4, 64, 64) # 512px output +@pytest.fixture +def on(monkeypatch): + """Run the estimate as on `platform`: the torch build, the allocator a CUDA device reports, and the OS.""" + + def pin(platform: Platform) -> torch.device: + monkeypatch.setattr(torch.version, "hip", "7.2.0" if platform.conv_backend == "miopen" else None) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr( + torch.cuda, + "get_allocator_backend", + lambda: "cudaMallocAsync" if platform.allocator == "cudaMallocAsync" else "native", + ) + if platform.allocator == "expandable": + monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") + monkeypatch.setattr(platform_module, "_host_os", lambda: platform.os) + return torch.device("cuda", 0) + + # The mid-block score matrix is the other term in these estimates and has its own tests; hold it at zero so the + # assertions below are about the convolution line alone. with ( - patch("torch.version.hip", hip), - # The mid-block score matrix is the other term in these estimates and has its own tests; hold it at zero so - # the assertions below are about the convolution coefficient alone. patch("invokeai.backend.util.vae_working_memory._vae_mid_block_score_matrix_bytes", return_value=0), + patch("invokeai.backend.util.vae_working_memory.sdpa_score_matrix_bytes", return_value=0), ): - if estimator is estimate_vae_working_memory_sd15_sdxl: - estimate = estimator( - operation="decode", image_tensor=latents, vae=_vae(), tile_size=None, fp32=False, device=device - ) - else: - estimate = estimator(operation="decode", image_tensor=latents, vae=_vae(), device=device) - return estimate.reservation_bytes + yield pin -def _decode_bytes(spec, conv_backend: str) -> float: - """A 512px fp16 decode at the shipped coefficient of that backend.""" - return 512 * 512 * 2 * shipped_coefficients(spec, Platform(conv_backend)).values["k_decode"] +def _line(spec, platform: Platform, operation: str, dtype: str, pixels: float) -> float: + values = shipped_coefficients(spec, platform).values + return values[f"k_{operation}_{dtype}"] * pixels + values[f"b_{operation}_{dtype}"] -ESTIMATORS = [ - pytest.param(estimate_vae_working_memory_sd15_sdxl, VAE_SD_SPEC, id="sd15-sdxl"), - pytest.param(estimate_vae_working_memory_sd3, VAE_SD3_SPEC, id="sd3"), - pytest.param(estimate_vae_working_memory_cogview4, VAE_COGVIEW4_SPEC, id="cogview4"), -] +LATENTS_1024 = torch.zeros(1, 16, 128, 128) + + +@pytest.mark.parametrize("platform", MEASURED, ids=str) +@pytest.mark.parametrize( + "spec, estimate, vae_dtype, run_dtype", + [ + # The SD nodes cast the VAE to fp16 for every GPU run that does not ask for fp32: bf16 never runs. + pytest.param( + VAE_SD_SPEC, + lambda device, vae: estimate_vae_working_memory_sd15_sdxl("decode", LATENTS_1024, vae, None, False, device), + torch.bfloat16, + "fp16", + id="sd", + ), + pytest.param( + VAE_SD3_SPEC, + lambda device, vae: estimate_vae_working_memory_sd3("decode", LATENTS_1024, vae, device), + torch.float32, + "fp32", + id="sd3", + ), + pytest.param( + VAE_COGVIEW4_SPEC, + lambda device, vae: estimate_vae_working_memory_cogview4("decode", LATENTS_1024, vae, device), + torch.bfloat16, + "bf16", + id="cogview4", + ), + pytest.param( + VAE_FLUX_SPEC, + lambda device, vae: estimate_vae_working_memory_flux("decode", LATENTS_1024, vae, device=device), + torch.bfloat16, + "bf16", + id="flux", + ), + pytest.param( + VAE_FLUX2_SPEC, + lambda device, vae: estimate_vae_working_memory_flux2( + "decode", torch.zeros(1, 32, 128, 128), vae, device=device + ), + torch.float32, + "fp32", + id="flux2", + ), + ], +) +def test_a_decode_reserves_the_line_of_its_run_dtype_on_its_platform( + on, platform, spec, estimate, vae_dtype, run_dtype +) -> None: + cls = AutoencoderKLFlux2 if spec is VAE_FLUX2_SPEC else AutoencoderKL + result = estimate(on(platform), _vae(cls, vae_dtype)) + + assert result.features["dtype"] == run_dtype + assert result.reservation_bytes == int(_line(spec, platform, "decode", run_dtype, 1024 * 1024)) + + +def test_the_platforms_differ_where_the_measurements_do(on) -> None: + """The point of keying on the allocator and, for MIOpen's native allocator, the OS: one SDXL fp16 decode, five + prices.""" + decodes = { + platform: estimate_vae_working_memory_sd15_sdxl( + "decode", LATENTS_1024, _vae(), None, False, on(platform) + ).reservation_bytes + for platform in MEASURED + } + assert decodes[Platform("cudnn", "cudaMallocAsync", "linux")] < decodes[Platform("cudnn", "native", "linux")] + assert decodes[Platform("miopen", "expandable", "windows")] < decodes[Platform("miopen", "native", "windows")] + assert decodes[Platform("miopen", "native", "windows")] != decodes[Platform("miopen", "native", "linux")] + + +def test_the_cogview4_vae_is_priced_on_its_own_lines_without_a_score_matrix(on) -> None: + """Its VAE is wider than the family's and has no mid-block attention: an FLUX.1-sized decode costs ~1.9x.""" + device = on(Platform("cudnn", "native", "linux")) + cogview4 = estimate_vae_working_memory_cogview4("decode", LATENTS_1024, _vae(dtype=torch.bfloat16), device) + flux = estimate_vae_working_memory_flux("decode", LATENTS_1024, _vae(dtype=torch.bfloat16), device=device) + assert "score_matrix_bytes" not in cogview4.features + assert cogview4.reservation_bytes > 1.8 * flux.reservation_bytes + + +@pytest.mark.parametrize( + "operation, fp32, device, run_dtype", + [ + ("decode", False, torch.device("cuda", 0), torch.float16), + ("decode", True, torch.device("cuda", 0), torch.float32), + # The decode node computes in fp32 on the CPU whatever it is asked; the encode node does not. + ("decode", False, torch.device("cpu"), torch.float32), + ("encode", False, torch.device("cpu"), torch.float16), + ("encode", True, torch.device("cuda", 0), torch.float32), + ], +) +@pytest.mark.parametrize("vae_dtype", [torch.float16, torch.bfloat16, torch.float32], ids=str) +def test_the_sd_estimate_prices_the_dtype_the_node_casts_to_not_the_cached_one( + on, operation, fp32, device, run_dtype, vae_dtype +) -> None: + """The cached VAE holds whatever the previous run cast it to; the node casts it again before it runs.""" + on(Platform("cudnn", "native", "linux")) + assert sd_vae_run_dtype(operation, fp32, device) == run_dtype + image = LATENTS_1024[:, :4] if operation == "decode" else torch.zeros(1, 3, 1024, 1024) + result = estimate_vae_working_memory_sd15_sdxl(operation, image, _vae(dtype=vae_dtype), None, fp32, device) + assert result.features["dtype"] == {torch.float16: "fp16", torch.float32: "fp32"}[run_dtype] -@pytest.mark.parametrize("estimator, spec", ESTIMATORS) -def test_a_rocm_vae_reserves_the_miopen_coefficient(estimator, spec): - assert _estimate(estimator, torch.device("cuda", 0), hip="7.2.0") == _decode_bytes(spec, "miopen") +def _tiny_sd_vae() -> AutoencoderKL: + return AutoencoderKL( + down_block_types=("DownEncoderBlock2D",), + up_block_types=("UpDecoderBlock2D",), + block_out_channels=(32,), + layers_per_block=1, + latent_channels=4, + norm_num_groups=32, + ) -@pytest.mark.parametrize("estimator, spec", ESTIMATORS) -def test_a_cuda_vae_reserves_the_cudnn_coefficient(estimator, spec): - assert _estimate(estimator, torch.device("cuda", 0), hip=None) == _decode_bytes(spec, "cudnn") +def _vae_info(vae: AutoencoderKL, estimates: list) -> MagicMock: + def model_on_device(working_memory): + estimates.append(working_memory) + cm = MagicMock() + cm.__enter__.return_value = (None, vae) + cm.__exit__.return_value = None + return cm + vae_info = MagicMock(model=vae, compute_device=torch.device("cpu")) + vae_info.model_on_device.side_effect = model_on_device + return vae_info -@pytest.mark.parametrize("estimator, spec", ESTIMATORS) -def test_a_cpu_only_vae_reserves_the_cudnn_coefficient_even_on_a_rocm_build(estimator, spec): - """A `cpu_only` VAE does not run on MIOpen, so naming its device has to change the answer.""" - assert _estimate(estimator, torch.device("cpu"), hip="7.2.0") == _decode_bytes(spec, "cudnn") +@pytest.mark.parametrize("fp32", [False, True]) +def test_the_decode_node_casts_the_vae_to_the_dtype_its_estimate_priced(fp32) -> None: + """On the CPU both requests decode in fp32: the node and the estimate share `sd_vae_run_dtype`.""" + vae, estimates = _tiny_sd_vae().to(torch.bfloat16), [] + context = MagicMock() + context.tensors.load.return_value = torch.zeros(1, 4, 8, 8) + context.config.get.return_value.force_tiled_decode = False + context.models.load.return_value = _vae_info(vae, estimates) + context.images.save.return_value = MagicMock(image_name="x.png", width=8, height=8) + node = LatentsToImageInvocation.model_construct( + latents=MagicMock(latents_name="x"), vae=MagicMock(seamless_axes=[]), tiled=False, tile_size=0, fp32=fp32 + ) -def _sd_fp32_decode(tile_size: int | None) -> int: - latents = torch.zeros(1, 4, 96, 128) # a 768x1024 output - with patch("invokeai.backend.util.vae_working_memory._vae_mid_block_score_matrix_bytes", return_value=0): - estimate = estimate_vae_working_memory_sd15_sdxl( - operation="decode", - image_tensor=latents, - vae=_vae(), - tile_size=tile_size, - fp32=True, - device=torch.device("cpu"), + node.invoke(context) + + (estimate,) = estimates + assert estimate.features["dtype"] == "fp32" + assert {p.dtype for p in vae.parameters()} == {torch.float32} + + +@pytest.mark.parametrize("upcast, run_dtype", [(False, torch.float16), (True, torch.float32)]) +def test_the_encode_node_casts_the_vae_to_the_dtype_its_estimate_priced(upcast, run_dtype) -> None: + vae, estimates, encoded_in = _tiny_sd_vae().to(torch.bfloat16), [], [] + + def encode(vae, image): + encoded_in.append(next(vae.encoder.parameters()).dtype) + return torch.zeros(1, 4, 2, 2) + + with patch.object(ImageToLatentsInvocation, "_encode_to_tensor", side_effect=encode): + ImageToLatentsInvocation.vae_encode(_vae_info(vae, estimates), upcast, False, torch.zeros(1, 3, 16, 16)) + + (estimate,) = estimates + assert encoded_in == [run_dtype] + assert estimate.features["dtype"] == {torch.float16: "fp16", torch.float32: "fp32"}[run_dtype] + + +@pytest.mark.parametrize( + "vae_dtype, fp32, growth", + [ + (torch.float16, True, 1000 * 2), + (torch.bfloat16, True, 1000 * 2), + (torch.float32, True, 0), + # Casting down frees memory rather than taking it. + (torch.float32, False, 0), + (torch.float16, False, 0), + ], +) +def test_an_sd_upcast_reserves_what_the_cast_adds_to_the_weights(on, vae_dtype, fp32, growth) -> None: + """The node casts the VAE in place once the cache has placed it, so the growth comes out of the reservation.""" + platform = Platform("cudnn", "native", "linux") + device = on(platform) + result = estimate_vae_working_memory_sd15_sdxl( + "decode", LATENTS_1024[:, :4], _vae(dtype=vae_dtype, numel=1000), None, fp32, device + ) + run_dtype = "fp32" if fp32 else "fp16" + assert result.reservation_bytes == int(_line(VAE_SD_SPEC, platform, "decode", run_dtype, 1024 * 1024) + growth) + + +@pytest.mark.parametrize("fp32, run_dtype, growth", [(False, "fp16", 0), (True, "fp32", 1000 * 2)]) +def test_a_tiled_sd_decode_is_one_tile_with_its_margin_the_base_once_and_the_upcast( + on, fp32, run_dtype, growth +) -> None: + platform = Platform("miopen", "expandable", "windows") + result = estimate_vae_working_memory_sd15_sdxl( + "decode", torch.zeros(1, 4, 192, 192), _vae(numel=1000), 512, fp32, on(platform) + ) + assert result.reservation_bytes == int(_line(VAE_SD_SPEC, platform, "decode", run_dtype, 512 * 512 * 1.25) + growth) + + +def test_a_dtype_the_estimator_has_no_line_for_is_refused_by_name(on) -> None: + """The FLUX.2 loaders never run fp16; a VAE that does anyway must not fail mid-generation on a bare KeyError.""" + device = on(Platform("cudnn", "native", "linux")) + with pytest.raises(ValueError, match=r"'vae\.flux2' has no fp16 decode line on Platform\(conv_backend='cudnn'"): + estimate_vae_working_memory_flux2( + "decode", torch.zeros(1, 32, 64, 64), _vae(AutoencoderKLFlux2, torch.float16), device=device ) - return estimate.reservation_bytes -FP32_WEIGHTS = 250 * 2**20 -"""What an fp32 decode adds for the upcast weights.""" +@pytest.mark.parametrize( + "leading, dtype", + [ + pytest.param(torch.zeros(4, dtype=torch.float8_e4m3fn), "bf16", id="fp8-storage"), + pytest.param(torch.zeros(4, dtype=torch.int8), "bf16", id="packed-integer"), + ], +) +def test_the_compute_dtype_skips_weights_stored_narrower_than_they_compute(on, leading, dtype) -> None: + """An SDNQ VAE's first parameter can be its packed integer storage; it computes in its floating-point layers'.""" + device = on(Platform("cudnn", "native", "linux")) + vae = _vae(params=[leading, torch.zeros(1, dtype=torch.bfloat16)]) + assert estimate_vae_working_memory_flux("decode", LATENTS_1024, vae, device=device).features["dtype"] == dtype + +def test_every_stand_in_slope_is_at_least_the_pre_dtype_figure() -> None: + """A stand-in is a declared conservative value on a measured platform; nothing else guards it.""" + stand_ins = [entry for data in shipped_data().values() for entry in data.coefficients if entry.stand_ins] + assert stand_ins, "the shipped data declares no stand-ins -- the check is broken" + for entry in stand_ins: + for name in entry.stand_ins: + part, operation, dtype = name.split("_") + value = entry.values[name] + if part == "k": + floor = PRE_DTYPE_BYTES_PER_PIXEL_PER_ELEMENT_BYTE[entry.platform["conv_backend"]][operation] + assert value >= floor * ELEMENT_SIZES[dtype], f"{entry.estimator} {entry.platform} {name}" + else: + assert value >= 0 -def test_a_tiled_fp32_sd_decode_is_one_tile_with_its_margin_plus_the_fp32_weights(): - k = shipped_coefficients(VAE_SD_SPEC, Platform("cudnn")).values["k_decode"] - assert _sd_fp32_decode(512) == int(512 * 512 * 4 * k * 1.25 + FP32_WEIGHTS) + +@pytest.mark.parametrize( + "spec", [VAE_SD_SPEC, VAE_SD3_SPEC, VAE_COGVIEW4_SPEC, VAE_FLUX_SPEC, VAE_FLUX2_SPEC], ids=lambda spec: spec.id +) +@pytest.mark.parametrize( + "platform", + [ + Platform("mps", "native", "other"), + Platform("xpu", "native", "windows"), + Platform("cpu", "native", "linux"), + Platform("cudnn", "expandable", "linux"), + Platform("miopen", "expandable", "linux"), + Platform("miopen", "expandable", "other"), + Platform("miopen", "cudaMallocAsync", "linux"), + Platform("miopen", "cudaMallocAsync", "windows"), + ], + ids=str, +) +def test_an_unmeasured_platform_never_prices_a_slope_below_the_pre_dtype_figure(spec, platform) -> None: + measured = priced_as(platform) + values = shipped_coefficients(spec, platform).values + for name, value in values.items(): + part, operation, dtype = name.split("_") + if part == "k": + floor = PRE_DTYPE_BYTES_PER_PIXEL_PER_ELEMENT_BYTE[measured.conv_backend][operation] * ELEMENT_SIZES[dtype] + assert value >= floor, f"{name} on {platform}: {value} under {floor}" + else: + assert value >= 0 -def test_an_untiled_fp32_sd_decode_is_the_whole_image_plus_the_fp32_weights(): - k = shipped_coefficients(VAE_SD_SPEC, Platform("cudnn")).values["k_decode"] - assert _sd_fp32_decode(None) == int(768 * 1024 * 4 * k + FP32_WEIGHTS) +def test_expandable_segments_off_windows_are_priced_as_the_native_allocator_there(on) -> None: + """Measured on Windows only, where they are Invoke's ROCm default. On Linux they are a user override nobody + measured: an SDXL fp16 decode there must not take the Windows 2.5 GiB at 1024px, nor go under the pre-dtype 7200 MiB.""" + linux = estimate_vae_working_memory_sd15_sdxl( + "decode", LATENTS_1024[:, :4], _vae(), None, False, on(Platform("miopen", "expandable", "linux")) + ) + native = estimate_vae_working_memory_sd15_sdxl( + "decode", LATENTS_1024[:, :4], _vae(), None, False, on(Platform("miopen", "native", "linux")) + ) + assert linux.reservation_bytes == native.reservation_bytes >= 3600 * 2 * 1024 * 1024 + assert "priced as conv_backend=miopen,allocator=native" in linux.source diff --git a/tests/app/invocations/test_z_image_tiled_decode.py b/tests/app/invocations/test_z_image_tiled_decode.py index eb9d3a3a9b6..df6167ae418 100644 --- a/tests/app/invocations/test_z_image_tiled_decode.py +++ b/tests/app/invocations/test_z_image_tiled_decode.py @@ -20,19 +20,23 @@ def _flux_reservation(*args, **kwargs) -> int: return estimate_vae_working_memory_flux(*args, **kwargs).reservation_bytes -def _k(conv_backend: str, operation: str) -> float: - """The shipped FLUX.1 VAE coefficient the estimator resolves for one convolution backend and operation.""" - return shipped_coefficients(VAE_FLUX_SPEC, Platform(conv_backend)).values[f"k_{operation}"] +CPU = Platform("cpu", "native", "linux") + + +def _line(platform: Platform, operation: str, pixels: float, dtype: str = "bf16") -> float: + """What the shipped FLUX.1 VAE line the estimator resolves on `platform` prices over `pixels` output pixels.""" + values = shipped_coefficients(VAE_FLUX_SPEC, platform).values + return values[f"k_{operation}_{dtype}"] * pixels + values[f"b_{operation}_{dtype}"] def _mock_flux_vae(element_size_bytes: int = 2) -> MagicMock: - """A stand-in for the estimator cells only: they read nothing but the parameter element size. + """A stand-in for the estimator cells only: they read nothing but the parameter dtype. The node-wiring cells below use a real `AutoencoderKL`, because they are about tiling state a mock cannot have -- `tile_overlap_factor` is set in `__init__`, so a spec'd mock lacks it. """ vae = MagicMock(spec=AutoencoderKL) - dtype = torch.float16 if element_size_bytes == 2 else torch.float32 + dtype = torch.bfloat16 if element_size_bytes == 2 else torch.float32 # A fresh iterator per call: the decode path reads `parameters()` after the estimator already # has, and a single stored iterator would be exhausted by then. vae.parameters.side_effect = lambda: iter([torch.zeros(1, dtype=dtype)]) @@ -85,14 +89,14 @@ def test_the_convolution_backend_of_the_vaes_device_picks_the_constant( operation=operation, image_tensor=tensor, vae=_mock_flux_vae(), device=torch.device("cpu") ) - assert estimate == 512 * 512 * 2 * _k(conv_backend, operation) - assert on_cpu == 512 * 512 * 2 * _k("cudnn", operation), "a cpu_only VAE runs on cuDNN terms" + assert estimate == int(_line(Platform(conv_backend, "native", "linux"), operation, 512 * 512)) + assert on_cpu == int(_line(CPU, operation, 512 * 512)), "a cpu_only VAE runs on cuDNN terms" def test_the_default_reproduces_the_untiled_estimate(self): """Regression guard for the six call sites that pass no tile_size at all.""" latents = torch.zeros(1, 16, 128, 128) - # 1024x1024 output px * 2 bytes * the cuDNN decode coefficient - expected = 1024 * 1024 * 2 * _k("cudnn", "decode") + # The bf16 decode line over 1024x1024 output pixels + expected = int(_line(CPU, "decode", 1024 * 1024)) actual = _flux_reservation(operation="decode", image_tensor=latents, vae=_mock_flux_vae()) assert actual == expected @@ -104,8 +108,8 @@ def test_the_tiled_estimate_is_a_tile_bounded_decode_plus_the_assembled_image(se margin or halving the per-pixel image cost would be a several-hundred-megabyte under-reservation that no other assertion here can see. """ - # 512px tiles, fp16, over a 1536x1536 output. - decode_term = 512 * 512 * 2 * _k("cudnn", "decode") * 1.25 + # 512px tiles, bf16, over a 1536x1536 output. + decode_term = _line(CPU, "decode", 512 * 512 * 1.25) # 4 RGB buffers at the VAE's element size, plus the 8-bit one the node hands to PIL. Four # is measured, not assumed: the assembled image grows the device peak by ~1.6 copies on a # diffusers VAE and ~0.1 on the host-merging BFL one, and the node adds its in-window @@ -116,17 +120,14 @@ def test_the_tiled_estimate_is_a_tile_bounded_decode_plus_the_assembled_image(se ) assert estimate == int(decode_term + image_term) - def test_both_tiled_terms_scale_with_the_vaes_element_size(self): - """An fp32 VAE costs twice the bytes per pixel, in the tile and in the assembled image - alike. Only the 8-bit buffer is fixed, which is why this is a little under 2x.""" + def test_both_tiled_terms_follow_the_vaes_dtype(self): + """An fp32 VAE prices its tile at the fp32 line, which is not twice the bf16 one, and its assembled image at + four bytes per value. Only the 8-bit buffer is fixed.""" latents = torch.zeros(1, 16, 192, 192) - fp16 = _flux_reservation( - operation="decode", image_tensor=latents, vae=_mock_flux_vae(element_size_bytes=2), tile_size=512 - ) fp32 = _flux_reservation( operation="decode", image_tensor=latents, vae=_mock_flux_vae(element_size_bytes=4), tile_size=512 ) - assert 1.95 < fp32 / fp16 < 2.0 + assert fp32 == int(_line(CPU, "decode", 512 * 512 * 1.25, "fp32") + 1536 * 1536 * 3 * (4 * 4 + 1)) @pytest.mark.parametrize("latent_hw", [(128, 128), (192, 192), (256, 256)]) def test_a_tiled_estimate_stays_a_fraction_of_the_untiled_one(self, latent_hw): @@ -179,7 +180,7 @@ def test_a_tiled_estimate_still_covers_the_assembled_image(self): estimate = _flux_reservation( operation="decode", image_tensor=torch.zeros(1, 16, 1024, 1024), vae=_mock_flux_vae(), tile_size=512 ) - tile_term = int(512 * 512 * 2 * _k("cudnn", "decode") * 1.25) + tile_term = int(_line(CPU, "decode", 512 * 512 * 1.25)) one_full_resolution_buffer = 8192 * 8192 * 3 * 2 # The assembled output plus at least one of the post-processing copies. assert estimate >= tile_term + 2 * one_full_resolution_buffer @@ -201,7 +202,7 @@ def test_the_sentinel_works_on_a_vae_without_that_attribute_at_all(self): # sibling dereferences `vae.tile_sample_min_size` directly and would raise here. latents = torch.zeros(1, 16, 192, 192) vae = MagicMock(spec=AutoencoderKL) - vae.parameters.side_effect = lambda: iter([torch.zeros(1, dtype=torch.float16)]) + vae.parameters.side_effect = lambda: iter([torch.zeros(1, dtype=torch.bfloat16)]) del vae.tile_sample_min_size estimate = _flux_reservation(operation="decode", image_tensor=latents, vae=vae, tile_size=0) assert estimate == _flux_reservation( @@ -235,7 +236,7 @@ def _fused_non_rocm_build(self, monkeypatch): def test_an_untiled_encode_is_the_whole_image_at_the_encode_coefficient(self): """What the call sites that pass no tile reserve: the full image, priced at the cuDNN encode coefficient.""" image = torch.zeros(1, 3, 1024, 1024) - assert _flux_reservation("encode", image, _mock_flux_vae()) == 1024 * 1024 * 2 * _k("cudnn", "encode") + assert _flux_reservation("encode", image, _mock_flux_vae()) == int(_line(CPU, "encode", 1024 * 1024)) def test_a_tiled_estimate_is_flat_in_the_image_size_where_the_untiled_one_is_not(self): vae = _mock_flux_vae() @@ -247,7 +248,7 @@ def test_a_tiled_estimate_is_flat_in_the_image_size_where_the_untiled_one_is_not # But they are a rounding error beside the tile term, which is what "flat" has to mean. assert max(estimates) < min(estimates) * 1.2 untiled = [_flux_reservation("encode", torch.zeros(1, 3, edge, edge), vae) for edge in (1024, 3072)] - assert untiled[1] == untiled[0] * 9 + assert untiled == [int(_line(CPU, "encode", edge * edge)) for edge in (1024, 3072)] def test_the_tiled_estimate_is_one_tile_plus_the_input_and_every_live_copy_of_the_moments(self): """The moments are not one copy. `AutoencoderKL._tiled_encode` keeps every encoded tile in @@ -256,7 +257,7 @@ def test_the_tiled_estimate_is_one_tile_plus_the_input_and_every_live_copy_of_th vae = _mock_flux_vae() edge, tile = 2048, 512 live_moment_copies = 1.0 / (1.0 - 0.25) ** 2 + 2.0 - expected = tile * tile * 2 * _k("cudnn", "encode") * 1.25 + expected = _line(CPU, "encode", tile * tile * 1.25) expected += edge * edge * 3 * 2 expected += (edge // 8) * (edge // 8) * 2 * 16 * 2 * live_moment_copies assert _flux_reservation("encode", torch.zeros(1, 3, edge, edge), vae, tile_size=tile) == int(expected) diff --git a/tests/backend/model_manager/load/model_cache/test_model_cache_vram_accounting.py b/tests/backend/model_manager/load/model_cache/test_model_cache_vram_accounting.py index a5f7f41ec81..da28b8c3736 100644 --- a/tests/backend/model_manager/load/model_cache/test_model_cache_vram_accounting.py +++ b/tests/backend/model_manager/load/model_cache/test_model_cache_vram_accounting.py @@ -323,7 +323,7 @@ def _release(self, monkeypatch, *, driver_free_mb: int, releasable_mb: int, work TorchDevice, "cuda_mem_get_info", classmethod(lambda cls, d: (driver_free_mb * MB, 24 * GB)) ) monkeypatch.setattr(cache, "_get_reclaimable_allocator_bytes", lambda: releasable_mb * MB) - monkeypatch.setattr(model_cache_module, "_expandable_segments_enabled", lambda: False) + monkeypatch.setattr(model_cache_module, "expandable_segments_enabled", lambda: False) monkeypatch.setattr(TorchDevice, "empty_cache", classmethod(lambda cls, force=False: calls.append(force))) cache._release_allocator_blocks_for_reserve(None if working_mb is None else working_mb * MB) return calls @@ -359,7 +359,7 @@ def test_under_expandable_segments_the_unused_reserved_pages_count(self, monkeyp cache = self._cache("cuda") calls: list = [] monkeypatch.setattr(TorchDevice, "cuda_mem_get_info", classmethod(lambda cls, d: (2 * GB, 24 * GB))) - monkeypatch.setattr(model_cache_module, "_expandable_segments_enabled", lambda: True) + monkeypatch.setattr(model_cache_module, "expandable_segments_enabled", lambda: True) monkeypatch.setattr(cache, "_get_reclaimable_allocator_bytes", lambda: 0) monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device=None: 9 * GB) monkeypatch.setattr(torch.cuda, "memory_allocated", lambda device=None: 6 * GB) diff --git a/tests/backend/util/test_rocm_sdpa_head_dim_guard.py b/tests/backend/util/test_rocm_sdpa_head_dim_guard.py index 6710b3afbf3..e86929f4413 100644 --- a/tests/backend/util/test_rocm_sdpa_head_dim_guard.py +++ b/tests/backend/util/test_rocm_sdpa_head_dim_guard.py @@ -494,6 +494,8 @@ def _rocm_cuda(self, monkeypatch): monkeypatch.setattr(attention, "_IS_ROCM", True) monkeypatch.setattr(torch.version, "hip", None) monkeypatch.setattr(vwm.TorchDevice, "choose_torch_device", classmethod(lambda cls: torch.device("cuda", 0))) + # The native allocator, whatever the machine running the suite configures. + monkeypatch.setattr("invokeai.backend.working_memory.platform._allocator_for", lambda device: "native") @staticmethod def _vae(spec): @@ -509,13 +511,14 @@ def _score_bytes(px: int) -> int: return tokens * tokens * attention.SDPA_MATH_BYTES_PER_SCORE_ELEMENT @staticmethod - def _cudnn_decode_bytes(px: int) -> float: - """A bf16 decode's convolution term at the shipped cuDNN coefficient.""" + def _cudnn_decode_bytes(px: int) -> int: + """An SD fp16 decode's convolution line at the shipped cuDNN coefficients: the GPU run casts the VAE to fp16.""" from invokeai.backend.util.vae_working_memory import VAE_SD_SPEC from invokeai.backend.working_memory.coefficients import shipped_coefficients from invokeai.backend.working_memory.platform import Platform - return px * px * 2 * shipped_coefficients(VAE_SD_SPEC, Platform("cudnn")).values["k_decode"] + values = shipped_coefficients(VAE_SD_SPEC, Platform("cudnn", "native", "linux")).values + return int(values["k_decode_fp16"] * px * px + values["b_decode_fp16"]) def test_sdxl_untiled_decode_reserves_the_score_matrix_when_it_dominates(self, monkeypatch): from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL @@ -542,16 +545,25 @@ def test_sdxl_untiled_decode_reserves_the_score_matrix_when_it_dominates(self, m == linear ) - @pytest.mark.parametrize("estimator", ["estimate_vae_working_memory_sd3", "estimate_vae_working_memory_cogview4"]) - def test_the_other_autoencoder_kl_estimators_price_it_too(self, estimator): + def test_the_sd3_estimator_prices_it_too(self): from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL - import invokeai.backend.util.vae_working_memory as vwm + from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_sd3 latents = torch.zeros(1, 16, 2048 // 8, 2048 // 8) - estimate = getattr(vwm, estimator)("decode", latents, self._vae(AutoencoderKL)) + estimate = estimate_vae_working_memory_sd3("decode", latents, self._vae(AutoencoderKL)) assert estimate.reservation_bytes == self._score_bytes(2048) + def test_the_cogview4_vae_has_no_attention_to_price(self): + """Its config sets `mid_block_add_attention: false`: the guard never sees it.""" + from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL + + from invokeai.backend.util.vae_working_memory import estimate_vae_working_memory_cogview4 + + latents = torch.zeros(1, 16, 2048 // 8, 2048 // 8) + estimate = estimate_vae_working_memory_cogview4("decode", latents, self._vae(AutoencoderKL)) + assert all(term.coefficient is not None for term in estimate.terms) + def test_a_tiny_autoencoder_has_no_attention_to_price(self): from diffusers.models.autoencoders.autoencoder_tiny import AutoencoderTiny diff --git a/tests/backend/working_memory/test_fit.py b/tests/backend/working_memory/test_fit.py new file mode 100644 index 00000000000..4979b1cca94 --- /dev/null +++ b/tests/backend/working_memory/test_fit.py @@ -0,0 +1,135 @@ +"""The shipped fit: an upper envelope of the points, with a non-negative base, plus the shipped margin.""" + +from fractions import Fraction + +import pytest + +from invokeai.backend.working_memory.coefficients import CoefficientEntry, MeasuredPoint, ValuesFrom, WorkingMemoryData +from invokeai.backend.working_memory.fit import SHIPPED_MARGIN, fitted_values, shipped_pair, upper_envelope + +MIB = 2**20 + +# A process base plus a per-pixel cost, read with the scatter real points have: the shape the VAE points take. +SCATTERED = [(262144, 940 * MIB), (589824, 1540 * MIB), (1048576, 2380 * MIB), (1638400, 3120 * MIB)] +PROPORTIONAL = [(262144, 1100 * MIB), (589824, 2457 * MIB), (1048576, 4330 * MIB), (4194304, 17194 * MIB)] + + +@pytest.mark.parametrize("points", [SCATTERED, PROPORTIONAL], ids=["with-a-base", "proportional"]) +def test_the_envelope_covers_every_point_with_a_non_negative_base(points) -> None: + k, b = upper_envelope(points) + assert b >= 0 + assert all(k * x + b >= y for x, y in points) + + +def test_the_envelope_is_the_hull_edge_that_over_reserves_its_worst_point_least() -> None: + """Through the origin and the 512px point the line covers everything too, but over-reserves 1280px by 1.48x; the + edge from 512px to 1024px over-reserves its worst point, 1280px, by 1.11x.""" + k, b = upper_envelope(SCATTERED) + assert (k, b) == (Fraction(600 * MIB, 327680), 940 * MIB - Fraction(600 * MIB, 327680) * 262144) + assert max((k * x + b) / y for x, y in SCATTERED) < Fraction(112, 100) + + +def test_the_largest_reading_of_a_size_is_the_one_covered() -> None: + k, b = upper_envelope([(262144, 1000 * MIB), (262144, 1100 * MIB), (1048576, 4400 * MIB)]) + assert k * 262144 + b >= 1100 * MIB + + +def test_a_tie_between_two_edges_goes_to_the_steeper_one() -> None: + """Both outer edges over-reserve their worst point by 1.25x; the steeper is the more conservative past the largest + point.""" + assert upper_envelope([(1, 10), (2, 15), (4, 20)]) == (5, 5) + + +def test_a_falling_edge_is_never_the_line() -> None: + """A line that falls with the size would price larger images below smaller ones.""" + assert upper_envelope([(1, 10), (4, 5)]) == (10, 0) + + +def test_the_shipped_margin_is_five_percent() -> None: + assert SHIPPED_MARGIN == Fraction(5, 100) + + +def test_the_shipped_pair_adds_the_margin_and_rounds_up() -> None: + k, b = upper_envelope(SCATTERED) + shipped_k, shipped_b = shipped_pair(SCATTERED) + assert shipped_k >= k * (1 + SHIPPED_MARGIN) > shipped_k - 1 + assert shipped_b % MIB == 0 + assert shipped_b >= b * (1 + SHIPPED_MARGIN) > shipped_b - MIB + + +@pytest.mark.parametrize( + "points", + [ + pytest.param([], id="none"), + pytest.param([(1048576, 4330 * MIB)], id="one-size"), + pytest.param([(1048576, 4330 * MIB), (1048576, 4300 * MIB)], id="one-size-twice"), + pytest.param([(1048576, 4330 * MIB), (1638400, 6748 * MIB)], id="sizes-too-close"), + ], +) +def test_a_line_needs_two_sizes_at_least_twice_apart(points) -> None: + with pytest.raises(ValueError): + upper_envelope(points) + + +def _point(estimator: str, operation: str, px: int, mib: int, **extra) -> MeasuredPoint: + features = {"operation": operation, "dtype": "bf16", "height": px, "width": px, "tile": None} | extra + return MeasuredPoint( + estimator=estimator, + version=1, + platform={"conv_backend": "cudnn"}, + features=features, + measured_bytes=mib * MIB, + method="reserved_peak", + device="-", + provenance="-", + ) + + +def _entry(estimator: str, values=None, lender: str | None = None) -> CoefficientEntry: + return CoefficientEntry( + estimator=estimator, + version=1, + platform={"conv_backend": "cudnn"}, + values=values, + values_from=ValuesFrom(estimator=lender, version=1) if lender else None, + provenance="-", + ) + + +def test_a_line_is_fitted_on_the_single_sample_untiled_points_of_every_member_that_borrows_it() -> None: + values = {"k_decode_bf16": 0.0, "b_decode_bf16": 0.0} + data = WorkingMemoryData( + schema_version=1, + coefficients=[_entry("lender", values), _entry("borrower", lender="lender")], + points=[ + _point("lender", "decode", 512, 1100), + _point("borrower", "decode", 1024, 4400), + # None of these is the line's: another operation, a tile, a batch, a point out of regime. + _point("borrower", "encode", 1024, 9000), + _point("borrower", "decode", 2048, 1000, tile=512), + _point("borrower", "decode", 1024, 9000, batch_size=2), + MeasuredPoint.model_validate(_point("lender", "decode", 2048, 90000).model_dump() | {"in_fit": False}), + ], + ) + fitted = fitted_values([data])[("lender", 1, (("conv_backend", "cudnn"),))] + assert (fitted["k_decode_bf16"], fitted["b_decode_bf16"]) == shipped_pair( + [(512 * 512, 1100 * MIB), (1024 * 1024, 4400 * MIB)] + ) + + +def test_a_line_without_enough_points_names_its_entry() -> None: + data = WorkingMemoryData( + schema_version=1, + coefficients=[_entry("lender", {"k_decode_bf16": 0.0, "b_decode_bf16": 0.0})], + points=[_point("lender", "decode", 512, 1100)], + ) + with pytest.raises(ValueError, match=r"lender on \{'conv_backend': 'cudnn'\}, decode_bf16"): + fitted_values([data]) + + +def test_a_declared_stand_in_line_is_not_fitted() -> None: + entry = _entry("lender", {"k_decode_bf16": 7200.0, "b_decode_bf16": 0.0}).model_copy( + update={"stand_ins": ["k_decode_bf16", "b_decode_bf16"]} + ) + data = WorkingMemoryData(schema_version=1, coefficients=[entry], points=[]) + assert fitted_values([data]) == {} diff --git a/tests/backend/working_memory/test_shipped_coefficients.py b/tests/backend/working_memory/test_shipped_coefficients.py index d369d03df90..829a3386083 100644 --- a/tests/backend/working_memory/test_shipped_coefficients.py +++ b/tests/backend/working_memory/test_shipped_coefficients.py @@ -1,17 +1,20 @@ """The shipped working-memory data, checked against the estimators that read it. -Every coefficient entry resolves for the spec it names on a measured platform value, and every value an entry carries -is decided by at least one measured point. A point is peak *reserved* bytes, one fresh process per point (see its -`provenance`), and bounds its estimate from both sides. The model cache consumes an estimate as `free >= estimate`, so -the estimate must be at least the point. Where a fitted term decides it, it must also be at most 1.3 times the point, -or the point's tighter `max_ratio`: a reservation the operation never uses is model weights evicted for nothing. Where -an analytic term (a materialized score matrix) decides it, that term bounds rather than describes, so it gets no upper -bound, and the cache's `device_working_mem_gb` floor counts toward the lower one. +Every coefficient entry resolves for the spec it names on a measured platform. Every value an entry carries is either a +declared stand-in or decided by at least one in-fit point and the shipped fit (`fit.py`) of the points it is resolved +for. A point is a peak over the operation, one fresh process per point (see its `method` and `provenance`), and bounds +its estimate from both sides. The model cache consumes an estimate as `free >= estimate`, so the estimate must be at +least the point. Where a fitted term decides an in-fit point, it must also be at most 1.3 times the point, or the +point's tighter `max_ratio`: a reservation the operation never uses is model weights evicted for nothing. The points no +line can hold that close are listed in `UPPER_BOUND_EXCEPTIONS`. Where an analytic term (a materialized score matrix) +decides it, that term bounds rather than describes, so it gets no upper bound, and the cache's `device_working_mem_gb` +floor counts toward the lower one. An out-of-fit point gets the lower bound only, and `OUT_OF_FIT` lists them. """ +import collections import itertools import re -from dataclasses import asdict +from dataclasses import replace from pathlib import Path from typing import get_args, get_type_hints @@ -21,6 +24,8 @@ import invokeai from invokeai.app.services.config.config_default import InvokeAIAppConfig +from invokeai.app.util.torch_cuda_allocator import ROCM_WINDOWS_ALLOC_CONF +from invokeai.backend.working_memory import platform as platform_module from invokeai.backend.working_memory.coefficients import ( CoefficientEntry, MeasuredPoint, @@ -33,7 +38,8 @@ shipped_data, ) from invokeai.backend.working_memory.estimate import WorkingMemoryEstimate -from invokeai.backend.working_memory.platform import UNMEASURED_PRICED_AS, Platform, platform_for +from invokeai.backend.working_memory.fit import carrier_of, entry_key, fit_points, fitted_values +from invokeai.backend.working_memory.platform import MEASURED, Platform, is_measured, platform_for, priced_as from invokeai.backend.working_memory.spec import ESTIMATOR_MODULES, EstimatorSpec, import_estimator_modules MAX_OVER_RESERVATION = 1.3 @@ -44,26 +50,41 @@ """What `ModelCache._working_mem_reserve` floors every reservation at. The field default, not a config instance an environment variable could move.""" +UPPER_BOUND_EXCEPTIONS = { + "vae.sd-miopen,native,windows-decode-fp16-1280x1280-RX9060XT-demand_peak": 1.33, + "vae.sd-miopen,native,windows-encode-fp16-1024x1024-RX9060XT-demand_peak": 1.57, + "vae.sd-miopen,native,windows-encode-fp16-1280x1280-RX9060XT-demand_peak": 1.78, + "vae.flux-miopen,native,windows-decode-bf16-512x512-RX9060XT-demand_peak": 1.33, + "vae.flux2-miopen,native,windows-decode-bf16-512x512-RX9060XT-demand_peak": 1.33, +} +"""In-fit points whose envelope sits more than 1.3x above them, with the ratio they are held to instead. All are MIOpen +with the native allocator on Windows, where the demand is concave in the size: first-run driver spikes at small sizes +and a second run reserving up to 1.6x the first at large ones, so a line that covers every point cannot stay within +1.3x of all of them. Windows ROCm defaults to expandable segments; this is the override's price.""" + +OUT_OF_FIT = {((("allocator", "native"), ("conv_backend", "cudnn")), "RTX 4090", "reserved_peak"): 6} +"""The out-of-fit points by platform, device and method: the six CUDA forced-math-kernel decodes and encodes, which +check the score-matrix term. A point that leaves the fit needs an edit here.""" + # Imported deliberately rather than relying on whichever modules ran first: a spec missing from the registry is a spec # whose data nobody checks. SPECS = import_estimator_modules() +EVERY_PLATFORM = [ + Platform(**dict(zip(get_type_hints(Platform), values, strict=True))) + for values in itertools.product(*(get_args(hint) for hint in get_type_hints(Platform).values())) +] +MEASURED_PLATFORMS = [platform for platform in EVERY_PLATFORM if is_measured(platform)] +UNMEASURED_PLATFORMS = [platform for platform in EVERY_PLATFORM if not is_measured(platform)] -def _platforms(*, measured: bool) -> list[Platform]: - """Every platform built from measured values only, or every platform with at least one unmeasured value.""" - every = {key: get_args(hint) for key, hint in get_type_hints(Platform).items()} - measured_values = { - key: [v for v in values if v not in UNMEASURED_PRICED_AS.get(key, {})] for key, values in every.items() - } - chosen = measured_values if measured else every - platforms = [Platform(**dict(zip(chosen, values, strict=True))) for values in itertools.product(*chosen.values())] - if measured: - return platforms - return [p for p in platforms if any(value in UNMEASURED_PRICED_AS.get(key, {}) for key, value in asdict(p).items())] +def _keys(platform: Platform, spec: EstimatorSpec) -> dict[str, str]: + return {key: getattr(platform, key) for key in spec.platform_keys} -def _platform_key(platform: dict[str, str]) -> tuple[tuple[str, str], ...]: - return tuple(sorted(platform.items())) + +def _platform_of(point: MeasuredPoint) -> Platform: + """The platform a point is evaluated on. One that names no OS measured what holds on every OS.""" + return Platform(**({"os": "other"} | point.platform)) def _point_id(point: MeasuredPoint) -> str: @@ -71,23 +92,28 @@ def _point_id(point: MeasuredPoint) -> str: shape = f"{features.get('height')}x{features.get('width')}" tile = f"-tile{features['tile']}" if features.get("tile") is not None else "" batch = f"-b{features['batch_size']}" if features.get("batch_size", 1) != 1 else "" - scores = "-scores" if features.get("score_matrix_bytes") else "" + out_of_fit = "-outfit" if not point.in_fit else "" platform = ",".join(point.platform.values()) card = point.device.split(" (")[0].replace(" ", "") - return f"{point.estimator}-{platform}-{features.get('operation')}-{shape}{tile}{batch}{scores}-{card}" + return ( + f"{point.estimator}-{platform}-{features.get('operation')}-{features.get('dtype')}-{shape}{tile}{batch}" + f"{out_of_fit}-{card}-{point.method}" + ) def _evaluate(point: MeasuredPoint) -> WorkingMemoryEstimate: spec = SPECS[point.estimator] assert point.version == spec.version - return spec.evaluate(point.features, shipped_coefficients(spec, Platform(**point.platform))) + return spec.evaluate(point.features, shipped_coefficients(spec, _platform_of(point))) def _fitted_decides(estimate: WorkingMemoryEstimate) -> bool: return any(term.included and term.coefficient is not None for term in estimate.terms) -POINTS = [pytest.param(point, id=_point_id(point)) for data in shipped_data().values() for point in data.points] +ALL_POINTS = [point for data in shipped_data().values() for point in data.points] +POINTS = [pytest.param(point, id=_point_id(point)) for point in ALL_POINTS] +ENTRIES = [entry for data in shipped_data().values() for entry in data.coefficients] def test_every_data_file_parses_and_declares_coefficients() -> None: @@ -95,70 +121,128 @@ def test_every_data_file_parses_and_declares_coefficients() -> None: assert data.coefficients, f"{name} declares no coefficients" -def test_every_coefficient_entry_resolves_for_its_spec_on_a_measured_platform_value() -> None: +def test_every_coefficient_entry_resolves_for_its_spec_on_a_measured_platform() -> None: """Resolving checks what an entry provides against what its spec reads, the borrowed values included. An entry for - an unmeasured value would never be read: the resolver prices that value as `UNMEASURED_PRICED_AS` says.""" + an unmeasured platform would never be read: the resolver prices that platform as `priced_as` says. Only the OS may + refine the spec's keys.""" seen: set[tuple[str, int, tuple[tuple[str, str], ...]]] = set() - for name, data in shipped_data().items(): - for entry in data.coefficients: - where = f"{name}: {entry.estimator} on {entry.platform}" - spec = SPECS.get(entry.estimator) - assert spec is not None, f"{where}: no estimator registers this id" - assert entry.version == spec.version, f"{where}: version {entry.version}, the estimator is {spec.version}" - assert set(entry.platform) == set(spec.platform_keys), f"{where}: the spec keys {spec.platform_keys}" - for key, value in entry.platform.items(): - assert value not in UNMEASURED_PRICED_AS.get(key, {}), f"{where}: {key}={value} is not measured" - key = (entry.estimator, entry.version, _platform_key(entry.platform)) - assert key not in seen, f"{where}: declared twice" - seen.add(key) - shipped_coefficients(spec, Platform(**entry.platform)) + for entry in ENTRIES: + where = f"{entry.estimator} on {entry.platform}" + spec = SPECS.get(entry.estimator) + assert spec is not None, f"{where}: no estimator registers this id" + assert entry.version == spec.version, f"{where}: version {entry.version}, the estimator is {spec.version}" + assert set(entry.platform) - {"os"} == set(spec.platform_keys), f"{where}: the spec keys {spec.platform_keys}" + platforms = [ + p for p in MEASURED_PLATFORMS if all(getattr(p, key) == value for key, value in entry.platform.items()) + ] + assert platforms, f"{where}: not a measured platform" + assert entry_key(entry) not in seen, f"{where}: declared twice" + seen.add(entry_key(entry)) + for platform in platforms: + shipped_coefficients(spec, platform) @pytest.mark.parametrize("spec", sorted(SPECS.values(), key=lambda spec: spec.id), ids=lambda spec: spec.id) -def test_every_estimator_has_coefficients_on_every_measured_platform_value(spec: EstimatorSpec) -> None: - for platform in _platforms(measured=True): +def test_every_estimator_has_coefficients_on_every_measured_platform(spec: EstimatorSpec) -> None: + for platform in MEASURED_PLATFORMS: assert set(shipped_coefficients(spec, platform).values) == spec.coefficients @pytest.mark.parametrize("spec", sorted(SPECS.values(), key=lambda spec: spec.id), ids=lambda spec: spec.id) -def test_an_unmeasured_platform_value_is_priced_as_the_measured_one_and_says_so(spec: EstimatorSpec) -> None: - for platform in _platforms(measured=False): - priced_as = Platform( - **{key: UNMEASURED_PRICED_AS.get(key, {}).get(value, value) for key, value in asdict(platform).items()} - ) +def test_the_os_only_matters_where_an_entry_names_it(spec: EstimatorSpec) -> None: + """cuDNN and MIOpen with expandable segments are priced the same on every OS.""" + os_specific = { + (entry.platform["conv_backend"], entry.platform["allocator"]) for entry in ENTRIES if "os" in entry.platform + } + assert os_specific == {("miopen", "native"), ("miopen", "expandable")} + for platform in MEASURED_PLATFORMS: + if (platform.conv_backend, platform.allocator) in os_specific: + continue + values = [shipped_coefficients(spec, replace(platform, os=os)).values for os in ("windows", "linux", "other")] + assert all(dict(v) == dict(values[0]) for v in values), platform + + +@pytest.mark.parametrize("spec", sorted(SPECS.values(), key=lambda spec: spec.id), ids=lambda spec: spec.id) +def test_an_unmeasured_platform_is_priced_as_the_measured_one_at_least_and_says_so(spec: EstimatorSpec) -> None: + for platform in UNMEASURED_PLATFORMS: + measured = priced_as(platform) + assert is_measured(measured) and measured.os == platform.os resolved = shipped_coefficients(spec, platform) - assert resolved.values == shipped_coefficients(spec, priced_as).values - assert resolved.source.startswith( - f"shipped:conv_backend={platform.conv_backend} ({priced_as.conv_backend} values" - ) + borrowed = shipped_coefficients(spec, measured) + assert all(resolved.values[name] >= value for name, value in borrowed.values.items()) + where = ",".join(f"{key}={value}" for key, value in _keys(measured, spec).items()) + assert resolved.source.startswith(f"shipped:conv_backend={platform.conv_backend},") + assert f"(priced as {where}" in resolved.source -def test_every_carried_value_is_decided_by_a_measured_point() -> None: +def test_every_fitted_value_is_decided_by_an_in_fit_point() -> None: """A value no point decides is unguarded: any number would pass. An estimator without points of its own takes a - measured one's values with `values_from`, and is then held to that one's points.""" - entries = { - (entry.estimator, entry.version, _platform_key(entry.platform)): entry - for data in shipped_data().values() - for entry in data.coefficients - } + measured one's values with `values_from`, and is then held to that one's points. Only declared stand-ins are + exempt, and they are listed in the entry.""" decided: set[tuple[tuple[str, int, tuple[tuple[str, str], ...]], str]] = set() - for data in shipped_data().values(): - for point in data.points: - key = (point.estimator, point.version, _platform_key(point.platform)) - lender = entries[key].values_from - owner = (lender.estimator, lender.version, key[2]) if lender is not None else key - estimate = _evaluate(point) - decided |= {(owner, term.coefficient) for term in estimate.terms if term.included and term.coefficient} + for point in ALL_POINTS: + if point.in_fit: + owner = entry_key(carrier_of(ENTRIES, point)) + decided |= {(owner, t.coefficient) for t in _evaluate(point).terms if t.included and t.coefficient} undecided = [ f"{entry.estimator} {entry.platform} {name}" - for key, entry in entries.items() + for entry in ENTRIES for name in entry.values or {} - if (key, name) not in decided + if (entry_key(entry), name) not in decided and name not in entry.stand_ins ] assert not undecided, f"no measured point decides these shipped values: {undecided}" +def test_every_fitted_value_is_the_shipped_fit_of_its_points() -> None: + """The values are reproducible from the points rather than hand-tuned. A new point that moves a fit fails here, + naming the values to ship.""" + fitted = fitted_values(shipped_data().values()) + stale = {} + for entry in ENTRIES: + if entry.values is None: + continue + expected = fitted.get(entry_key(entry), {}) + differs = { + name: expected.get(name) + for name, value in entry.values.items() + if name not in entry.stand_ins and expected.get(name) != value + } + if differs: + stale[f"{entry.estimator} {entry.platform}"] = differs + assert not stale, f"ship these fitted values: {stale}" + + +def test_a_stand_in_is_a_whole_line_no_in_fit_point_measures() -> None: + """A stand-in is for what a platform has not measured. Once an in-fit point measures its line, the line is fitted + instead, and this names it.""" + groups = fit_points(shipped_data().values()) + for entry in ENTRIES: + lines = {name[2:] for name in entry.stand_ins} + assert {f"{part}_{line}" for line in lines for part in ("k", "b")} == set(entry.stand_ins), entry.platform + measured = sorted(line for line in lines if (entry_key(entry), line) in groups) + assert not measured, f"{entry.estimator} on {entry.platform}: fit {measured}, they are measured now" + + +def test_the_out_of_fit_points_are_the_listed_ones() -> None: + """Leaving the fit removes a point's upper bound, so it is never a quiet flag flip.""" + out_of_fit = collections.Counter( + (tuple(sorted(point.platform.items())), point.device, point.method) for point in ALL_POINTS if not point.in_fit + ) + assert out_of_fit == OUT_OF_FIT + + +def test_every_listed_upper_bound_exception_is_a_shipped_point_that_needs_it() -> None: + """A stale exception would let a new point at that id over-reserve unseen.""" + ids = {_point_id(point): point for point in ALL_POINTS} + for point_id, ratio in UPPER_BOUND_EXCEPTIONS.items(): + point = ids.get(point_id) + assert point is not None, f"{point_id}: no such point" + assert point.in_fit + reserved = _evaluate(point).reservation_bytes / point.measured_bytes + assert MAX_OVER_RESERVATION < reserved <= ratio, f"{point_id}: {reserved:.3f}x" + + @pytest.mark.parametrize("point", POINTS) def test_every_measured_point_is_bounded_by_its_estimate(point: MeasuredPoint) -> None: estimate = _evaluate(point) @@ -168,8 +252,10 @@ def test_every_measured_point_is_bounded_by_its_estimate(point: MeasuredPoint) - assert reserved >= point.measured_bytes, ( f"reserves {reserved / 2**30:.3f} GiB for a measured {point.measured_bytes / 2**30:.3f} GiB" ) - if point.max_ratio is not None or fitted: - max_ratio = point.max_ratio if point.max_ratio is not None else MAX_OVER_RESERVATION + # An out-of-fit point checks another term: it gets the lower bound only. + if point.max_ratio is not None or (fitted and point.in_fit): + default = UPPER_BOUND_EXCEPTIONS.get(_point_id(point), MAX_OVER_RESERVATION) + max_ratio = point.max_ratio if point.max_ratio is not None else default assert estimate.reservation_bytes <= max_ratio * point.measured_bytes, ( f"reserves {estimate.reservation_bytes / point.measured_bytes:.2f}x the measured peak, over {max_ratio}x" ) @@ -177,10 +263,9 @@ def test_every_measured_point_is_bounded_by_its_estimate(point: MeasuredPoint) - def test_a_point_bound_is_never_looser_than_the_default() -> None: """`max_ratio` tightens the bound for a point the estimate has to stay close to; it never excuses one.""" - for name, data in shipped_data().items(): - for point in data.points: - if point.max_ratio is not None: - assert 1.0 <= point.max_ratio < MAX_OVER_RESERVATION, f"{name}: {_point_id(point)}" + for point in ALL_POINTS: + if point.max_ratio is not None: + assert 1.0 <= point.max_ratio < MAX_OVER_RESERVATION, _point_id(point) @pytest.mark.parametrize( @@ -197,26 +282,95 @@ def test_a_point_bound_is_never_looser_than_the_default() -> None: ) def test_the_platform_is_the_convolution_backend_the_device_really_runs(device, hip, conv_backend, monkeypatch) -> None: monkeypatch.setattr(torch.version, "hip", hip) - assert platform_for(device) == Platform(conv_backend=conv_backend) + assert platform_for(device).conv_backend == conv_backend + + +HOST_OS = platform_module._host_os +"""Bound at import, before the suite's fixture replaces it.""" + +ALLOC_VARS = ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_HIP_ALLOC_CONF") + + +@pytest.mark.parametrize("var", ALLOC_VARS) +@pytest.mark.parametrize( + "backend, alloc_conf, allocator", + [ + ("native", None, "native"), + ("native", "expandable_segments: True", "expandable"), + # What Invoke sets for a ROCm build on Windows. + ("native", ROCM_WINDOWS_ALLOC_CONF, "expandable"), + ("native", "expandable_segments:False", "native"), + # The driver's pool ignores expandable segments. + ("cudaMallocAsync", "backend:cudaMallocAsync,expandable_segments:True", "cudaMallocAsync"), + ], +) +def test_the_platform_is_the_allocator_a_gpu_allocates_through( + var, backend, alloc_conf, allocator, monkeypatch +) -> None: + for name in ALLOC_VARS: + monkeypatch.delenv(name, raising=False) + if alloc_conf is not None: + monkeypatch.setenv(var, alloc_conf) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_allocator_backend", lambda: backend) + + assert platform_for(torch.device("cuda", 0)).allocator == allocator + # Nothing but a CUDA/HIP GPU allocates through torch's caching allocator. + assert platform_for(torch.device("cpu")).allocator == "native" + assert platform_for(torch.device("mps")).allocator == "native" + + +@pytest.mark.parametrize( + "sys_platform, os", [("win32", "windows"), ("linux", "linux"), ("darwin", "other"), ("freebsd14", "other")] +) +def test_the_platform_is_the_os_the_process_runs_on(sys_platform, os, monkeypatch) -> None: + # The suite pins `_host_os` to Linux (tests/conftest.py); this is the function it replaces. + monkeypatch.setattr(platform_module.sys, "platform", sys_platform) + assert HOST_OS() == os + monkeypatch.setattr(platform_module, "_host_os", HOST_OS) + assert platform_for(torch.device("cpu")).os == os + + +@pytest.mark.parametrize( + "platform, measured", + [ + (("mps", "native", "linux"), ("cudnn", "native", "linux")), + (("xpu", "native", "windows"), ("cudnn", "native", "windows")), + (("cpu", "native", "other"), ("cudnn", "native", "other")), + (("cudnn", "expandable", "windows"), ("cudnn", "native", "windows")), + (("miopen", "cudaMallocAsync", "linux"), ("miopen", "native", "linux")), + # Expandable segments on MIOpen are measured on Windows only, where they are Invoke's default. + (("miopen", "expandable", "windows"), ("miopen", "expandable", "windows")), + (("miopen", "expandable", "linux"), ("miopen", "native", "linux")), + (("miopen", "expandable", "other"), ("miopen", "native", "other")), + ], +) +def test_an_unmeasured_platform_takes_cudnn_or_its_backends_native_allocator(platform, measured) -> None: + assert priced_as(Platform(*platform)) == Platform(*measured) + assert is_measured(Platform(*measured)) + assert {m for m in MEASURED if m[2] is not None} == {("miopen", "expandable", "windows")} def _no_terms(features, values): return () -def _spec(spec_id: str, version: int = 1) -> EstimatorSpec: +def _spec(spec_id: str, version: int = 1, coefficients: frozenset[str] = frozenset({"k"})) -> EstimatorSpec: return EstimatorSpec( - id=spec_id, version=version, coefficients=frozenset({"k"}), platform_keys=("conv_backend",), terms=_no_terms + id=spec_id, version=version, coefficients=coefficients, platform_keys=("conv_backend",), terms=_no_terms ) -def _entry(estimator: str, *, version: int = 1, values=None, values_from: str | None = None) -> CoefficientEntry: +def _entry( + estimator: str, *, version: int = 1, values=None, values_from: str | None = None, floor=None, os=None +) -> CoefficientEntry: return CoefficientEntry( estimator=estimator, version=version, - platform={"conv_backend": "cudnn"}, + platform={"conv_backend": "cudnn"} | ({"os": os} if os else {}), values=values, values_from=ValuesFrom(estimator=values_from, version=1) if values_from else None, + unmeasured_floor=floor or {}, provenance="-", ) @@ -225,38 +379,79 @@ def _data(*entries: CoefficientEntry) -> list[WorkingMemoryData]: return [WorkingMemoryData(schema_version=1, coefficients=list(entries), points=[])] +CUDNN = Platform("cudnn", "native", "linux") + + def test_a_missing_or_stale_entry_names_the_estimator_and_platform() -> None: """No fallback to another platform or version: coefficients fitted for something else are the shortfall the data exists to prevent.""" spec = _spec("test.estimator", version=2) assert resolve_coefficients( - _data(_entry(spec.id, version=2, values={"k": 7.0})), spec, Platform("cudnn"), origin="test" + _data(_entry(spec.id, version=2, values={"k": 7.0})), spec, CUDNN, origin="test" ) == ResolvedCoefficients(values={"k": 7.0}, source="test:conv_backend=cudnn") with pytest.raises(LookupError, match=r"'test\.estimator' version 2 on conv_backend=miopen\."): - resolve_coefficients(_data(_entry(spec.id, version=2, values={"k": 7.0})), spec, Platform("miopen"), "test") + resolve_coefficients( + _data(_entry(spec.id, version=2, values={"k": 7.0})), spec, replace(CUDNN, conv_backend="miopen"), "test" + ) with pytest.raises(LookupError, match=r"'test\.estimator' version 2 on conv_backend=cudnn \(found only version"): - resolve_coefficients(_data(_entry(spec.id, version=1, values={"k": 7.0})), spec, Platform("cudnn"), "test") + resolve_coefficients(_data(_entry(spec.id, version=1, values={"k": 7.0})), spec, CUDNN, "test") with pytest.raises(LookupError, match=r"declared 2 times"): resolve_coefficients( _data(_entry(spec.id, version=2, values={"k": 7.0}), _entry(spec.id, version=2, values={"k": 8.0})), spec, - Platform("cudnn"), + CUDNN, "test", ) +def test_an_entry_that_names_the_os_takes_precedence_there_and_only_there() -> None: + data = _data( + _entry("test.lender", values={"k": 7.0}), + _entry("test.lender", values={"k": 9.0}, os="windows"), + _entry("test.borrower", values_from="test.lender"), + ) + on_windows = resolve_coefficients(data, _spec("test.borrower"), replace(CUDNN, os="windows"), origin="test") + assert on_windows == ResolvedCoefficients( + values={"k": 9.0}, source="test:conv_backend=cudnn,os=windows (values of test.lender)" + ) + on_linux = resolve_coefficients(data, _spec("test.borrower"), CUDNN, origin="test") + assert dict(on_linux.values) == {"k": 7.0} + + def test_borrowed_values_resolve_one_hop_and_the_source_names_both_borrowings() -> None: data = _data(_entry("test.lender", values={"k": 7.0}), _entry("test.borrower", values_from="test.lender")) - on_cudnn = resolve_coefficients(data, _spec("test.borrower"), Platform("cudnn"), origin="test") + on_cudnn = resolve_coefficients(data, _spec("test.borrower"), CUDNN, origin="test") assert on_cudnn == ResolvedCoefficients(values={"k": 7.0}, source="test:conv_backend=cudnn (values of test.lender)") - on_xpu = resolve_coefficients(data, _spec("test.borrower"), Platform("xpu"), origin="test") + on_xpu = resolve_coefficients(data, _spec("test.borrower"), replace(CUDNN, conv_backend="xpu"), origin="test") assert on_xpu == ResolvedCoefficients( - values={"k": 7.0}, source="test:conv_backend=xpu (cudnn values of test.lender)" + values={"k": 7.0}, source="test:conv_backend=xpu (priced as conv_backend=cudnn, values of test.lender)" ) +def test_a_borrower_reads_the_lines_it_runs_of_a_lender_that_carries_more() -> None: + """A family member that runs fewer dtypes borrows from the one that runs them all; it sees only its own.""" + data = _data( + _entry("test.lender", values={"k": 7.0, "k_other": 9.0}), _entry("test.borrower", values_from="test.lender") + ) + resolved = resolve_coefficients(data, _spec("test.borrower"), CUDNN, origin="test") + assert dict(resolved.values) == {"k": 7.0} + # An entry's own values are exactly what its spec reads: an extra one would be a value nothing checks. + with pytest.raises(LookupError, match=r"provide \['k', 'k_other'\]"): + resolve_coefficients(data, _spec("test.lender"), CUDNN, origin="test") + + +def test_an_unmeasured_platform_raises_a_value_to_its_floor_and_a_measured_one_does_not() -> None: + data = _data(_entry("test.lender", values={"k": 7.0, "j": 5.0}, floor={"k": 8.0, "j": 4.0})) + spec = _spec("test.lender", coefficients=frozenset({"k", "j"})) + + assert dict(resolve_coefficients(data, spec, CUDNN, origin="test").values) == {"k": 7.0, "j": 5.0} + on_mps = resolve_coefficients(data, spec, replace(CUDNN, conv_backend="mps"), origin="test") + assert dict(on_mps.values) == {"k": 8.0, "j": 5.0} + assert on_mps.source == "test:conv_backend=mps (priced as conv_backend=cudnn, floored)" + + @pytest.mark.parametrize( "entries, message", [ @@ -276,7 +471,7 @@ def test_borrowed_values_resolve_one_hop_and_the_source_names_both_borrowings() ) def test_a_borrowing_that_is_not_one_hop_to_matching_values_is_refused(entries, message) -> None: with pytest.raises(LookupError, match=message): - resolve_coefficients(_data(*entries), _spec("a"), Platform("cudnn"), origin="test") + resolve_coefficients(_data(*entries), _spec("a"), CUDNN, origin="test") @pytest.mark.parametrize("values, values_from", [({"k": 1.0}, "b"), (None, None)], ids=["both", "neither"]) @@ -285,6 +480,13 @@ def test_an_entry_carries_exactly_one_of_values_and_values_from(values, values_f _entry("a", values=values, values_from=values_from) +def test_a_floor_or_stand_in_names_a_value_the_entry_carries() -> None: + with pytest.raises(ValidationError, match=r"unmeasured floors \['j'\]"): + _entry("a", values={"k": 1.0}, floor={"j": 2.0}) + with pytest.raises(ValidationError, match=r"stand-ins \['j'\]"): + CoefficientEntry(estimator="a", version=1, platform={}, values={"k": 1.0}, stand_ins=["j"], provenance="-") + + def test_a_data_directory_without_data_files_is_named(tmp_path: Path) -> None: with pytest.raises(FileNotFoundError, match=re.escape(str(tmp_path))): load_data(tmp_path) diff --git a/tests/conftest.py b/tests/conftest.py index 9bf75bc752a..c2e48b57ae6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -110,6 +110,18 @@ def _fast_service_polling(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(image_index_default, "_POLL_SECONDS", 0.02) +@pytest.fixture(autouse=True) +def _working_memory_platform_from_the_test_not_the_host(monkeypatch: pytest.MonkeyPatch) -> None: + """A working-memory estimate prices the allocator torch is configured with and the OS it runs on. Read from the + host, a developer's `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` or a Windows runner moves every expected + value. Tests start from no allocator configuration on Linux and set what they are about.""" + from invokeai.backend.working_memory import platform + + for var in ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_HIP_ALLOC_CONF"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setattr(platform, "_host_os", lambda: "linux") + + @pytest.fixture def mock_sqlite_database() -> Database: """The in-memory database behind `mock_services`, for tests that need SQL of their own or a service that From 0fd56fe092b4fb6b201727040b8e993c1dce7bf0 Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Sat, 17 Oct 2026 17:56:22 +0200 Subject: [PATCH 3/5] fix(working-memory): fit VAE lines against the highest reading per size A lower second reading at a measured size no longer tilts a fitted line, and points that include cast weight growth no longer fit one; two shipped lines move by under 1 %. --- invokeai/backend/working_memory/data/vae.json | 8 +++---- invokeai/backend/working_memory/fit.py | 21 +++++++++++++------ tests/backend/working_memory/test_fit.py | 10 ++++++++- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/invokeai/backend/working_memory/data/vae.json b/invokeai/backend/working_memory/data/vae.json index c018e4cf8c5..5321cdbec12 100644 --- a/invokeai/backend/working_memory/data/vae.json +++ b/invokeai/backend/working_memory/data/vae.json @@ -24,8 +24,8 @@ "values": { "k_decode_fp16": 4597, "b_decode_fp16": 83886080, - "k_decode_bf16": 4590, - "b_decode_bf16": 14680064, + "k_decode_bf16": 4500, + "b_decode_bf16": 68157440, "k_decode_fp32": 7702, "b_decode_fp32": 0, "k_encode_fp16": 2226, @@ -115,8 +115,8 @@ "b_decode_fp16": 0, "k_decode_bf16": 3614, "b_decode_bf16": 0, - "k_decode_fp32": 5964, - "b_decode_fp32": 0, + "k_decode_fp32": 5914, + "b_decode_fp32": 13631488, "k_encode_fp16": 1882, "b_encode_fp16": 17825792, "k_encode_bf16": 1882, diff --git a/invokeai/backend/working_memory/fit.py b/invokeai/backend/working_memory/fit.py index 94ac5da208c..8ca220c0f1c 100644 --- a/invokeai/backend/working_memory/fit.py +++ b/invokeai/backend/working_memory/fit.py @@ -5,7 +5,9 @@ size. Each carried pair is the upper envelope of the points it is resolved for (see `upper_envelope`), plus `SHIPPED_MARGIN`. A point joins the fit of the entry its estimator resolves to on its platform -- the lender's, where the estimator borrows -- when it is a single-sample, untiled operation of that pair (`in_fit` unless measured out of -regime). Tiled points bound their estimates but fit nothing: the tile term reuses the untiled pair. +regime) with no weight growth. Tiled points bound their estimates but fit nothing: the tile term reuses the untiled +pair. Nor does a point whose VAE an in-place cast grew (`upcast_weight_bytes`): its demand includes the growth, which +the estimate prices as a term of its own. `tests/backend/working_memory/test_shipped_coefficients.py` holds every carried value that is not a declared stand-in to `fitted_values` of the shipped points, so a new point that moves a fit fails there with the value to ship. @@ -40,9 +42,11 @@ def upper_envelope(points: Sequence[tuple[int, int]]) -> tuple[Fraction, Fractio """The line ``y = k * x + b`` on or above every ``(x, y)`` point, with ``b >= 0``, that over-reserves least. The candidates are the edges of the upper convex hull of the points and the origin: each such line lies on or - above every point, and the origin keeps its intercept non-negative. Of those, the one whose largest ratio to a - point is smallest wins, the steeper one on a tie, since it is the more conservative past the largest point. - Raises unless the points span two sizes at least `MIN_AREA_RATIO` apart. + above every point, and the origin keeps its intercept non-negative. Of those, the one whose largest ratio to the + highest reading of a size is smallest wins, the steeper one on a tie, since it is the more conservative past the + largest point. Only the highest reading counts: a lower one at a measured size is covered whatever the line, and + rating against it would tilt the line though no point lies above it. Raises unless the points span two sizes at + least `MIN_AREA_RATIO` apart. """ if not points or any(x <= 0 or y <= 0 for x, y in points): raise ValueError(f"an envelope needs points with positive sizes and bytes, got {list(points)}") @@ -69,7 +73,7 @@ def upper_envelope(points: Sequence[tuple[int, int]]) -> tuple[Fraction, Fractio b = y0 - k * x0 if k < 0: continue - worst = max((k * x + b) / y for x, y in points) + worst = max((k * x + b) / y for x, y in highest.items()) if best is None or (worst, -k) < (best[0], -best[1]): best = (worst, k, b) assert best is not None, "the edge from the origin always rises" @@ -101,7 +105,12 @@ def fit_points(data: Iterable[WorkingMemoryData]) -> Mapping[tuple[EntryKey, str groups: dict[tuple[EntryKey, str], list[tuple[int, int]]] = {} for point in (point for file in files for point in file.points): features = point.features - if not point.in_fit or features.get("tile") is not None or features.get("batch_size", 1) != 1: + if ( + not point.in_fit + or features.get("tile") is not None + or features.get("batch_size", 1) != 1 + or features.get("upcast_weight_bytes", 0) + ): continue owner = carrier_of(entries, point) suffix = f"{features.get('operation')}_{features.get('dtype')}" diff --git a/tests/backend/working_memory/test_fit.py b/tests/backend/working_memory/test_fit.py index 4979b1cca94..005c3614fbe 100644 --- a/tests/backend/working_memory/test_fit.py +++ b/tests/backend/working_memory/test_fit.py @@ -40,6 +40,12 @@ def test_a_tie_between_two_edges_goes_to_the_steeper_one() -> None: assert upper_envelope([(1, 10), (2, 15), (4, 20)]) == (5, 5) +def test_a_lower_reading_at_a_measured_size_does_not_move_the_line() -> None: + """Rated against every reading, the 12 at size 4 would hand the line to the flatter edge (it over-reserves 12 by + 1.67x instead of 2.08x) though no point lies above the line the highest readings give.""" + assert upper_envelope([(1, 10), (2, 15), (4, 20), (4, 12)]) == upper_envelope([(1, 10), (2, 15), (4, 20)]) + + def test_a_falling_edge_is_never_the_line() -> None: """A line that falls with the size would price larger images below smaller ones.""" assert upper_envelope([(1, 10), (4, 5)]) == (10, 0) @@ -104,10 +110,12 @@ def test_a_line_is_fitted_on_the_single_sample_untiled_points_of_every_member_th points=[ _point("lender", "decode", 512, 1100), _point("borrower", "decode", 1024, 4400), - # None of these is the line's: another operation, a tile, a batch, a point out of regime. + # None of these is the line's: another operation, a tile, a batch, a point out of regime, and a point whose + # demand includes an in-place upcast the estimate prices as a term of its own. _point("borrower", "encode", 1024, 9000), _point("borrower", "decode", 2048, 1000, tile=512), _point("borrower", "decode", 1024, 9000, batch_size=2), + _point("borrower", "decode", 2048, 90000, upcast_weight_bytes=160 * MIB), MeasuredPoint.model_validate(_point("lender", "decode", 2048, 90000).model_dump() | {"in_fit": False}), ], ) From e3bb17e3cd8b56aec75e5bfce5695ade1c4c522d Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Fri, 23 Oct 2026 03:20:40 +0200 Subject: [PATCH 4/5] fix(working-memory): price SDNQ VAEs at their compute dtype and restore cast weights on offload An SDNQ VAE is priced in its packed weights' compute dtype, and weights a node cast in place go back to the dtype the cache counts when offloaded without a RAM copy. The FLUX.2 calibration script refuses a VAE dtype it has no shipped line for. --- .../cached_model_only_full_load.py | 33 ++++++++++++++- .../cached_model_with_partial_load.py | 11 ++++- .../load/model_cache/tensor_aliases.py | 12 ++++-- invokeai/backend/util/vae_working_memory.py | 20 ++++++--- scripts/calibrate_flux2_working_memory.py | 18 +++++++- .../test_vae_working_memory_backend.py | 17 ++++++++ .../test_cached_model_only_full_load.py | 32 +++++++++++++++ .../test_cached_model_with_partial_load.py | 24 +++++++++++ tests/test_calibrate_flux2_working_memory.py | 41 +++++++++++++++++++ 9 files changed, 196 insertions(+), 12 deletions(-) create mode 100644 tests/test_calibrate_flux2_working_memory.py diff --git a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py index efa0de36d42..dbd0232e416 100644 --- a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py +++ b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py @@ -4,7 +4,12 @@ import torch from invokeai.backend.model_manager.load.model_cache.shared_cpu_weights import SharedCpuWeightsStore -from invokeai.backend.model_manager.load.model_cache.tensor_aliases import StorageKey, analyze_state_dict, storage_key +from invokeai.backend.model_manager.load.model_cache.tensor_aliases import ( + StorageKey, + analyze_state_dict, + move_shared, + storage_key, +) from invokeai.backend.quantization.gguf.ggml_tensor import GGMLTensor @@ -50,6 +55,13 @@ def __init__( self._alias_groups: dict[str, tuple[str, ...]] = ( analyze_state_dict(model.state_dict()).groups if isinstance(model, torch.nn.Module) else {} ) + # The dtype the cache counted each weight in, for an offload without a RAM copy to return it to (see + # `_offload_weights_cast_in_place`). With a RAM copy the offload restores the copy instead. + self._counted_dtypes: dict[str, torch.dtype] | None = ( + {key: tensor.dtype for key, tensor in model.state_dict().items()} + if isinstance(model, torch.nn.Module) and not keep_ram_copy + else None + ) # A CPU read-only copy of the model's state dict. self._cpu_state_dict: dict[str, torch.Tensor] | None = None @@ -196,6 +208,8 @@ def full_unload_from_vram(self) -> int: if self._cpu_state_dict is not None: # The RAM copy still has its tied names pointing at one tensor, so this restores the tie as well. self._model.load_state_dict(self._cpu_state_dict, assign=True) + elif self._counted_dtypes is not None: + self._offload_weights_cast_in_place(self._counted_dtypes) check_for_gguf = hasattr(self._model, "state_dict") and self._model.state_dict().get("img_in.weight") if isinstance(check_for_gguf, GGMLTensor): @@ -210,6 +224,23 @@ def full_unload_from_vram(self) -> int: self._is_in_vram = False return self._total_bytes + def _offload_weights_cast_in_place(self, counted_dtypes: dict[str, torch.dtype]) -> None: + """Move weights a caller cast in place back to the dtype they had when the model was cached, so they again + occupy the `total_bytes` the cache charges for them. + + The SD VAE nodes cast the cached VAE to the dtype they run, fp16 to fp32 for an upcast. The cache charges the + model `total_bytes` for as long as it holds it -- in RAM, and as the room it makes for every later load -- so a + weight kept in its new dtype holds more than that, and the next load overshoots the room by the difference. + """ + moved: dict[StorageKey, torch.Tensor] = {} + restored = { + key: move_shared(tensor, self._offload_device, moved, counted_dtypes[key]) + for key, tensor in self._model.state_dict().items() + if key in counted_dtypes and tensor.dtype != counted_dtypes[key] + } + if restored: + self._model.load_state_dict(restored, strict=False, assign=True) + def _retie_shared_weights(self) -> None: """Re-point the names of a tied weight at one tensor after a device change. diff --git a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py index 44098361340..5fecd800fd8 100644 --- a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py +++ b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py @@ -65,6 +65,10 @@ def __init__( self._aliases = aliases.groups self._state_dict_bytes = aliases.bytes_by_key self._total_bytes = sum(self._state_dict_bytes.values()) + # The dtype each of those byte counts is in. A caller may cast weights in place while the model is cached (the + # SD VAE nodes upcast the VAE to fp32). An offload without a RAM copy casts such weights back to these dtypes, + # as reloading the RAM copy does, so the byte counts stay true. + self._state_dict_dtypes = {key: tensor.dtype for key, tensor in model_state_dict.items()} self._cur_vram_bytes: int | None = None self._modules_that_support_autocast = self._find_modules_that_support_autocast() self._keys_in_modules_that_do_not_support_autocast = self._find_keys_in_modules_that_do_not_support_autocast( @@ -336,7 +340,12 @@ def _load_state_dict_with_jit_device_conversion( if key in keys_to_convert: # It is important that we overwrite `state_dict[key]` to avoid keeping two copies of the same # parameter. - state_dict[key] = move_shared(state_dict[key], target_device, moved) + state_dict[key] = move_shared( + state_dict[key], + target_device, + moved, + self._state_dict_dtypes[key] if target_device.type == "cpu" else None, + ) # Note that we keep parameters that have not been moved to a new device in case the module implements # weird custom state dict loading logic that requires all parameters to be present. module_state_dict[key[prefix_len:]] = state_dict[key] diff --git a/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py b/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py index ebb64cc50e7..53321e574ab 100644 --- a/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py +++ b/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py @@ -68,8 +68,14 @@ def analyze_state_dict(state_dict: dict[str, torch.Tensor]) -> StateDictAliases: return StateDictAliases(bytes_by_key=bytes_by_key, groups=groups) -def move_shared(tensor: torch.Tensor, device: torch.device, moved: dict[StorageKey, torch.Tensor]) -> torch.Tensor: - """Move `tensor` to `device`, reusing the result for every other entry that is the same memory. +def move_shared( + tensor: torch.Tensor, + device: torch.device, + moved: dict[StorageKey, torch.Tensor], + dtype: torch.dtype | None = None, +) -> torch.Tensor: + """Move `tensor` to `device`, and to `dtype` if one is given, reusing the result for every other entry that is the + same memory. `moved` accumulates across one conversion pass, so the tensors stay tied on the target device instead of becoming independent copies. @@ -77,6 +83,6 @@ def move_shared(tensor: torch.Tensor, device: torch.device, moved: dict[StorageK key = storage_key(tensor) already_moved = moved.get(key) if already_moved is None: - already_moved = tensor.to(device) + already_moved = tensor.to(device) if dtype is None or dtype == tensor.dtype else tensor.to(device, dtype) moved[key] = already_moved return already_moved diff --git a/invokeai/backend/util/vae_working_memory.py b/invokeai/backend/util/vae_working_memory.py index 91f2acd4318..51b75a3a248 100644 --- a/invokeai/backend/util/vae_working_memory.py +++ b/invokeai/backend/util/vae_working_memory.py @@ -158,14 +158,24 @@ def _dtype_name(dtype: torch.dtype) -> str: def _compute_dtype(vae: torch.nn.Module) -> torch.dtype: - """The dtype a VAE no node casts computes in: that of its first floating-point parameter wider than a byte. + """The dtype a VAE no node casts computes in. - Narrower or integer weights (fp8 storage, an SDNQ VAE's packed ones) are what a layer dequantizes from, not what - its activations are. + An SDNQ VAE computes in its packed weights' `compute_dtype`: they dequantize to it, and the layer casts every other + operand to it, an fp32 bias included. Otherwise it is that of the first floating-point parameter wider than a byte: + narrower weights (fp8 storage) are what a layer dequantizes from, not what its activations are. + + A packed parameter's `compute_dtype` wins over any floating-point one, so an SDNQ VAE that kept whole layers + unquantized in fp32 would be priced below its real cost; that case is not verified for the shipped SDNQ VAEs. """ + first_wide_float: torch.dtype | None = None for param in vae.parameters(): - if param.is_floating_point() and param.element_size() > 1: - return param.dtype + packed_compute_dtype = getattr(param, "compute_dtype", None) + if isinstance(packed_compute_dtype, torch.dtype): + return packed_compute_dtype + if first_wide_float is None and param.is_floating_point() and param.element_size() > 1: + first_wide_float = param.dtype + if first_wide_float is not None: + return first_wide_float raise ValueError(f"{type(vae).__name__} has no floating-point parameters to compute in.") diff --git a/scripts/calibrate_flux2_working_memory.py b/scripts/calibrate_flux2_working_memory.py index 783fe00d04f..39d52fb7a2f 100644 --- a/scripts/calibrate_flux2_working_memory.py +++ b/scripts/calibrate_flux2_working_memory.py @@ -76,10 +76,18 @@ MIB = 1024**2 +_VAE_LINE_DTYPES = {"float16": "fp16", "bfloat16": "bf16", "float32": "fp32"} + + +def _vae_line_dtypes() -> list[str]: + """The `--dtype` values the FLUX.2 VAE ships a line for: the loaders run it in bf16 or fp32, never fp16.""" + return [name for name, dtype in _VAE_LINE_DTYPES.items() if f"k_decode_{dtype}" in VAE_FLUX2_SPEC.coefficients] + + def _shipped_vae_line(operation: str, dtype_name: str) -> tuple[float, float]: """The VAE line this build's estimate uses for `operation` in `dtype_name`: bytes per pixel and base bytes, as the platform (convolution backend, allocator) selects them.""" - dtype = {"float16": "fp16", "bfloat16": "bf16", "float32": "fp32"}[dtype_name] + dtype = _VAE_LINE_DTYPES[dtype_name] values = shipped_coefficients(VAE_FLUX2_SPEC, platform_for(torch.device("cuda"))).values return values[f"k_{operation}_{dtype}"], values[f"b_{operation}_{dtype}"] @@ -623,6 +631,13 @@ def main() -> None: # Internal: measure one point in this process and print one JSON line. parser.add_argument("--single", nargs="*", default=None, help=argparse.SUPPRESS) args = parser.parse_args() + sections = args.only or ["dispatch", "sdpa", "vae", "denoise"] + if "vae" in sections and args.dtype not in _vae_line_dtypes(): + # Refused up front: the VAE section compares against the shipped line, and runs after dispatch and sdpa. + parser.error( + f"--dtype {args.dtype} has no FLUX.2 VAE line to measure against; the VAE runs in " + f"{', '.join(_vae_line_dtypes())}. Pick one of those, or leave the VAE out with --only." + ) if not torch.cuda.is_available(): raise SystemExit("No CUDA/HIP device available.") @@ -650,7 +665,6 @@ def main() -> None: raise SystemExit(f"unknown point kind {kind}") return - sections = args.only or ["dispatch", "sdpa", "vae", "denoise"] print( f"torch {torch.__version__} | device {torch.cuda.get_device_name(0)} | " f"hip={torch.version.hip} | dtype={args.dtype}" diff --git a/tests/app/invocations/test_vae_working_memory_backend.py b/tests/app/invocations/test_vae_working_memory_backend.py index 63f8f092335..8f6721d8e02 100644 --- a/tests/app/invocations/test_vae_working_memory_backend.py +++ b/tests/app/invocations/test_vae_working_memory_backend.py @@ -15,6 +15,8 @@ from invokeai.app.invocations.vae.image_to_latents import ImageToLatentsInvocation from invokeai.app.invocations.vae.latents_to_image import LatentsToImageInvocation +from invokeai.backend.quantization.sdnq.sdnq_tensor import SDNQTensor +from invokeai.backend.quantization.sdnq.utils import SDNQQuantizationType from invokeai.backend.util.vae_working_memory import ( VAE_COGVIEW4_SPEC, VAE_FLUX2_SPEC, @@ -308,6 +310,21 @@ def test_the_compute_dtype_skips_weights_stored_narrower_than_they_compute(on, l assert estimate_vae_working_memory_flux("decode", LATENTS_1024, vae, device=device).features["dtype"] == dtype +def test_an_sdnq_vae_is_priced_in_its_packed_weights_compute_dtype_not_an_fp32_bias(on) -> None: + """The packed weight dequantizes to its `compute_dtype`, and every other operand of its layer is cast to that dtype + with it, so a bias stored in fp32 does not make the layer compute in fp32.""" + device = on(Platform("cudnn", "native", "linux")) + packed_weight = SDNQTensor( + data=torch.zeros(4, 4, dtype=torch.int8), + quantization_type=SDNQQuantizationType.INT8_SYM, + tensor_shape=torch.Size([4, 4]), + compute_dtype=torch.bfloat16, + scale=torch.ones(4, 1, dtype=torch.float32), + ) + vae = _vae(params=[packed_weight, torch.zeros(4, dtype=torch.float32)]) + assert estimate_vae_working_memory_flux("decode", LATENTS_1024, vae, device=device).features["dtype"] == "bf16" + + def test_every_stand_in_slope_is_at_least_the_pre_dtype_figure() -> None: """A stand-in is a declared conservative value on a measured platform; nothing else guards it.""" stand_ins = [entry for data in shipped_data().values() for entry in data.coefficients if entry.stand_ins] diff --git a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py index 3c193863506..eea2b55df64 100644 --- a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py +++ b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py @@ -3,6 +3,7 @@ from invokeai.backend.model_manager.load.model_cache.cached_model.cached_model_only_full_load import ( CachedModelOnlyFullLoad, ) +from invokeai.backend.util.calc_tensor_size import calc_tensor_size from tests.backend.model_manager.load.model_cache.cached_model.utils import ( DummyModule, TiedWeightsModule, @@ -106,6 +107,37 @@ def test_a_tied_weight_survives_a_device_round_trip(device: str, keep_ram_copy: assert model.head.weight.data_ptr() == model.embed.weight.data_ptr() +def _resident_bytes(model: torch.nn.Module) -> int: + """The bytes the model's weights hold, each storage once however many names share it.""" + return sum({t.data_ptr(): calc_tensor_size(t) for t in model.state_dict().values()}.values()) + + +@parameterize_keep_ram_copy +def test_an_offload_returns_weights_cast_in_place_to_the_dtype_the_cache_counts(keep_ram_copy: bool): + """The SD VAE nodes cast the cached VAE in place, fp16 to fp32 for an upcast decode. The cache counts the model at + the size it was added with, so an offload has to leave it at that size: otherwise the next load puts more on the + device than the cache made room for, and the VAE estimate, seeing fp32 weights, reserves nothing for the cast. + + The compute device is the CPU here so this runs on any machine; the offload path is the same. + """ + model = TiedWeightsModule().to(torch.float16) + total_bytes = _resident_bytes(model) + cached_model = CachedModelOnlyFullLoad( + model=model, compute_device=torch.device("cpu"), total_bytes=total_bytes, keep_ram_copy=keep_ram_copy + ) + cached_model.full_load_to_vram() + model.to(dtype=torch.float32) + + cached_model.full_unload_from_vram() + + assert {p.dtype for p in model.parameters()} == {torch.float16} + assert _resident_bytes(model) == cached_model.total_bytes() + assert model.head.weight.data_ptr() == model.embed.weight.data_ptr() + + cached_model.full_load_to_vram() + assert _resident_bytes(model) == cached_model.cur_vram_bytes() + + @parameterize_mps_and_cuda def test_cached_model_get_cpu_state_dict(device: str): model = DummyModule() diff --git a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py index 50e77c7f72c..b76ca4b16a7 100644 --- a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py +++ b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py @@ -278,6 +278,30 @@ def test_cached_model_full_load_and_unload(device: str, model: DummyModule, keep ) +@parameterize_mps_and_cuda +@parameterize_keep_ram_copy +def test_an_offload_returns_weights_cast_in_place_to_the_dtype_the_cache_counts( + device: str, model: DummyModule, keep_ram_copy: bool +): + """The SD VAE nodes cast the cached VAE in place, fp16 to fp32 for an upcast decode. The cache counts each weight + at the size it was added with, so an offload has to leave it at that size: otherwise the next load puts more on the + device than the cache made room for, and the VAE estimate, seeing fp32 weights, reserves nothing for the cast.""" + model.to(torch.float16) + cached_model = CachedModelWithPartialLoad( + model=model, compute_device=torch.device(device), keep_ram_copy=keep_ram_copy + ) + cached_model.full_load_to_vram() + model.to(dtype=torch.float32) + + cached_model.full_unload_from_vram() + + assert {t.dtype for t in model.state_dict().values()} == {torch.float16} + assert sum(calc_tensor_size(t) for t in model.state_dict().values()) == cached_model.total_bytes() + + cached_model.full_load_to_vram() + assert sum(calc_tensor_size(t) for t in model.state_dict().values()) == cached_model.cur_vram_bytes() + + @parameterize_mps_and_cuda @parameterize_keep_ram_copy def test_cached_model_full_load_from_partial(device: str, model: DummyModule, keep_ram_copy: bool): diff --git a/tests/test_calibrate_flux2_working_memory.py b/tests/test_calibrate_flux2_working_memory.py new file mode 100644 index 00000000000..3045bb1ddcc --- /dev/null +++ b/tests/test_calibrate_flux2_working_memory.py @@ -0,0 +1,41 @@ +import sys +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest +import torch + + +def _load_calibration_script(): + path = Path(__file__).parents[1] / "scripts" / "calibrate_flux2_working_memory.py" + spec = spec_from_file_location("calibrate_flux2_working_memory", path) + assert spec is not None and spec.loader is not None + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _run(monkeypatch, *args: str): + script = _load_calibration_script() + # Off the device either way, so an accepted command line stops at the device check instead of measuring. + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + monkeypatch.setattr(sys, "argv", ["calibrate_flux2_working_memory.py", *args]) + with pytest.raises(SystemExit) as exit_info: + script.main() + return exit_info.value.code + + +@pytest.mark.parametrize("sections", [["--only", "vae"], []], ids=["only-vae", "all-sections"]) +def test_a_vae_dtype_with_no_shipped_line_is_refused_before_anything_runs(monkeypatch, capsys, sections) -> None: + """The FLUX.2 VAE ships bf16 and fp32 lines only; an fp16 run would measure everything else first and then fail + on a bare `KeyError` when it compared against the line.""" + assert _run(monkeypatch, *sections, "--dtype", "float16") == 2 + err = capsys.readouterr().err + assert "--dtype float16" in err + assert "bfloat16, float32" in err + + +def test_float16_is_accepted_for_the_sections_that_do_not_compare_against_a_vae_line(monkeypatch) -> None: + assert _run(monkeypatch, "--only", "sdpa", "--only", "denoise", "--dtype", "float16") == ( + "No CUDA/HIP device available." + ) From 47eac035028d5c1feff89731dadcb3b7464940b8 Mon Sep 17 00:00:00 2001 From: Alexander Eichhorn Date: Sat, 24 Oct 2026 04:12:43 +0200 Subject: [PATCH 5/5] fix(model-cache): return weights cast in place to their counted dtype at the last unlock The last unlock re-points every weight a holder cast in place (the SD VAE upcast) back to the dtype the cache counts, on its device, keeping ties and the RAM copy, without load_state_dict. Loads and offloads then never move cast weights, and the SD VAE estimate prices the full upcast on every run. --- .../cached_model_only_full_load.py | 37 +-- .../cached_model_with_partial_load.py | 22 +- .../load/model_cache/counted_dtypes.py | 48 +++ .../load/model_cache/model_cache.py | 29 +- .../load/model_cache/tensor_aliases.py | 12 +- invokeai/backend/util/vae_working_memory.py | 13 +- .../test_vae_working_memory_backend.py | 91 ++++-- .../test_cached_model_only_full_load.py | 32 -- .../test_cached_model_with_partial_load.py | 24 -- .../test_model_cache_unlock_restores_casts.py | 304 ++++++++++++++++++ 10 files changed, 485 insertions(+), 127 deletions(-) create mode 100644 invokeai/backend/model_manager/load/model_cache/counted_dtypes.py create mode 100644 tests/backend/model_manager/load/model_cache/test_model_cache_unlock_restores_casts.py diff --git a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py index dbd0232e416..18a36bf996f 100644 --- a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py +++ b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_only_full_load.py @@ -3,13 +3,9 @@ import torch +from invokeai.backend.model_manager.load.model_cache.counted_dtypes import return_to_counted_dtypes from invokeai.backend.model_manager.load.model_cache.shared_cpu_weights import SharedCpuWeightsStore -from invokeai.backend.model_manager.load.model_cache.tensor_aliases import ( - StorageKey, - analyze_state_dict, - move_shared, - storage_key, -) +from invokeai.backend.model_manager.load.model_cache.tensor_aliases import StorageKey, analyze_state_dict, storage_key from invokeai.backend.quantization.gguf.ggml_tensor import GGMLTensor @@ -55,11 +51,11 @@ def __init__( self._alias_groups: dict[str, tuple[str, ...]] = ( analyze_state_dict(model.state_dict()).groups if isinstance(model, torch.nn.Module) else {} ) - # The dtype the cache counted each weight in, for an offload without a RAM copy to return it to (see - # `_offload_weights_cast_in_place`). With a RAM copy the offload restores the copy instead. + # The dtype the cache counted each weight in, which the last unlock returns weights a holder cast in place to + # (see `restore_counted_dtypes`). None for models that are not `nn.Module`s. self._counted_dtypes: dict[str, torch.dtype] | None = ( {key: tensor.dtype for key, tensor in model.state_dict().items()} - if isinstance(model, torch.nn.Module) and not keep_ram_copy + if isinstance(model, torch.nn.Module) else None ) @@ -208,8 +204,6 @@ def full_unload_from_vram(self) -> int: if self._cpu_state_dict is not None: # The RAM copy still has its tied names pointing at one tensor, so this restores the tie as well. self._model.load_state_dict(self._cpu_state_dict, assign=True) - elif self._counted_dtypes is not None: - self._offload_weights_cast_in_place(self._counted_dtypes) check_for_gguf = hasattr(self._model, "state_dict") and self._model.state_dict().get("img_in.weight") if isinstance(check_for_gguf, GGMLTensor): @@ -224,22 +218,15 @@ def full_unload_from_vram(self) -> int: self._is_in_vram = False return self._total_bytes - def _offload_weights_cast_in_place(self, counted_dtypes: dict[str, torch.dtype]) -> None: - """Move weights a caller cast in place back to the dtype they had when the model was cached, so they again - occupy the `total_bytes` the cache charges for them. + def restore_counted_dtypes(self) -> None: + """Return every weight a holder cast in place to the dtype the cache counts it in. - The SD VAE nodes cast the cached VAE to the dtype they run, fp16 to fp32 for an upcast. The cache charges the - model `total_bytes` for as long as it holds it -- in RAM, and as the room it makes for every later load -- so a - weight kept in its new dtype holds more than that, and the next load overshoots the room by the difference. + `ModelCache.unlock` calls this when the last lock ends; see `return_to_counted_dtypes`. The cache charges the + model `total_bytes` in RAM and as the room it makes in VRAM, and a `cpu_only` model is never moved, so nothing + else would put it back. """ - moved: dict[StorageKey, torch.Tensor] = {} - restored = { - key: move_shared(tensor, self._offload_device, moved, counted_dtypes[key]) - for key, tensor in self._model.state_dict().items() - if key in counted_dtypes and tensor.dtype != counted_dtypes[key] - } - if restored: - self._model.load_state_dict(restored, strict=False, assign=True) + if self._counted_dtypes is not None: + return_to_counted_dtypes(self._model, self._counted_dtypes, self._alias_groups, self._cpu_state_dict) def _retie_shared_weights(self) -> None: """Re-point the names of a tied weight at one tensor after a device change. diff --git a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py index 5fecd800fd8..84e3e51468b 100644 --- a/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py +++ b/invokeai/backend/model_manager/load/model_cache/cached_model/cached_model_with_partial_load.py @@ -4,6 +4,7 @@ import torch +from invokeai.backend.model_manager.load.model_cache.counted_dtypes import return_to_counted_dtypes from invokeai.backend.model_manager.load.model_cache.shared_cpu_weights import SharedCpuWeightsStore from invokeai.backend.model_manager.load.model_cache.tensor_aliases import ( StorageKey, @@ -65,9 +66,9 @@ def __init__( self._aliases = aliases.groups self._state_dict_bytes = aliases.bytes_by_key self._total_bytes = sum(self._state_dict_bytes.values()) - # The dtype each of those byte counts is in. A caller may cast weights in place while the model is cached (the - # SD VAE nodes upcast the VAE to fp32). An offload without a RAM copy casts such weights back to these dtypes, - # as reloading the RAM copy does, so the byte counts stay true. + # The dtype each of those byte counts is in. A holder may cast weights in place (the SD VAE nodes upcast the + # VAE to fp32); the last unlock returns them to these dtypes (see `restore_counted_dtypes`), so between locks + # the bytes on each device are what `cur_vram_bytes()` and `total_bytes()` report. self._state_dict_dtypes = {key: tensor.dtype for key, tensor in model_state_dict.items()} self._cur_vram_bytes: int | None = None self._modules_that_support_autocast = self._find_modules_that_support_autocast() @@ -295,6 +296,14 @@ def repair_required_tensors_on_compute_device(self) -> int: self._cur_vram_bytes = None return repaired_required_tensors + def restore_counted_dtypes(self) -> None: + """Return every weight a holder cast in place to the dtype `cur_vram_bytes()` and `total_bytes()` count it in. + + `ModelCache.unlock` calls this when the last lock ends; see `return_to_counted_dtypes`. No weight changes device, + so a VRAM count already taken stays true. + """ + return_to_counted_dtypes(self._model, self._state_dict_dtypes, self._aliases, self._cpu_state_dict) + def _load_state_dict_with_device_conversion( self, state_dict: dict[str, torch.Tensor], keys_to_convert: set[str], target_device: torch.device ): @@ -340,12 +349,7 @@ def _load_state_dict_with_jit_device_conversion( if key in keys_to_convert: # It is important that we overwrite `state_dict[key]` to avoid keeping two copies of the same # parameter. - state_dict[key] = move_shared( - state_dict[key], - target_device, - moved, - self._state_dict_dtypes[key] if target_device.type == "cpu" else None, - ) + state_dict[key] = move_shared(state_dict[key], target_device, moved) # Note that we keep parameters that have not been moved to a new device in case the module implements # weird custom state dict loading logic that requires all parameters to be present. module_state_dict[key[prefix_len:]] = state_dict[key] diff --git a/invokeai/backend/model_manager/load/model_cache/counted_dtypes.py b/invokeai/backend/model_manager/load/model_cache/counted_dtypes.py new file mode 100644 index 00000000000..4d25be1eb11 --- /dev/null +++ b/invokeai/backend/model_manager/load/model_cache/counted_dtypes.py @@ -0,0 +1,48 @@ +"""Returning weights a holder cast in place to the dtypes the model cache counts them in.""" + +import torch + + +@torch.no_grad() +def return_to_counted_dtypes( + model: torch.nn.Module, + counted_dtypes: dict[str, torch.dtype], + alias_groups: dict[str, tuple[str, ...]], + cpu_state_dict: dict[str, torch.Tensor] | None, +) -> None: + """Return every weight of `model` whose dtype differs from `counted_dtypes` to that dtype, on the device it is on. + + `counted_dtypes` maps each state dict key to the dtype the cache counted it in, and `alias_groups` names the keys of + each tied weight (see `tensor_aliases`). A holder may cast the model it holds (the SD VAE nodes upcast the VAE to + fp32), and `Module.to(dtype)` casts each name of a tied weight on its own, so the cast unties it. Each group gets one + tensor again under all of its names; on the CPU that is the RAM copy in `cpu_state_dict` when there is one. + + The tensors are re-pointed through `Parameter.data` and `Module._buffers`, never `load_state_dict(assign=True)`: + assignment goes through `register_parameter`, which a construction on another thread + (`accelerate.init_empty_weights` patches it process-wide) would route to the meta device, and this runs without + MODEL_LOAD_LOCK. Parameter objects and devices stay as they are, and one weight at a time holds both dtypes. Packed + tensors (GGUF, SDNQ) refuse a dtype change, so a cast never changed theirs and nothing here selects them. + """ + restored: dict[str, torch.Tensor] = {} + # A module registered under two names is visited under both, as `state_dict()` lists it under both. + for module_name, module in model.named_modules(remove_duplicate=False): + prefix = f"{module_name}." if module_name else "" + for slots in (module._parameters, module._buffers): + for name, tensor in slots.items(): + key = prefix + name + counted_dtype = counted_dtypes.get(key) + # Keys outside the state dict (non-persistent buffers) are not counted. + if tensor is None or counted_dtype is None or tensor.dtype == counted_dtype: + continue + group = alias_groups.get(key, (key,))[0] + new = restored.get(group) + if new is None: + if tensor.device.type == "cpu" and cpu_state_dict is not None: + new = cpu_state_dict[key] + else: + new = tensor.to(dtype=counted_dtype) + restored[group] = new + if isinstance(tensor, torch.nn.Parameter): + tensor.data = new + else: + slots[name] = new diff --git a/invokeai/backend/model_manager/load/model_cache/model_cache.py b/invokeai/backend/model_manager/load/model_cache/model_cache.py index 4127ddddb12..6cf115257dd 100644 --- a/invokeai/backend/model_manager/load/model_cache/model_cache.py +++ b/invokeai/backend/model_manager/load/model_cache/model_cache.py @@ -1998,7 +1998,11 @@ def continue_lock( @synchronized @record_activity def unlock(self, cache_entry: CacheRecord) -> None: - """Unlock a model.""" + """Unlock a model. + + The last unlock returns weights a holder cast in place to the dtypes the cache counts them in (see + `_restore_counted_dtypes`). + """ if cache_entry.key not in self._cached_models: self._logger.info( f"Unlocking model cache entry {cache_entry.key} " @@ -2008,6 +2012,8 @@ def unlock(self, cache_entry: CacheRecord) -> None: ) # cache_entry = self._cached_models[key] cache_entry.unlock() + if not cache_entry.is_locked: + self._restore_counted_dtypes(cache_entry) self._logger.debug( f"Unlocked model {cache_entry.key} (Type: {cache_entry.cached_model.model.__class__.__name__})" ) @@ -2053,6 +2059,27 @@ def unlock(self, cache_entry: CacheRecord) -> None: if self._ram_budget is not None and self._ram_budget.available() < 0: self._budget_reconcile_pending.set() + def _restore_counted_dtypes(self, cache_entry: CacheRecord) -> None: + """Return the weights the entry's holders cast in place to the dtypes the cache counts them in. + + The SD VAE nodes cast the cached VAE to the dtype they run, fp16 to fp32 for an upcast. The cache counts every + weight in the dtype it was cached with, and the next decode's estimate prices its upcast from the dtype the VAE + holds before its lock, so a VAE left in fp32 holds more than the cache charges and gets nothing reserved for the + next cast. Only the last unlock restores: until then another holder may be running the model in its cast dtype. + The passes do not restore on purpose; only an offload with a RAM copy puts the RAM copy back for the weights it + moves, and with it their counted dtype, even while the model is held, as it always has. + """ + try: + cache_entry.cached_model.restore_counted_dtypes() + except Exception as e: + # As after a failed move: partly restored, the model no longer matches its counts, and dropping it contains + # that. Not raised: the unlock runs in the `finally` of a finished run, whose result is not saved yet. + self._logger.warning( + f"Could not return the weights of '{cache_entry.key}' to the dtypes the cache counts them in ({e}); " + "dropped it from the cache." + ) + self._delete_cache_entry(cache_entry) + def _load_locked_model( self, cache_entry: CacheRecord, diff --git a/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py b/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py index 53321e574ab..ebb64cc50e7 100644 --- a/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py +++ b/invokeai/backend/model_manager/load/model_cache/tensor_aliases.py @@ -68,14 +68,8 @@ def analyze_state_dict(state_dict: dict[str, torch.Tensor]) -> StateDictAliases: return StateDictAliases(bytes_by_key=bytes_by_key, groups=groups) -def move_shared( - tensor: torch.Tensor, - device: torch.device, - moved: dict[StorageKey, torch.Tensor], - dtype: torch.dtype | None = None, -) -> torch.Tensor: - """Move `tensor` to `device`, and to `dtype` if one is given, reusing the result for every other entry that is the - same memory. +def move_shared(tensor: torch.Tensor, device: torch.device, moved: dict[StorageKey, torch.Tensor]) -> torch.Tensor: + """Move `tensor` to `device`, reusing the result for every other entry that is the same memory. `moved` accumulates across one conversion pass, so the tensors stay tied on the target device instead of becoming independent copies. @@ -83,6 +77,6 @@ def move_shared( key = storage_key(tensor) already_moved = moved.get(key) if already_moved is None: - already_moved = tensor.to(device) if dtype is None or dtype == tensor.dtype else tensor.to(device, dtype) + already_moved = tensor.to(device) moved[key] = already_moved return already_moved diff --git a/invokeai/backend/util/vae_working_memory.py b/invokeai/backend/util/vae_working_memory.py index 51b75a3a248..f1b0e841ca9 100644 --- a/invokeai/backend/util/vae_working_memory.py +++ b/invokeai/backend/util/vae_working_memory.py @@ -275,8 +275,8 @@ def sd_vae_run_dtype(operation: Literal["encode", "decode"], fp32: bool, device: `estimate_vae_working_memory_sd15_sdxl` prices it, so the two cannot drift. fp32 when the node asks for it, and for a decode on the CPU; fp16 otherwise, on every GPU and for every - `precision`, so bf16 never runs. The cached VAE's own dtype says nothing about it: it is whatever the previous run - cast it to. + `precision`, so bf16 never runs. The cached VAE's own dtype says nothing about it: it is the one the VAE was loaded + in, which the model cache returns it to when the node's lock ends. """ if fp32 or (operation == "decode" and device.type == "cpu"): return torch.float32 @@ -287,10 +287,11 @@ def _upcast_weight_bytes(vae: torch.nn.Module, run_dtype: torch.dtype) -> int: """What the node's in-place cast to `run_dtype` adds to the VAE's resident weights. The cast runs once the cache has placed the VAE, so the growth comes out of the reservation: 160 MiB for the 83.7M - parameters of the SD1/SDXL VAE going from fp16 to fp32, nothing for a VAE a previous run left in fp32. Measured - points exclude resident weights by construction (they record 0 here), so the fitted line does not cover it. An - upcast encode casts the decoder's mid-block and post-quant conv back, so for it this counts a little more than - grows: conservative. + parameters of the SD1/SDXL VAE going from fp16 to fp32, nothing for a VAE loaded in fp32. Read before the lock, + the VAE holds the dtype it was loaded in: the model cache returns it there when the previous node's lock ended. + Measured points exclude resident weights by construction (they record 0 here), so the fitted line does not cover + it. An upcast encode casts the decoder's mid-block and post-quant conv back, so for it this counts a little more + than grows: conservative. """ return sum(p.numel() * max(0, run_dtype.itemsize - p.element_size()) for p in vae.parameters()) diff --git a/tests/app/invocations/test_vae_working_memory_backend.py b/tests/app/invocations/test_vae_working_memory_backend.py index 8f6721d8e02..d3c96e1403b 100644 --- a/tests/app/invocations/test_vae_working_memory_backend.py +++ b/tests/app/invocations/test_vae_working_memory_backend.py @@ -6,6 +6,7 @@ whatever dtype the cached VAE holds -- so it is pinned here rather than left to the card. """ +import logging from unittest.mock import MagicMock, patch import pytest @@ -15,6 +16,8 @@ from invokeai.app.invocations.vae.image_to_latents import ImageToLatentsInvocation from invokeai.app.invocations.vae.latents_to_image import LatentsToImageInvocation +from invokeai.backend.model_manager.load.load_base import LoadedModel +from invokeai.backend.model_manager.load.model_cache.model_cache import ModelCache from invokeai.backend.quantization.sdnq.sdnq_tensor import SDNQTensor from invokeai.backend.quantization.sdnq.utils import SDNQQuantizationType from invokeai.backend.util.vae_working_memory import ( @@ -186,7 +189,7 @@ def test_the_cogview4_vae_is_priced_on_its_own_lines_without_a_score_matrix(on) def test_the_sd_estimate_prices_the_dtype_the_node_casts_to_not_the_cached_one( on, operation, fp32, device, run_dtype, vae_dtype ) -> None: - """The cached VAE holds whatever the previous run cast it to; the node casts it again before it runs.""" + """The node casts the VAE from the dtype it was loaded in to the one it runs, and prices the latter.""" on(Platform("cudnn", "native", "linux")) assert sd_vae_run_dtype(operation, fp32, device) == run_dtype image = LATENTS_1024[:, :4] if operation == "decode" else torch.zeros(1, 3, 1024, 1024) @@ -205,41 +208,74 @@ def _tiny_sd_vae() -> AutoencoderKL: ) -def _vae_info(vae: AutoencoderKL, estimates: list) -> MagicMock: - def model_on_device(working_memory): - estimates.append(working_memory) - cm = MagicMock() - cm.__enter__.return_value = (None, vae) - cm.__exit__.return_value = None - return cm +@pytest.fixture +def cached_vae(): + """The VAE as a node loads it: cached for the CPU, locked and unlocked by the model cache. Each lock's working + memory is recorded.""" + caches: list[ModelCache] = [] + + def load(vae: AutoencoderKL, estimates: list) -> LoadedModel: + logger = MagicMock() + logger.getEffectiveLevel.return_value = logging.INFO + cache = ModelCache( + execution_device_working_mem_gb=1.0, + enable_partial_loading=True, + keep_ram_copy_of_weights=True, + execution_device="cpu", + storage_device="cpu", + logger=logger, + ) + caches.append(cache) + cache.put("vae", vae) + vae_info = LoadedModel(config=None, cache_record=cache.get("vae"), cache=cache) + model_on_device = vae_info.model_on_device + + def recording_model_on_device(working_memory): + estimates.append(working_memory) + return model_on_device(working_memory=working_memory) + + vae_info.model_on_device = recording_model_on_device # type: ignore[method-assign] + return vae_info - vae_info = MagicMock(model=vae, compute_device=torch.device("cpu")) - vae_info.model_on_device.side_effect = model_on_device - return vae_info + yield load + for cache in caches: + cache.shutdown() @pytest.mark.parametrize("fp32", [False, True]) -def test_the_decode_node_casts_the_vae_to_the_dtype_its_estimate_priced(fp32) -> None: - """On the CPU both requests decode in fp32: the node and the estimate share `sd_vae_run_dtype`.""" - vae, estimates = _tiny_sd_vae().to(torch.bfloat16), [] +def test_every_decode_runs_the_dtype_its_estimate_priced_and_prices_the_same_upcast(cached_vae, fp32) -> None: + """On the CPU both requests decode in fp32: the node and the estimate share `sd_vae_run_dtype`. The cache returns the + VAE to bf16 when a decode's lock ends, so the next decode's estimate prices the same upcast: reading a VAE left in + fp32, it reserved nothing for the cast.""" + vae, estimates, decoded_in = _tiny_sd_vae().to(torch.bfloat16), [], [] + decode = vae.decode + + def recording_decode(*args, **kwargs): + decoded_in.append(next(vae.decoder.parameters()).dtype) + return decode(*args, **kwargs) + context = MagicMock() context.tensors.load.return_value = torch.zeros(1, 4, 8, 8) context.config.get.return_value.force_tiled_decode = False - context.models.load.return_value = _vae_info(vae, estimates) + context.models.load.return_value = cached_vae(vae, estimates) context.images.save.return_value = MagicMock(image_name="x.png", width=8, height=8) node = LatentsToImageInvocation.model_construct( latents=MagicMock(latents_name="x"), vae=MagicMock(seamless_axes=[]), tiled=False, tile_size=0, fp32=fp32 ) - node.invoke(context) + with patch.object(vae, "decode", side_effect=recording_decode): + node.invoke(context) + node.invoke(context) - (estimate,) = estimates - assert estimate.features["dtype"] == "fp32" - assert {p.dtype for p in vae.parameters()} == {torch.float32} + assert decoded_in == [torch.float32, torch.float32] + assert [estimate.features["dtype"] for estimate in estimates] == ["fp32", "fp32"] + assert estimates[1].features["upcast_weight_bytes"] == estimates[0].features["upcast_weight_bytes"] > 0 @pytest.mark.parametrize("upcast, run_dtype", [(False, torch.float16), (True, torch.float32)]) -def test_the_encode_node_casts_the_vae_to_the_dtype_its_estimate_priced(upcast, run_dtype) -> None: +def test_the_encode_node_runs_the_dtype_its_estimate_priced_and_returns_latents_in_the_vaes( + cached_vae, upcast, run_dtype +) -> None: vae, estimates, encoded_in = _tiny_sd_vae().to(torch.bfloat16), [], [] def encode(vae, image): @@ -247,11 +283,24 @@ def encode(vae, image): return torch.zeros(1, 4, 2, 2) with patch.object(ImageToLatentsInvocation, "_encode_to_tensor", side_effect=encode): - ImageToLatentsInvocation.vae_encode(_vae_info(vae, estimates), upcast, False, torch.zeros(1, 3, 16, 16)) + latents = ImageToLatentsInvocation.vae_encode( + cached_vae(vae, estimates), upcast, False, torch.zeros(1, 3, 16, 16) + ) (estimate,) = estimates assert encoded_in == [run_dtype] assert estimate.features["dtype"] == {torch.float16: "fp16", torch.float32: "fp32"}[run_dtype] + assert latents.dtype == torch.bfloat16 + + +def test_a_failed_encode_leaves_the_vae_in_the_dtype_it_was_loaded_in(cached_vae) -> None: + vae = _tiny_sd_vae().to(torch.bfloat16) + + with patch.object(ImageToLatentsInvocation, "_encode_to_tensor", side_effect=RuntimeError("encode failed")): + with pytest.raises(RuntimeError, match="encode failed"): + ImageToLatentsInvocation.vae_encode(cached_vae(vae, []), True, False, torch.zeros(1, 3, 16, 16)) + + assert {p.dtype for p in vae.parameters()} == {torch.bfloat16} @pytest.mark.parametrize( diff --git a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py index eea2b55df64..3c193863506 100644 --- a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py +++ b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_only_full_load.py @@ -3,7 +3,6 @@ from invokeai.backend.model_manager.load.model_cache.cached_model.cached_model_only_full_load import ( CachedModelOnlyFullLoad, ) -from invokeai.backend.util.calc_tensor_size import calc_tensor_size from tests.backend.model_manager.load.model_cache.cached_model.utils import ( DummyModule, TiedWeightsModule, @@ -107,37 +106,6 @@ def test_a_tied_weight_survives_a_device_round_trip(device: str, keep_ram_copy: assert model.head.weight.data_ptr() == model.embed.weight.data_ptr() -def _resident_bytes(model: torch.nn.Module) -> int: - """The bytes the model's weights hold, each storage once however many names share it.""" - return sum({t.data_ptr(): calc_tensor_size(t) for t in model.state_dict().values()}.values()) - - -@parameterize_keep_ram_copy -def test_an_offload_returns_weights_cast_in_place_to_the_dtype_the_cache_counts(keep_ram_copy: bool): - """The SD VAE nodes cast the cached VAE in place, fp16 to fp32 for an upcast decode. The cache counts the model at - the size it was added with, so an offload has to leave it at that size: otherwise the next load puts more on the - device than the cache made room for, and the VAE estimate, seeing fp32 weights, reserves nothing for the cast. - - The compute device is the CPU here so this runs on any machine; the offload path is the same. - """ - model = TiedWeightsModule().to(torch.float16) - total_bytes = _resident_bytes(model) - cached_model = CachedModelOnlyFullLoad( - model=model, compute_device=torch.device("cpu"), total_bytes=total_bytes, keep_ram_copy=keep_ram_copy - ) - cached_model.full_load_to_vram() - model.to(dtype=torch.float32) - - cached_model.full_unload_from_vram() - - assert {p.dtype for p in model.parameters()} == {torch.float16} - assert _resident_bytes(model) == cached_model.total_bytes() - assert model.head.weight.data_ptr() == model.embed.weight.data_ptr() - - cached_model.full_load_to_vram() - assert _resident_bytes(model) == cached_model.cur_vram_bytes() - - @parameterize_mps_and_cuda def test_cached_model_get_cpu_state_dict(device: str): model = DummyModule() diff --git a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py index b76ca4b16a7..50e77c7f72c 100644 --- a/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py +++ b/tests/backend/model_manager/load/model_cache/cached_model/test_cached_model_with_partial_load.py @@ -278,30 +278,6 @@ def test_cached_model_full_load_and_unload(device: str, model: DummyModule, keep ) -@parameterize_mps_and_cuda -@parameterize_keep_ram_copy -def test_an_offload_returns_weights_cast_in_place_to_the_dtype_the_cache_counts( - device: str, model: DummyModule, keep_ram_copy: bool -): - """The SD VAE nodes cast the cached VAE in place, fp16 to fp32 for an upcast decode. The cache counts each weight - at the size it was added with, so an offload has to leave it at that size: otherwise the next load puts more on the - device than the cache made room for, and the VAE estimate, seeing fp32 weights, reserves nothing for the cast.""" - model.to(torch.float16) - cached_model = CachedModelWithPartialLoad( - model=model, compute_device=torch.device(device), keep_ram_copy=keep_ram_copy - ) - cached_model.full_load_to_vram() - model.to(dtype=torch.float32) - - cached_model.full_unload_from_vram() - - assert {t.dtype for t in model.state_dict().values()} == {torch.float16} - assert sum(calc_tensor_size(t) for t in model.state_dict().values()) == cached_model.total_bytes() - - cached_model.full_load_to_vram() - assert sum(calc_tensor_size(t) for t in model.state_dict().values()) == cached_model.cur_vram_bytes() - - @parameterize_mps_and_cuda @parameterize_keep_ram_copy def test_cached_model_full_load_from_partial(device: str, model: DummyModule, keep_ram_copy: bool): diff --git a/tests/backend/model_manager/load/model_cache/test_model_cache_unlock_restores_casts.py b/tests/backend/model_manager/load/model_cache/test_model_cache_unlock_restores_casts.py new file mode 100644 index 00000000000..81f730ee9f6 --- /dev/null +++ b/tests/backend/model_manager/load/model_cache/test_model_cache_unlock_restores_casts.py @@ -0,0 +1,304 @@ +"""The last `ModelCache.unlock` returns weights a holder cast in place to the dtypes the cache counts them in. + +The SD VAE nodes cast the cached VAE to the dtype they run, fp16 to fp32 for an upcast, and the cast reaches the weights +on the CPU as well as those on the compute device. The cache counts every weight in the dtype it was cached with, so a +weight left in fp32 holds more than the cache charges: the next load moves it into VRAM past the room the cache made, +and the next decode's estimate, reading an fp32 VAE, reserves nothing for its upcast. + +Partial loading runs on dedicated VRAM only. The meta device stands in for it here so the tests run on any machine: it +keeps every tensor's dtype and size but no data, so values are compared wherever a weight holds data. +""" + +import logging +import threading +from unittest.mock import MagicMock, patch + +import accelerate +import pytest +import torch + +from invokeai.backend.model_manager.load.model_cache import model_cache as model_cache_module +from invokeai.backend.model_manager.load.model_cache.cached_model.cached_model_only_full_load import ( + CachedModelOnlyFullLoad, +) +from invokeai.backend.model_manager.load.model_cache.cached_model.cached_model_with_partial_load import ( + CachedModelWithPartialLoad, +) +from invokeai.backend.model_manager.load.model_cache.model_cache import ModelCache +from invokeai.backend.util.calc_tensor_size import calc_tensor_size +from tests.backend.model_manager.load.model_cache.cached_model.utils import TiedWeightsModule + +devices = pytest.mark.parametrize( + "device", + [ + "meta", + pytest.param("cuda", marks=pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is not available.")), + ], +) +keep_ram_copy = pytest.mark.parametrize("keep_ram_copy", [True, False]) +partial_loading = pytest.mark.parametrize("partial_loading", [True, False], ids=["partial", "full"]) + + +@pytest.fixture +def make_cache(): + caches: list[ModelCache] = [] + + def make(keep_ram_copy: bool, enable_partial_loading: bool = True) -> ModelCache: + logger = MagicMock() + logger.getEffectiveLevel.return_value = logging.INFO + cache = ModelCache( + execution_device_working_mem_gb=1.0, + enable_partial_loading=enable_partial_loading, + keep_ram_copy_of_weights=keep_ram_copy, + execution_device="cpu", + storage_device="cpu", + logger=logger, + ) + caches.append(cache) + return cache + + yield make + for cache in caches: + cache.shutdown() + + +def _put(cache: ModelCache, model: torch.nn.Module, partial_loading: bool): + """Cache `model` and return its record: for partial loading on the meta stand-in for VRAM, else to run on the CPU.""" + if partial_loading: + with patch.object(model_cache_module, "device_has_dedicated_vram", return_value=True): + cache.put("vae", model, execution_device=torch.device("meta")) + else: + cache.put("vae", model) + record = cache.get("vae") + assert isinstance(record.cached_model, CachedModelWithPartialLoad if partial_loading else CachedModelOnlyFullLoad) + return record + + +def _bytes_on(model: torch.nn.Module, device_type: str) -> int: + """The bytes the model's weights hold on a device, each storage once however many names share it. Storages are told + apart by identity: the meta device has no data pointers to compare.""" + storages = { + t.untyped_storage()._cdata: calc_tensor_size(t) + for t in model.state_dict().values() + if t.device.type == device_type + } + return sum(storages.values()) + + +def _snapshot(model: torch.nn.Module) -> dict[str, torch.Tensor]: + return {key: t.clone() for key, t in model.state_dict().items() if t.device.type != "meta"} + + +def _is_tied(model: TiedWeightsModule) -> bool: + return model.head.weight.untyped_storage()._cdata == model.embed.weight.untyped_storage()._cdata + + +def _assert_at_the_counted_state( + model: TiedWeightsModule, + cached_model: CachedModelWithPartialLoad | CachedModelOnlyFullLoad, + device: str, + snapshot: dict[str, torch.Tensor], +) -> None: + """Every weight in the dtype the cache counts and at the value it was cached with, the tied matrix one tensor, the + CPU weights the RAM copy when there is one, and the bytes on each device what the cache reports.""" + state_dict = model.state_dict() + assert {t.dtype for t in state_dict.values()} == {torch.float16} + for key, before in snapshot.items(): + if state_dict[key].device.type != "meta": + assert torch.equal(state_dict[key].cpu(), before.cpu()), key + assert _is_tied(model) + cpu_state_dict = cached_model.get_cpu_state_dict() + if cpu_state_dict is not None: + for key, t in state_dict.items(): + if t.device.type == "cpu": + assert t.data_ptr() == cpu_state_dict[key].data_ptr(), key + on_device = _bytes_on(model, device) if device != "cpu" else 0 + assert on_device == cached_model.cur_vram_bytes() + assert on_device + _bytes_on(model, "cpu") == cached_model.total_bytes() + + +@devices +@keep_ram_copy +def test_the_last_unlock_returns_a_partially_resident_model_to_its_counted_state( + make_cache, device: str, keep_ram_copy: bool +): + """The cast reaches the weights on both devices and unties the tied matrix, which `Module.to(dtype)` casts name by + name: restored name by name, the matrix took twice its count in RAM and then in VRAM. Each weight is restored where + it is, so the VRAM count the lock took stays true.""" + cache = make_cache(keep_ram_copy) + model = TiedWeightsModule().to(torch.float16) + with patch.object(model_cache_module, "device_has_dedicated_vram", return_value=True): + cache.put("vae", model, execution_device=torch.device(device)) + record = cache.get("vae") + cached_model = record.cached_model + assert isinstance(cached_model, CachedModelWithPartialLoad) + # The untied linear on the device, the tied matrix on the CPU. + cached_model.partial_load_to_vram(calc_tensor_size(model.linear.weight) + calc_tensor_size(model.linear.bias)) + cached_model.cur_vram_bytes() # Counted now, as `lock()` counts it before it loads. + placement = {key: t.device.type for key, t in model.state_dict().items()} + snapshot = _snapshot(model) + cache.lock_in_ram(record) + model.to(dtype=torch.float32) + + cache.unlock(record) + + assert {key: t.device.type for key, t in model.state_dict().items()} == placement + _assert_at_the_counted_state(model, cached_model, device, snapshot) + cached_model.partial_load_to_vram(cached_model.total_bytes()) + assert cached_model.cur_vram_bytes() == cached_model.total_bytes() + _assert_at_the_counted_state(model, cached_model, device, snapshot) + if device == "meta" and not keep_ram_copy: + # Without a RAM copy an offload copies the weights off the device, and the meta device holds no data. + return + cached_model.partial_unload_from_vram(calc_tensor_size(model.linear.weight)) + assert model.embed.weight.device.type == "cpu" and model.linear.weight.device.type == device + _assert_at_the_counted_state(model, cached_model, device, snapshot) + + +@devices +@keep_ram_copy +def test_the_last_unlock_returns_a_fully_loaded_model_to_its_counted_state( + make_cache, device: str, keep_ram_copy: bool +): + cache = make_cache(keep_ram_copy, enable_partial_loading=False) + model = TiedWeightsModule().to(torch.float16) + cache.put("vae", model, execution_device=torch.device(device)) + record = cache.get("vae") + cached_model = record.cached_model + assert isinstance(cached_model, CachedModelOnlyFullLoad) + snapshot = _snapshot(model) + cached_model.full_load_to_vram() + cache.lock_in_ram(record) + model.to(dtype=torch.float32) + + cache.unlock(record) + + _assert_at_the_counted_state(model, cached_model, device, snapshot) + if device == "meta" and not keep_ram_copy: + # Without a RAM copy an offload copies the weights off the device, and the meta device holds no data. + return + cached_model.full_unload_from_vram() + _assert_at_the_counted_state(model, cached_model, device, snapshot) + + +@keep_ram_copy +def test_the_last_unlock_returns_a_cpu_only_model_to_its_counted_state(make_cache, keep_ram_copy: bool): + """A model that runs on the CPU is never moved, so nothing but the unlock would ever put it back.""" + cache = make_cache(keep_ram_copy) + model = TiedWeightsModule().to(torch.float16) + record = _put(cache, model, partial_loading=False) + snapshot = _snapshot(model) + cache.lock(record, None) + model.to(dtype=torch.float32) + + cache.unlock(record) + + _assert_at_the_counted_state(model, record.cached_model, "cpu", snapshot) + + +class _MixedDtypeModule(TiedWeightsModule): + """Weights counted in two dtypes, and a persistent buffer.""" + + def __init__(self): + super().__init__() + self.register_buffer("scale", torch.ones(64)) + + +@partial_loading +def test_every_weight_returns_to_its_own_counted_dtype_buffers_included(make_cache, partial_loading: bool): + cache = make_cache(keep_ram_copy=False, enable_partial_loading=partial_loading) + model = _MixedDtypeModule().to(torch.float16) + model.linear.to(torch.float32) + counted = {key: t.dtype for key, t in model.state_dict().items()} + record = _put(cache, model, partial_loading) + if partial_loading: + record.cached_model.partial_load_to_vram( + calc_tensor_size(model.linear.weight) + calc_tensor_size(model.linear.bias) + ) + cache.lock_in_ram(record) + model.to(dtype=torch.bfloat16) + + cache.unlock(record) + + assert {key: t.dtype for key, t in model.state_dict().items()} == counted + + +def test_a_model_cast_back_in_part_is_restored_in_full(make_cache): + """The upcast encode casts the decoder's mid-block back to the VAE's dtype before it runs, so at the unlock some + weights already hold their counted dtype and the rest do not.""" + cache = make_cache(keep_ram_copy=False) + model = TiedWeightsModule().to(torch.float16) + record = _put(cache, model, partial_loading=True) + cache.lock_in_ram(record) + model.to(dtype=torch.float32) + model.linear.to(torch.float16) + + cache.unlock(record) + + assert {t.dtype for t in model.state_dict().values()} == {torch.float16} + assert _is_tied(model) + + +def test_an_inner_unlock_keeps_the_cast_an_outer_holder_runs_with(make_cache): + cache = make_cache(keep_ram_copy=True) + model = TiedWeightsModule().to(torch.float16) + record = _put(cache, model, partial_loading=False) + cache.lock(record, None) + model.to(dtype=torch.float32) + cache.lock(record, None) + + cache.unlock(record) + assert {p.dtype for p in model.parameters()} == {torch.float32} + + cache.unlock(record) + assert {p.dtype for p in model.parameters()} == {torch.float16} + + +@partial_loading +@keep_ram_copy +def test_a_restore_during_a_model_construction_keeps_real_weights( + make_cache, partial_loading: bool, keep_ram_copy: bool +): + """A construction on another thread routes every `register_parameter` to the meta device while it runs + (`accelerate.init_empty_weights` patches it process-wide), and an unlock does not wait for one.""" + cache = make_cache(keep_ram_copy, enable_partial_loading=partial_loading) + model = TiedWeightsModule().to(torch.float16) + record = _put(cache, model, partial_loading) + cache.lock_in_ram(record) + model.to(dtype=torch.float32) + constructing, construction_done = threading.Event(), threading.Event() + + def construction() -> None: + with accelerate.init_empty_weights(): + constructing.set() + construction_done.wait(timeout=10) + + builder = threading.Thread(target=construction, daemon=True) + builder.start() + try: + assert constructing.wait(timeout=10) + cache.unlock(record) + finally: + construction_done.set() + builder.join(timeout=10) + + assert {t.device.type for t in model.state_dict().values()} == {"cpu"} + assert {t.dtype for t in model.state_dict().values()} == {torch.float16} + assert _is_tied(model) + + +def test_a_restore_that_fails_drops_the_entry_and_lets_the_run_finish(make_cache, monkeypatch: pytest.MonkeyPatch): + """Partly restored, the model no longer matches its counts, as after a failed move. The unlock runs in the `finally` + of a run whose result is not saved yet, so it warns instead of raising.""" + cache = make_cache(keep_ram_copy=True) + record = _put(cache, TiedWeightsModule().to(torch.float16), partial_loading=False) + cache.lock(record, None) + monkeypatch.setattr(record.cached_model, "restore_counted_dtypes", MagicMock(side_effect=RuntimeError("no room"))) + monkeypatch.setattr(cache, "_logger", MagicMock()) + + cache.unlock(record) + + assert not record.is_locked + cache._logger.warning.assert_called_once() + with pytest.raises(IndexError): + cache.get("vae")