diff --git a/.github/workflows/pr_modular_tests.yml b/.github/workflows/pr_modular_tests.yml index 68708fdff5af..f679c0e89f31 100644 --- a/.github/workflows/pr_modular_tests.yml +++ b/.github/workflows/pr_modular_tests.yml @@ -130,6 +130,8 @@ jobs: run: | printf 'torch==2.10.0\ntorchvision==0.25.0\ntorchaudio==2.10.0\n' > "$UV_OVERRIDE" uv pip install -e ".[quality]" + # TODO (sayakpaul, DN6): revisit `--no-deps` + uv pip install -U peft@git+https://github.com/huggingface/peft.git --no-deps uv pip uninstall transformers huggingface_hub && UV_PRERELEASE=allow uv pip install -U transformers@git+https://github.com/huggingface/transformers.git uv pip uninstall accelerate && uv pip install -U accelerate@git+https://github.com/huggingface/accelerate.git --no-deps diff --git a/tests/lora/test_lora_layers_minimax_h3.py b/tests/lora/test_lora_layers_minimax_h3.py deleted file mode 100644 index 997cfe91c1dd..000000000000 --- a/tests/lora/test_lora_layers_minimax_h3.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright 2026 HuggingFace Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import tempfile - -import pytest -import safetensors.torch -import torch - -from diffusers.modular_pipelines import MiniMaxH3Blocks, MiniMaxH3ModularPipeline -from diffusers.utils import is_peft_available, logging - -from ..testing_utils import CaptureLogger, require_peft_backend - - -if is_peft_available(): - from peft.utils import get_peft_model_state_dict - - -@require_peft_backend -class TestMiniMaxH3LoraLayers: - """ - The MiniMax-H3 LoRA surface that is specific to this model and its two checkpoint partitions: loading the layouts - that circulate, the alpha handling that gets each of them to its trained scale, and the routing between the - `transformer` and `transformer_ref` partitions. Generic LoRA behavior is not tested here. - """ - - pipeline_class = MiniMaxH3ModularPipeline - pipeline_blocks_class = MiniMaxH3Blocks - pretrained_model_name_or_path = "hf-internal-testing/tiny-minimax-h3-modular-pipe" - - def get_pipeline(self): - pipeline = self.pipeline_blocks_class().init_pipeline(self.pretrained_model_name_or_path) - pipeline.load_components(dtype=torch.float32) - pipeline.set_progress_bar_config(disable=None) - return pipeline - - def test_load_lora_weights_warns_when_nothing_is_targeted(self): - r""" - An unrecognized layout that keeps the substring `lora` in every key passes the format check and then filters to - nothing in both partitions. Neither partition branch would run, so without this warning the load is a silent - no-op. The message follows `PeftAdapterMixin.load_lora_adapter`'s wording, which is what every single-denoiser - model emits in the same situation. - """ - pipe = self.get_pipeline() - state_dict = { - "some_other_model.layers.0.lora_A.weight": torch.randn(4, 24), - "some_other_model.layers.0.lora_B.weight": torch.randn(24, 4), - } - - logger = logging.get_logger("diffusers.loaders.lora_pipeline") - logger.setLevel(logging.WARNING) - with CaptureLogger(logger) as cap_logger: - pipe.load_lora_weights(state_dict, adapter_name="dummy") - - assert cap_logger.out.startswith("No LoRA keys associated to MiniMaxH3ModularPipeline") - assert "some_other_model.layers.0.lora_A.weight" in cap_logger.out - # Nothing was loaded into either partition. - for component in [pipe.transformer, pipe.transformer_ref]: - assert "dummy" not in getattr(component, "peft_config", {}) - assert not [module for module in component.modules() if "dummy" in getattr(module, "scaling", {})] - - def save_with_file_metadata(self, state_dict, tmpdir, alpha="8"): - r""" - One producer records the alpha it trained with in the safetensors `__metadata__` instead of in per-module - scalars, so the value exists only on disk — a state dict handed over in memory cannot carry it. - """ - weight_name = "pytorch_lora_weights.safetensors" - safetensors.torch.save_file( - state_dict, os.path.join(tmpdir, weight_name), metadata={"floating_dtype": "bfloat16", "alpha": alpha} - ) - return weight_name - - def test_load_lora_weights_honors_the_metadata_alpha(self): - r""" - A file with one uniform rank, no `.alpha` scalars and `alpha` "8" in its own `__metadata__`: rank 128 - against alpha 8 is a trained scale of 0.0625; synthesizing `alpha == rank` would apply the adapter 16x too - strongly. - """ - state_dict = self.get_dummy_diffusers_lora_state_dict(prefix="transformer", rank=128, adaln_rank=128) - - with tempfile.TemporaryDirectory() as tmpdir: - weight_name = self.save_with_file_metadata(state_dict, tmpdir) - - pipe = self.get_pipeline() - pipe.load_lora_weights(tmpdir, weight_name=weight_name, adapter_name="dummy") - - injected = [module for module in pipe.transformer.modules() if "dummy" in getattr(module, "scaling", {})] - assert len(injected) == 6 - assert {module.scaling["dummy"] for module in injected} == {0.0625} - - def get_dummy_diffusers_lora_state_dict(self, prefix="transformer", rank=8, adaln_rank=2): - r""" - The same adapter already converted to diffusers keys and republished — which is how the public turbo LoRA also - circulates. Mixed-rank, still alpha-less, so it needs the same treatment as the original layout even though no - conversion runs. - """ - transformer = self.get_pipeline().transformer - config = transformer.config - hidden = config.hidden_size - inner = config.num_attention_heads * config.attention_head_dim - - state_dict = {} - for module, in_features, out_features, module_rank in [ - ("transformer_blocks.0.attn.to_q", hidden, inner, rank), - ("transformer_blocks.0.attn.to_out.0", inner, hidden, rank), - ("transformer_blocks.0.ff.net.0.proj", hidden, 2 * config.ffn_dim, rank), - ("transformer_blocks.0.ff.net.2", config.ffn_dim, hidden, rank), - ("transformer_blocks.0.adaln_proj.linear", config.time_embed_dim, 6 * 3 * hidden, adaln_rank), - ("norm_out.linear", config.time_embed_dim, 2 * hidden, adaln_rank), - ]: - state_dict[f"{prefix}.{module}.lora_A.weight"] = torch.randn(module_rank, in_features) - state_dict[f"{prefix}.{module}.lora_B.weight"] = torch.randn(out_features, module_rank) - return state_dict - - @pytest.mark.parametrize("prefix", ["transformer", "transformer_ref"], ids=["transformer", "transformer_ref"]) - def test_load_lora_weights_diffusers_format_mixed_rank(self, prefix): - r""" - A mixed-rank, alpha-less adapter already in diffusers keys bypasses the converter, so the alpha handling cannot - live there: without it `get_peft_kwargs` takes `lora_alpha` from whichever rank it sees first and one of the - two rank groups is applied at `alpha / r`. - """ - pipe = self.get_pipeline() - - pipe.load_lora_weights(self.get_dummy_diffusers_lora_state_dict(prefix=prefix), adapter_name="dummy") - - component = getattr(pipe, prefix) - injected = [module for module in component.modules() if "dummy" in getattr(module, "scaling", {})] - assert len(injected) == 6 - assert {module.scaling["dummy"] for module in injected} == {1.0} - for module in injected: - assert module.lora_alpha["dummy"] == module.r["dummy"] - assert component.transformer_blocks[0].attn.to_q.r["dummy"] == 8 - assert component.transformer_blocks[0].adaln_proj.linear.r["dummy"] == 2 - assert component.norm_out.linear.r["dummy"] == 2 - - def test_load_lora_weights_into_transformer_ref(self): - pipe = self.get_pipeline() - - pipe.load_lora_weights( - self.get_dummy_diffusers_lora_state_dict(), adapter_name="dummy", load_into_transformer_ref=True - ) - - assert "dummy" in pipe.transformer_ref.peft_config - assert "dummy" not in getattr(pipe.transformer, "peft_config", {}) - - def test_save_load_lora_weights_round_trip(self): - r""" - `save_lora_weights` is the only mechanism that records which partition a LoRA belongs to, so the round trip - has to preserve it. - """ - pipe = self.get_pipeline() - pipe.load_lora_weights( - self.get_dummy_diffusers_lora_state_dict(), adapter_name="dummy", load_into_transformer_ref=True - ) - layers = get_peft_model_state_dict(pipe.transformer_ref, adapter_name="dummy") - - with tempfile.TemporaryDirectory() as tmpdir: - self.pipeline_class.save_lora_weights(tmpdir, transformer_ref_lora_layers=layers) - reloaded = self.pipeline_class.lora_state_dict(tmpdir) - - assert reloaded - assert all(key.startswith("transformer_ref.") for key in reloaded) - - fresh = self.get_pipeline() - fresh.load_lora_weights(reloaded, adapter_name="dummy") - assert "dummy" in fresh.transformer_ref.peft_config - assert "dummy" not in getattr(fresh.transformer, "peft_config", {}) - - def test_load_lora_weights_routes_to_the_only_partition(self): - r""" - `workflow="ref2va"` loads `transformer_ref` and no `transformer`, and nothing in a published H3 LoRA says which - partition it targets, so the one partition that is present is the unambiguous destination. - """ - pipe = self.pipeline_blocks_class().get_workflow("ref2va").init_pipeline(self.pretrained_model_name_or_path) - pipe.load_components(dtype=torch.float32) - assert getattr(pipe, "transformer", None) is None - - state_dict = self.get_dummy_diffusers_lora_state_dict() - pipe.load_lora_weights(state_dict, adapter_name="dummy") - - assert "dummy" in pipe.transformer_ref.peft_config - - def test_load_lora_weights_raises_without_the_requested_partition(self): - pipe = self.pipeline_blocks_class().get_workflow("t2va").init_pipeline(self.pretrained_model_name_or_path) - pipe.load_components(dtype=torch.float32) - assert getattr(pipe, "transformer_ref", None) is None - - state_dict = self.get_dummy_diffusers_lora_state_dict() - with pytest.raises(ValueError, match="load_into_transformer_ref"): - pipe.load_lora_weights(state_dict, load_into_transformer_ref=True) diff --git a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py index 2f653b28e296..a5eb5095366e 100644 --- a/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py +++ b/tests/modular_pipelines/minimax_h3/test_modular_pipeline_minimax_h3.py @@ -13,8 +13,11 @@ # limitations under the License. +import os + import numpy as np import pytest +import safetensors.torch import torch from PIL import Image @@ -35,16 +38,23 @@ MiniMaxH3TextEncoderStep, ) from diffusers.modular_pipelines.minimax_h3.modular_pipeline import MINIMAX_H3_FPS +from diffusers.utils import is_peft_available, logging +from ...testing_utils import CaptureLogger from ..testing_utils import ( BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, + ModularLoraTesterMixin, ModularMemoryTesterMixin, ModularPipelineTesterMixin, ModularWorkflowTesterMixin, ) +if is_peft_available(): + from peft.utils import get_peft_model_state_dict + + # The blocks every workflow of [`MiniMaxH3Blocks`] runs, in order. A keyframe adds the canvas block and the one # block that encodes it; whether it anchors the first or the last frame is a matter of the packed layout, not of # which blocks run, so `fl2va` covers both. @@ -415,6 +425,183 @@ class TestMiniMaxH3ModularPipelineMemory(MiniMaxH3ModularPipelineTesterConfig, M pass +class TestMiniMaxH3ModularPipelineLoRA(MiniMaxH3ModularPipelineTesterConfig, ModularLoraTesterMixin): + r""" + Generic LoRA behavior from `ModularLoraTesterMixin`, plus the MiniMax-H3 LoRA surface that is specific to this + model and its two checkpoint partitions: loading the layouts that circulate, the alpha handling that gets each of + them to its trained scale, and the routing between the `transformer` and `transformer_ref` partitions. + """ + + # MiniMax-H3 ships two independent DiT partitions, both LoRA-loadable and identically named inside. + denoiser_target_modules = { + "transformer": ["to_q", "to_k", "to_v", "to_out.0"], + "transformer_ref": ["to_q", "to_k", "to_v", "to_out.0"], + } + + def get_dummy_diffusers_lora_state_dict(self, transformer, prefix="transformer", rank=8, adaln_rank=2): + r""" + The same adapter already converted to diffusers keys and republished — which is how the public turbo LoRA also + circulates. Mixed-rank, still alpha-less, so it needs the same treatment as the original layout even though no + conversion runs. + + `transformer` is the partition the state dict is shaped against — the caller passes the one it is about to + load into, so the shapes cannot drift apart. Only its config is read; either partition does, both being the + same architecture. + """ + config = transformer.config + hidden = config.hidden_size + inner = config.num_attention_heads * config.attention_head_dim + + state_dict = {} + for module, in_features, out_features, module_rank in [ + ("transformer_blocks.0.attn.to_q", hidden, inner, rank), + ("transformer_blocks.0.attn.to_out.0", inner, hidden, rank), + ("transformer_blocks.0.ff.net.0.proj", hidden, 2 * config.ffn_dim, rank), + ("transformer_blocks.0.ff.net.2", config.ffn_dim, hidden, rank), + ("transformer_blocks.0.adaln_proj.linear", config.time_embed_dim, 6 * 3 * hidden, adaln_rank), + ("norm_out.linear", config.time_embed_dim, 2 * hidden, adaln_rank), + ]: + state_dict[f"{prefix}.{module}.lora_A.weight"] = torch.randn(module_rank, in_features) + state_dict[f"{prefix}.{module}.lora_B.weight"] = torch.randn(out_features, module_rank) + return state_dict + + def save_with_file_metadata(self, state_dict, tmp_path, alpha="8"): + r""" + One producer records the alpha it trained with in the safetensors `__metadata__` instead of in per-module + scalars, so the value exists only on disk — a state dict handed over in memory cannot carry it. + """ + weight_name = "pytorch_lora_weights.safetensors" + safetensors.torch.save_file( + state_dict, os.path.join(tmp_path, weight_name), metadata={"floating_dtype": "bfloat16", "alpha": alpha} + ) + return weight_name + + def test_load_lora_weights_warns_when_nothing_is_targeted(self): + r""" + An unrecognized layout that keeps the substring `lora` in every key passes the format check and then filters to + nothing in both partitions. Neither partition branch would run, so without this warning the load is a silent + no-op. The message follows `PeftAdapterMixin.load_lora_adapter`'s wording, which is what every single-denoiser + model emits in the same situation. + """ + pipe = self.get_pipeline() + state_dict = { + "some_other_model.layers.0.lora_A.weight": torch.randn(4, 24), + "some_other_model.layers.0.lora_B.weight": torch.randn(24, 4), + } + + logger = logging.get_logger("diffusers.loaders.lora_pipeline") + logger.setLevel(logging.WARNING) + with CaptureLogger(logger) as cap_logger: + pipe.load_lora_weights(state_dict, adapter_name="dummy") + + assert cap_logger.out.startswith("No LoRA keys associated to MiniMaxH3ModularPipeline") + assert "some_other_model.layers.0.lora_A.weight" in cap_logger.out + # Nothing was loaded into either partition. + for component in [pipe.transformer, pipe.transformer_ref]: + assert "dummy" not in getattr(component, "peft_config", {}) + assert not [module for module in component.modules() if "dummy" in getattr(module, "scaling", {})] + + def test_load_lora_weights_honors_the_metadata_alpha(self, tmp_path): + r""" + A file with one uniform rank, no `.alpha` scalars and `alpha` "8" in its own `__metadata__`: rank 128 + against alpha 8 is a trained scale of 0.0625; synthesizing `alpha == rank` would apply the adapter 16x too + strongly. + """ + pipe = self.get_pipeline() + state_dict = self.get_dummy_diffusers_lora_state_dict( + pipe.transformer, prefix="transformer", rank=128, adaln_rank=128 + ) + weight_name = self.save_with_file_metadata(state_dict, tmp_path) + + pipe.load_lora_weights(tmp_path, weight_name=weight_name, adapter_name="dummy") + + injected = [module for module in pipe.transformer.modules() if "dummy" in getattr(module, "scaling", {})] + assert len(injected) == 6 + assert {module.scaling["dummy"] for module in injected} == {0.0625} + + @pytest.mark.parametrize("prefix", ["transformer", "transformer_ref"], ids=["transformer", "transformer_ref"]) + def test_load_lora_weights_diffusers_format_mixed_rank(self, prefix): + r""" + A mixed-rank, alpha-less adapter already in diffusers keys bypasses the converter, so the alpha handling cannot + live there: without it `get_peft_kwargs` takes `lora_alpha` from whichever rank it sees first and one of the + two rank groups is applied at `alpha / r`. + """ + pipe = self.get_pipeline() + + component = getattr(pipe, prefix) + pipe.load_lora_weights( + self.get_dummy_diffusers_lora_state_dict(component, prefix=prefix), adapter_name="dummy" + ) + + injected = [module for module in component.modules() if "dummy" in getattr(module, "scaling", {})] + assert len(injected) == 6 + assert {module.scaling["dummy"] for module in injected} == {1.0} + for module in injected: + assert module.lora_alpha["dummy"] == module.r["dummy"] + assert component.transformer_blocks[0].attn.to_q.r["dummy"] == 8 + assert component.transformer_blocks[0].adaln_proj.linear.r["dummy"] == 2 + assert component.norm_out.linear.r["dummy"] == 2 + + def test_load_lora_weights_into_transformer_ref(self): + pipe = self.get_pipeline() + + pipe.load_lora_weights( + self.get_dummy_diffusers_lora_state_dict(pipe.transformer), + adapter_name="dummy", + load_into_transformer_ref=True, + ) + + assert "dummy" in pipe.transformer_ref.peft_config + assert "dummy" not in getattr(pipe.transformer, "peft_config", {}) + + def test_save_load_lora_weights_round_trip(self, tmp_path): + r""" + `save_lora_weights` is the only mechanism that records which partition a LoRA belongs to, so the round trip + has to preserve it. + """ + pipe = self.get_pipeline() + pipe.load_lora_weights( + self.get_dummy_diffusers_lora_state_dict(pipe.transformer), + adapter_name="dummy", + load_into_transformer_ref=True, + ) + layers = get_peft_model_state_dict(pipe.transformer_ref, adapter_name="dummy") + + self.pipeline_class.save_lora_weights(tmp_path, transformer_ref_lora_layers=layers) + reloaded = self.pipeline_class.lora_state_dict(tmp_path) + + assert reloaded + assert all(key.startswith("transformer_ref.") for key in reloaded) + + fresh = self.get_pipeline() + fresh.load_lora_weights(reloaded, adapter_name="dummy") + assert "dummy" in fresh.transformer_ref.peft_config + assert "dummy" not in getattr(fresh.transformer, "peft_config", {}) + + def test_load_lora_weights_routes_to_the_only_partition(self): + r""" + `workflow="ref2va"` loads `transformer_ref` and no `transformer`, and nothing in a published H3 LoRA says which + partition it targets, so the one partition that is present is the unambiguous destination. + """ + pipe = self.pipeline_blocks_class().get_workflow("ref2va").init_pipeline(self.pretrained_model_name_or_path) + pipe.load_components(dtype=torch.float32) + assert getattr(pipe, "transformer", None) is None + + state_dict = self.get_dummy_diffusers_lora_state_dict(pipe.transformer_ref) + pipe.load_lora_weights(state_dict, adapter_name="dummy") + + assert "dummy" in pipe.transformer_ref.peft_config + + def test_load_lora_weights_raises_without_the_requested_partition(self): + pipe = self.pipeline_blocks_class().get_workflow("t2va").init_pipeline(self.pretrained_model_name_or_path) + pipe.load_components(dtype=torch.float32) + assert getattr(pipe, "transformer_ref", None) is None + + state_dict = self.get_dummy_diffusers_lora_state_dict(pipe.transformer) + with pytest.raises(ValueError, match="load_into_transformer_ref"): + pipe.load_lora_weights(state_dict, load_into_transformer_ref=True) + + class MiniMaxH3Ref2VAModularPipelineTesterConfig(BaseModularPipelineTesterConfig): """The `ref2va` requests of [`MiniMaxH3Blocks`]: a prompt and an ordered list of references.""" diff --git a/tests/modular_pipelines/testing_utils/__init__.py b/tests/modular_pipelines/testing_utils/__init__.py index ce6ab1b8f216..e02c3a1be337 100644 --- a/tests/modular_pipelines/testing_utils/__init__.py +++ b/tests/modular_pipelines/testing_utils/__init__.py @@ -5,6 +5,7 @@ ) from .guider import ModularGuiderTesterMixin from .loading import ModularLoadingTesterMixin +from .lora import ModularLoraMemoryTesterMixin, ModularLoraTesterMixin from .memory import ( ModularAutoOffloadTesterMixin, ModularGroupOffloadTesterMixin, @@ -22,6 +23,8 @@ "ModularGroupOffloadTesterMixin", "ModularGuiderTesterMixin", "ModularLoadingTesterMixin", + "ModularLoraMemoryTesterMixin", + "ModularLoraTesterMixin", "ModularMemoryTesterMixin", "ModularOffloadTesterMixin", "ModularPipelineTesterMixin", diff --git a/tests/modular_pipelines/testing_utils/common.py b/tests/modular_pipelines/testing_utils/common.py index 47614fd51005..494c8dcdae13 100644 --- a/tests/modular_pipelines/testing_utils/common.py +++ b/tests/modular_pipelines/testing_utils/common.py @@ -164,11 +164,22 @@ def get_pipeline(self, components_manager=None, dtype=torch.float32): pipeline.set_progress_bar_config(disable=None) return pipeline + def run_pipe(self, pipe, **extra_inputs): + """Run the pipeline on the standard dummy inputs (fresh seeded generator) and return its output. + + `base_pipe_output` is produced by this same helper, so outputs are directly comparable against it. Pass + `extra_inputs` to override individual dummy inputs. Mirrors the non-modular `BasePipelineOutputMixin.run_pipe`, + which is what lets the pipeline-level tester mixins (the LoRA ones, in particular) run unchanged here. + """ + inputs = self.get_dummy_inputs() + inputs.update(extra_inputs) + torch.manual_seed(0) + return pipe(**inputs, output=self.output_name) + @pytest.fixture(scope="class") def base_pipe_output(self): """Output of a freshly built pipeline on the standard dummy inputs, computed once per test class.""" - pipe = self.get_pipeline().to(torch_device) - return pipe(**self.get_dummy_inputs(), output=self.output_name) + return self.run_pipe(self.get_pipeline().to(torch_device)) class ModularPipelineTesterMixin(BaseModularPipelineOutputMixin): diff --git a/tests/modular_pipelines/testing_utils/lora.py b/tests/modular_pipelines/testing_utils/lora.py new file mode 100644 index 000000000000..46b9d8e54b4f --- /dev/null +++ b/tests/modular_pipelines/testing_utils/lora.py @@ -0,0 +1,51 @@ +# coding=utf-8 +# Copyright 2026 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from ...pipelines.testing_utils.lora import LoraMemoryTesterMixin, LoraTesterMixin +from .common import BaseModularPipelineOutputMixin + + +class ModularLoraTesterMixin(BaseModularPipelineOutputMixin, LoraTesterMixin): + """ + The pipeline-level LoRA tests, run against a modular pipeline. + + A modular pipeline inherits the very same `LoraBaseMixin` subclass a standard one does + (`MiniMaxH3ModularPipeline` is a `MiniMaxH3LoraLoaderMixin`, and so on), so the tests are the same. Only building + and calling the pipeline differs, and that is `BaseModularPipelineOutputMixin`, listed first so its + `get_pipeline`/`run_pipe`/`base_pipe_output` win over the non-modular ones. + + Compose with a `BaseModularPipelineTesterConfig` subclass, and override `denoiser_target_modules` when the + denoiser components are not named `transformer`. + """ + + @pytest.mark.skip( + reason="`ModularPipeline.save_pretrained` writes the component index, not the weights, so there is no " + "pipeline directory to reload an attached adapter from." + ) + def test_simple_inference_save_pretrained_with_text_lora(self): + pass + + +class ModularLoraMemoryTesterMixin(BaseModularPipelineOutputMixin, LoraMemoryTesterMixin): + """LoRA x offloading tests for modular pipelines: group offloading composed with `load_lora_weights`.""" + + @pytest.mark.skip( + reason="`ModularPipeline` has no `enable_model_cpu_offload`; a modular pipeline offloads through " + "`ComponentsManager.enable_auto_cpu_offload`, which gets a LoRA test of its own." + ) + def test_lora_loading_model_cpu_offload(self): + pass diff --git a/tests/pipelines/testing_utils/lora.py b/tests/pipelines/testing_utils/lora.py index 4fbdfd066f85..744e8ceffbfa 100644 --- a/tests/pipelines/testing_utils/lora.py +++ b/tests/pipelines/testing_utils/lora.py @@ -20,6 +20,7 @@ import pytest import torch +from diffusers import ModularPipeline from diffusers.hooks.group_offloading import ( _GROUP_OFFLOADING, _get_top_level_group_offload_hook, @@ -66,6 +67,11 @@ def check_module_lora_metadata(parsed_metadata: dict, lora_metadatas: dict, modu def determine_attention_kwargs_name(pipeline_class): + # A modular pipeline takes its inputs from its blocks rather than from a `__call__` signature, and the denoiser + # block of every LoRA-capable one declares `InputParam.template("attention_kwargs")`. + if issubclass(pipeline_class, ModularPipeline): + return "attention_kwargs" + call_signature_keys = inspect.signature(pipeline_class.__call__).parameters.keys() # TODO(diffusers): Discuss a common naming convention across library for 1.0.0 release @@ -77,16 +83,21 @@ def determine_attention_kwargs_name(pipeline_class): @is_lora @require_peft_backend -class BaseLoraTesterMixin(BasePipelineOutputMixin): +class BaseLoraTesterMixin: """ - Shared LoRA helpers for the pipeline-level LoRA tester mixins. Not collected on its own — - compose `LoraTesterMixin`, `LoraMemoryTesterMixin` or `UNetLoraTesterMixin` with a `BasePipelineTesterConfig` - subclass instead. + Shared LoRA helpers for the LoRA tester mixins. Not collected on its own — compose `LoraTesterMixin`, + `LoraMemoryTesterMixin` or `UNetLoraTesterMixin` with a `BasePipelineTesterConfig` subclass instead. + + Deliberately free of `BasePipelineOutputMixin`: the test bodies below only ever reach the pipeline through the + seam listed here, which the modular tester mixins implement too (see + `tests/modular_pipelines/testing_utils/lora.py`). Binding the builder happens on the concrete mixins, so the + modular ones can bind a different one and reuse every test body unchanged. - Expected from the config mixin: + Expected from the config and output mixins: - pipeline_class - - get_dummy_components() - - get_dummy_inputs() (with `output_type="pt"`) + - get_pipeline() + - run_pipe(pipe, **extra_inputs) + - the `base_pipe_output` fixture Pytest mark: lora Use `pytest -m "not lora"` to skip these tests, `pytest -m lora` to run only them. @@ -184,7 +195,7 @@ def _get_lora_adapter_metadata(self, modules_to_save): } -class LoraTesterMixin(BaseLoraTesterMixin): +class LoraTesterMixin(BaseLoraTesterMixin, BasePipelineOutputMixin): """ Core LoRA/PEFT tests for pipelines: adapter attach/detach, scale kwargs, fuse/unfuse, multi-adapter handling, save/load roundtrips and metadata. Runnable on CPU. @@ -961,7 +972,7 @@ def test_inference_load_delete_load_adapters(self, tmp_path, base_pipe_output): ) -class LoraMemoryTesterMixin(BaseLoraTesterMixin): +class LoraMemoryTesterMixin(BaseLoraTesterMixin, BasePipelineOutputMixin): """LoRA x offloading tests: group offloading and model CPU offload composed with `load_lora_weights`.""" @pytest.mark.parametrize( @@ -1078,7 +1089,7 @@ def test_lora_group_offloading_delete_adapters(self, tmp_path): denoiser._diffusers_hook.remove_hook(_GROUP_OFFLOADING, recurse=True) -class UNetLoraTesterMixin(BaseLoraTesterMixin): +class UNetLoraTesterMixin(BaseLoraTesterMixin, BasePipelineOutputMixin): """ LoRA tests that only apply to UNet-based pipelines (block-scale weight dicts). Compose only into pipeline test classes whose denoiser is a UNet (e.g. SD, SDXL).