From 6c217a48a2d71fefaed1e6d6ff0f97e5996f31ed Mon Sep 17 00:00:00 2001 From: Fanrong Li <23290157+lfr-0531@users.noreply.github.com> Date: Tue, 4 Aug 2026 04:27:04 -0700 Subject: [PATCH 1/3] [https://nvbugs/6434512][fix] select Marlin for Qwen3.5 NVFP4 on Hopper Signed-off-by: Fanrong Li <23290157+lfr-0531@users.noreply.github.com> --- .../_torch/models/modeling_qwen3_5.py | 43 ++++++++++++- .../modeling/test_modeling_qwen3_5_vl_moe.py | 61 ++++++++++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_5.py b/tensorrt_llm/_torch/models/modeling_qwen3_5.py index 0e753d8ace4a..e1607ea1ceab 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_5.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_5.py @@ -15,11 +15,14 @@ import re from types import SimpleNamespace -from typing import Dict, List, Literal +from typing import TYPE_CHECKING, Dict, List, Literal import torch from transformers import PretrainedConfig +if TYPE_CHECKING: + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + from tensorrt_llm._utils import get_sm_version from tensorrt_llm.logger import logger from tensorrt_llm.quantization import QuantAlgo @@ -32,6 +35,7 @@ support_multimodal_disaggregated, ) from ..pyexecutor.config_utils import get_qwen3_hybrid_layer_types +from ..utils import is_nvfp4_marlin_supported_sm from .checkpoints.base_weight_mapper import BaseWeightMapper from .checkpoints.hf.qwen3_5_weight_mapper import Qwen3_5MoeHfWeightMapper from .modeling_qwen3_next import Qwen3NextForCausalLM @@ -54,6 +58,35 @@ } +def _get_qwen35_moe_model_defaults(llm_args: "TorchLlmArgs") -> dict: + """Return Ada/Hopper defaults for globally NVFP4-quantized Qwen3.5 MoE. + + MIXED_PRECISION checkpoints are excluded because this hook cannot inspect + their per-layer expert algorithms, some of which Marlin cannot consume. + Dense Qwen3.5 checkpoints are outside this MoE-specific default. + """ + defaults = Qwen3NextForCausalLM.get_model_defaults(llm_args) + quant_config = getattr(llm_args, "quant_config", None) + if getattr( + quant_config, "quant_algo", None + ) == QuantAlgo.NVFP4 and is_nvfp4_marlin_supported_sm(get_sm_version()): + # The CUTLASS W4A4 NVFP4 kernels require Blackwell. Marlin consumes + # the same checkpoint weights with BF16 activations on Ada/Hopper. + # Marlin does not support EPLB, but an explicit user backend remains + # authoritative when model defaults are applied. + defaults.update( + { + "moe_config": { + "backend": "MARLIN", + }, + "nvfp4_gemm_config": { + "allowed_backends": ["marlin"], + }, + } + ) + return defaults + + def _translate_mtp_pattern(name, n_hidden_layers): """Translate an HF ``mtp.*`` exclude pattern to a TRT-LLM module path. @@ -614,6 +647,10 @@ class Qwen3_5MoeForCausalLM(Qwen3NextForCausalLM): class that serves the vanilla Qwen3NextForCausalLM architecture. """ + @classmethod + def get_model_defaults(cls, llm_args: "TorchLlmArgs") -> dict: + return _get_qwen35_moe_model_defaults(llm_args) + def __init__(self, model_config): keep_lm_head_quant = _lm_head_nvfp4_enabled(model_config) _normalize_qwen35_exclude_modules(model_config, keep_lm_head_quant=keep_lm_head_quant) @@ -745,6 +782,10 @@ def load_weights( class Qwen3_5MoeVLModel(_Qwen3_5VLModel): """VLM wrapper composing Qwen3 vision encoder with Qwen3.5 MoE text decoder.""" + @classmethod + def get_model_defaults(cls, llm_args: "TorchLlmArgs") -> dict: + return _get_qwen35_moe_model_defaults(llm_args) + # TODO(TRTLLM-13417): Add tests for disaggregated support. @support_multimodal_disaggregated diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py index e715c53a3ac2..cafa7cc1fe9b 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py @@ -5,6 +5,7 @@ import os from copy import deepcopy from pathlib import Path +from types import SimpleNamespace from typing import List, Optional import torch @@ -15,7 +16,7 @@ from utils.util import skip_pre_hopper from tensorrt_llm._torch.model_config import ModelConfig -from tensorrt_llm._torch.models import Qwen3_5MoeVLModel +from tensorrt_llm._torch.models import Qwen3_5MoeForCausalLM, Qwen3_5MoeVLModel from tensorrt_llm._torch.models.checkpoints.auto_mapper import AutoCheckpointMapper from tensorrt_llm._torch.models.checkpoints.hf.qwen3_5_weight_mapper import Qwen3_5MoeHfWeightMapper from tensorrt_llm._torch.models.modeling_auto import AutoModelForCausalLM @@ -27,6 +28,7 @@ from tensorrt_llm._torch.pyexecutor.model_loader import validate_and_set_mamba_ssm_cache_dtype from tensorrt_llm.inputs import ContentFormat from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY +from tensorrt_llm.quantization import QuantAlgo def _write_qwen35_moe_vl_config(tmp_path: Path) -> Path: @@ -156,6 +158,63 @@ def test_qwen35_moe_vl_resolves_model_and_mapper(tmp_path: Path) -> None: ) +def test_qwen35_moe_model_defaults_select_marlin_on_hopper(monkeypatch) -> None: + monkeypatch.setattr( + "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", + lambda: 90, + ) + + expected = { + "kv_cache_config": { + "enable_block_reuse": False, + "use_kv_cache_manager_v2": True, + }, + "moe_config": { + "backend": "MARLIN", + }, + "nvfp4_gemm_config": { + "allowed_backends": ["marlin"], + }, + } + llm_args = SimpleNamespace(quant_config=SimpleNamespace(quant_algo=QuantAlgo.NVFP4)) + assert Qwen3_5MoeForCausalLM.get_model_defaults(llm_args) == expected + assert Qwen3_5MoeVLModel.get_model_defaults(llm_args) == expected + + +def test_qwen35_moe_model_defaults_keep_non_nvfp4_backends(monkeypatch) -> None: + monkeypatch.setattr( + "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", + lambda: 90, + ) + + expected = { + "kv_cache_config": { + "enable_block_reuse": False, + "use_kv_cache_manager_v2": True, + } + } + llm_args = SimpleNamespace(quant_config=SimpleNamespace(quant_algo=QuantAlgo.FP8)) + assert Qwen3_5MoeForCausalLM.get_model_defaults(llm_args) == expected + assert Qwen3_5MoeVLModel.get_model_defaults(llm_args) == expected + + +def test_qwen35_moe_model_defaults_keep_blackwell_backends(monkeypatch) -> None: + monkeypatch.setattr( + "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", + lambda: 100, + ) + + expected = { + "kv_cache_config": { + "enable_block_reuse": False, + "use_kv_cache_manager_v2": True, + } + } + llm_args = SimpleNamespace(quant_config=SimpleNamespace(quant_algo=QuantAlgo.NVFP4)) + assert Qwen3_5MoeForCausalLM.get_model_defaults(llm_args) == expected + assert Qwen3_5MoeVLModel.get_model_defaults(llm_args) == expected + + def test_qwen35_moe_vl_placeholder_metadata_registered() -> None: metadata = MULTIMODAL_PLACEHOLDER_REGISTRY.get_placeholder_metadata("qwen3_5_moe") From 69b54e43cf50dffc907995891e27c3f96bc3a76f Mon Sep 17 00:00:00 2001 From: Fanrong Li <23290157+lfr-0531@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:18:22 -0700 Subject: [PATCH 2/3] [https://nvbugs/6434512][test] consolidate Qwen3.5 MoE default coverage Signed-off-by: Fanrong Li <23290157+lfr-0531@users.noreply.github.com> --- .../modeling/test_modeling_qwen3_5_vl_moe.py | 83 +++++++------------ 1 file changed, 31 insertions(+), 52 deletions(-) diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py index cafa7cc1fe9b..37fcb8eb01db 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py @@ -5,9 +5,9 @@ import os from copy import deepcopy from pathlib import Path -from types import SimpleNamespace from typing import List, Optional +import pytest import torch import transformers from test_modeling_multimodal import MultimodalScenario, TestModelingMultimodal @@ -28,6 +28,9 @@ from tensorrt_llm._torch.pyexecutor.model_loader import validate_and_set_mamba_ssm_cache_dtype from tensorrt_llm.inputs import ContentFormat from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY +from tensorrt_llm.llmapi.llm_args import TorchLlmArgs +from tensorrt_llm.llmapi.llm_utils import apply_model_defaults_to_llm_args +from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization import QuantAlgo @@ -158,61 +161,37 @@ def test_qwen35_moe_vl_resolves_model_and_mapper(tmp_path: Path) -> None: ) -def test_qwen35_moe_model_defaults_select_marlin_on_hopper(monkeypatch) -> None: - monkeypatch.setattr( - "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", - lambda: 90, - ) - - expected = { - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": True, - }, - "moe_config": { - "backend": "MARLIN", - }, - "nvfp4_gemm_config": { - "allowed_backends": ["marlin"], - }, - } - llm_args = SimpleNamespace(quant_config=SimpleNamespace(quant_algo=QuantAlgo.NVFP4)) - assert Qwen3_5MoeForCausalLM.get_model_defaults(llm_args) == expected - assert Qwen3_5MoeVLModel.get_model_defaults(llm_args) == expected - - -def test_qwen35_moe_model_defaults_keep_non_nvfp4_backends(monkeypatch) -> None: - monkeypatch.setattr( - "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", - lambda: 90, - ) - - expected = { - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": True, - } - } - llm_args = SimpleNamespace(quant_config=SimpleNamespace(quant_algo=QuantAlgo.FP8)) - assert Qwen3_5MoeForCausalLM.get_model_defaults(llm_args) == expected - assert Qwen3_5MoeVLModel.get_model_defaults(llm_args) == expected - - -def test_qwen35_moe_model_defaults_keep_blackwell_backends(monkeypatch) -> None: +@pytest.mark.parametrize( + ("quant_algo", "sm_version", "use_marlin"), + [ + pytest.param(QuantAlgo.NVFP4, 90, True, id="hopper-nvfp4"), + pytest.param(QuantAlgo.MIXED_PRECISION, 90, False, id="hopper-mixed-precision"), + pytest.param(QuantAlgo.NVFP4, 100, False, id="blackwell-nvfp4"), + ], +) +def test_qwen35_moe_model_defaults( + monkeypatch: pytest.MonkeyPatch, + quant_algo: QuantAlgo, + sm_version: int, + use_marlin: bool, +) -> None: monkeypatch.setattr( "tensorrt_llm._torch.models.modeling_qwen3_5.get_sm_version", - lambda: 100, + lambda: sm_version, ) - expected = { - "kv_cache_config": { - "enable_block_reuse": False, - "use_kv_cache_manager_v2": True, - } - } - llm_args = SimpleNamespace(quant_config=SimpleNamespace(quant_algo=QuantAlgo.NVFP4)) - assert Qwen3_5MoeForCausalLM.get_model_defaults(llm_args) == expected - assert Qwen3_5MoeVLModel.get_model_defaults(llm_args) == expected + expected_moe_backend = "MARLIN" if use_marlin else "AUTO" + expected_gemm_backends = ["marlin"] if use_marlin else ["cutlass", "cublaslt", "cuda_core"] + for model_cls in (Qwen3_5MoeForCausalLM, Qwen3_5MoeVLModel): + llm_args = TorchLlmArgs(model="/tmp/dummy_model") + llm_args.quant_config = QuantConfig(quant_algo=quant_algo) + defaults = model_cls.get_model_defaults(llm_args) + apply_model_defaults_to_llm_args(llm_args, defaults) + + assert llm_args.kv_cache_config.enable_block_reuse is False + assert llm_args.kv_cache_config.use_kv_cache_manager_v2 is True + assert llm_args.moe_config.backend == expected_moe_backend + assert llm_args.nvfp4_gemm_config.allowed_backends == expected_gemm_backends def test_qwen35_moe_vl_placeholder_metadata_registered() -> None: From 050a0d30361218cd3080675a4ac8f54f44c2a3d4 Mon Sep 17 00:00:00 2001 From: Fanrong Li <23290157+lfr-0531@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:21:59 +0000 Subject: [PATCH 3/3] [https://nvbugs/6434512][fix] support mixed-precision Qwen3.5 MoE on Hopper Signed-off-by: Fanrong Li <23290157+lfr-0531@users.noreply.github.com> --- .../_torch/models/modeling_qwen3_5.py | 19 ++++++------------- .../modeling/test_modeling_qwen3_5_vl_moe.py | 2 +- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_qwen3_5.py b/tensorrt_llm/_torch/models/modeling_qwen3_5.py index e1607ea1ceab..8acf89ab927b 100644 --- a/tensorrt_llm/_torch/models/modeling_qwen3_5.py +++ b/tensorrt_llm/_torch/models/modeling_qwen3_5.py @@ -59,21 +59,14 @@ def _get_qwen35_moe_model_defaults(llm_args: "TorchLlmArgs") -> dict: - """Return Ada/Hopper defaults for globally NVFP4-quantized Qwen3.5 MoE. - - MIXED_PRECISION checkpoints are excluded because this hook cannot inspect - their per-layer expert algorithms, some of which Marlin cannot consume. - Dense Qwen3.5 checkpoints are outside this MoE-specific default. - """ + """Return Marlin defaults for Qwen3.5 MoE with NVFP4 experts on Ada/Hopper.""" defaults = Qwen3NextForCausalLM.get_model_defaults(llm_args) quant_config = getattr(llm_args, "quant_config", None) - if getattr( - quant_config, "quant_algo", None - ) == QuantAlgo.NVFP4 and is_nvfp4_marlin_supported_sm(get_sm_version()): - # The CUTLASS W4A4 NVFP4 kernels require Blackwell. Marlin consumes - # the same checkpoint weights with BF16 activations on Ada/Hopper. - # Marlin does not support EPLB, but an explicit user backend remains - # authoritative when model defaults are applied. + if getattr(quant_config, "quant_algo", None) in ( + QuantAlgo.NVFP4, + QuantAlgo.MIXED_PRECISION, + ) and is_nvfp4_marlin_supported_sm(get_sm_version()): + # CUTLASS W4A4 requires Blackwell; use Marlin's W4A16 path instead. defaults.update( { "moe_config": { diff --git a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py index 37fcb8eb01db..9aa8e7e11a06 100644 --- a/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py +++ b/tests/unittest/_torch/modeling/test_modeling_qwen3_5_vl_moe.py @@ -165,7 +165,7 @@ def test_qwen35_moe_vl_resolves_model_and_mapper(tmp_path: Path) -> None: ("quant_algo", "sm_version", "use_marlin"), [ pytest.param(QuantAlgo.NVFP4, 90, True, id="hopper-nvfp4"), - pytest.param(QuantAlgo.MIXED_PRECISION, 90, False, id="hopper-mixed-precision"), + pytest.param(QuantAlgo.MIXED_PRECISION, 90, True, id="hopper-mixed-precision"), pytest.param(QuantAlgo.NVFP4, 100, False, id="blackwell-nvfp4"), ], )