diff --git a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py index 1c183310c83a..a96025c1d036 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py @@ -647,6 +647,12 @@ def create_weights(self): assert hasattr(self.backend, "create_weights"), ( f"Backend {self.backend.__class__.__name__} must implement create_weights()" ) + # Sync quant_config to backend before creating weights. + # apply_layerwise_quant_config() matches modules by name (e.g. "model.layers.X.mlp.experts"), + # but the backend's module name has a ".backend" suffix that prevents matching. + # Propagate the (possibly updated) quant_config from ConfigurableMoE to backend here + # so that backend._get_quant_method() uses the correct per-layer quantization config. + self.backend.quant_config = self.quant_config return self.backend.create_weights() def load_weights(self, weights: List[Dict], allow_partial_loading: bool = False): diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index a870fbba6402..96a1f143ca7b 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -247,6 +247,8 @@ Qwen3/Qwen3-30B-A3B: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 79.53 + - quant_algo: W4A8_AWQ + accuracy: 79.53 - quant_algo: W4A8_MXFP4_FP8 accuracy: 79.78 - quant_algo: W4A8_MXFP4_MXFP8 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 378aa74751db..e2a8e8a98af4 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -4757,6 +4757,29 @@ def test_nvfp4( task = GSM8K(self.MODEL_NAME) task.evaluate(llm) + @skip_pre_hopper + @skip_post_blackwell + @pytest.mark.parametrize( + "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler", + [(2, 1, 1, False, False, True)], + ids=["tp2_ep1"]) + def test_w4a8(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, + overlap_scheduler): + pytorch_config = dict( + disable_overlap_scheduler=not overlap_scheduler, + cuda_graph_config=CudaGraphConfig() if cuda_graph else None) + + llm = LLM( + f"{llm_models_root()}/Qwen3/saved_models_Qwen3-30B-A3B_w4a8_hf", + tensor_parallel_size=tp_size, + pipeline_parallel_size=pp_size, + moe_expert_parallel_size=ep_size, + **pytorch_config, + enable_attention_dp=attention_dp) + with llm: + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + @pytest.mark.parametrize("moe_backend", ["CUTLASS", "TRTLLM"]) @pytest.mark.parametrize("tp_size,pp_size,ep_size", [ (1, 1, 1), diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index fa1882623a3a..05a9a6eb8799 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -725,6 +725,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRITON] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM] +accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8[tp2_ep1] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS] diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 6d63a7782a53..aa32681a3abe 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -62,6 +62,8 @@ l0_dgx_h100: # llmapi - unittest/llmapi/test_mpi_session.py::test_llmapi_launch_multiple_tasks - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_gen_only_spec_dec + # ------------- Model specific tests --------------- + - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8[tp2_ep1] - condition: ranges: system_gpu_count: