From 8de485afaaad8fc9faed50e69fa50a3bb77b8efa Mon Sep 17 00:00:00 2001 From: ylichen Date: Sun, 17 May 2026 18:05:59 -0700 Subject: [PATCH 1/3] Fix the runtime error of w4a8 model with ConfiguragleMoE Signed-off-by: ylichen --- tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py index 1c183310c83a..a96025c1d036 100644 --- a/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/modules/fused_moe/configurable_moe.py @@ -647,6 +647,12 @@ def create_weights(self): assert hasattr(self.backend, "create_weights"), ( f"Backend {self.backend.__class__.__name__} must implement create_weights()" ) + # Sync quant_config to backend before creating weights. + # apply_layerwise_quant_config() matches modules by name (e.g. "model.layers.X.mlp.experts"), + # but the backend's module name has a ".backend" suffix that prevents matching. + # Propagate the (possibly updated) quant_config from ConfigurableMoE to backend here + # so that backend._get_quant_method() uses the correct per-layer quantization config. + self.backend.quant_config = self.quant_config return self.backend.create_weights() def load_weights(self, weights: List[Dict], allow_partial_loading: bool = False): From 54950d7d080d57d25e3beb35e301d6dc2564ea0c Mon Sep 17 00:00:00 2001 From: ylichen Date: Mon, 25 May 2026 01:54:43 -0700 Subject: [PATCH 2/3] Add unit test for w4a8 model Signed-off-by: ylichen --- .../defs/accuracy/test_llm_api_pytorch.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 550ff8641d80..951049ebe83f 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -4707,6 +4707,29 @@ def test_nvfp4( task = GSM8K(self.MODEL_NAME) task.evaluate(llm) + @skip_pre_hopper + @skip_post_blackwell + @pytest.mark.parametrize( + "tp_size,pp_size,ep_size,attention_dp,cuda_graph,overlap_scheduler", + [(2, 1, 1, False, False, True)], + ids=["tp2_ep1"]) + def test_w4a8(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, + overlap_scheduler): + pytorch_config = dict( + disable_overlap_scheduler=not overlap_scheduler, + cuda_graph_config=CudaGraphConfig() if cuda_graph else None) + + llm = LLM( + f"{llm_models_root()}/Qwen3/saved_models_Qwen3-30B-A3B_w4a8_hf", + tensor_parallel_size=tp_size, + pipeline_parallel_size=pp_size, + moe_expert_parallel_size=ep_size, + **pytorch_config, + enable_attention_dp=attention_dp) + with llm: + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) + @pytest.mark.parametrize("moe_backend", ["CUTLASS", "TRTLLM"]) @pytest.mark.parametrize("tp_size,pp_size,ep_size", [ (1, 1, 1), From 79af0a2c7fd8fcbafc1dc16d784d197661003252 Mon Sep 17 00:00:00 2001 From: ylichen Date: Sun, 28 Jun 2026 06:54:34 -0700 Subject: [PATCH 3/3] Add QA test for Qwen3-30B-A3B w4a8 model - Add test_w4a8[tp2_ep1] to llm_function_core.txt QA test list - Add test to l0_dgx_h100.yml 2-GPU pre_merge section - Add W4A8_AWQ accuracy reference (79.53) to mmlu.yaml Signed-off-by: ylichen --- tests/integration/defs/accuracy/references/mmlu.yaml | 2 ++ tests/integration/test_lists/qa/llm_function_core.txt | 1 + tests/integration/test_lists/test-db/l0_dgx_h100.yml | 2 ++ 3 files changed, 5 insertions(+) diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index cbce62591221..7ed23eaba7bf 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -247,6 +247,8 @@ Qwen3/Qwen3-30B-A3B: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 accuracy: 79.53 + - quant_algo: W4A8_AWQ + accuracy: 79.53 - quant_algo: W4A8_MXFP4_FP8 accuracy: 79.78 - quant_algo: W4A8_MXFP4_MXFP8 diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index a981dddd2bab..904171feeccf 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -721,6 +721,7 @@ accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_nvfp4[tep4_latency_moe_trtllm-torch_compile=True] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRITON] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a16_mxfp4[latency-TRTLLM] +accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8[tp2_ep1] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-CUTLASS] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[fp8-latency-TRTLLM] accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8_mxfp4[mxfp8-latency-CUTLASS] diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 2ed7c363fe3a..39ad8dad8522 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -64,6 +64,8 @@ l0_dgx_h100: # llmapi - unittest/llmapi/test_mpi_session.py::test_llmapi_launch_multiple_tasks - accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_gen_only_spec_dec + # ------------- Model specific tests --------------- + - accuracy/test_llm_api_pytorch.py::TestQwen3_30B_A3B::test_w4a8[tp2_ep1] - condition: ranges: system_gpu_count: