From e41964135a6dace5aa5ca5356233703515fb95d1 Mon Sep 17 00:00:00 2001 From: nv-guomingz <137257613+nv-guomingz@users.noreply.github.com> Date: Wed, 8 Jul 2026 22:21:43 +0800 Subject: [PATCH] [None][test] Reduce Nemotron V3 Super/Ultra pre-merge test cases Consolidate Nemotron V3 Super/Ultra accuracy tests since both models share the same NemotronH architecture and code paths: - Remove Super BF16 tests (test_auto_dtype_4gpus, test_bf16_trtllm_gen_moe_backend, test_bf16_4gpu_mtp_ar) - Remove Super test_nvfp4_8gpus; 8-GPU MoE backend coverage is represented by the Ultra variants - Remove Ultra test_nvfp4_4gpu_mtp_ar (duplicate of the Super case) and drop the B300 waive of the Super case (nvbugs/6336990) - Use context manager for LLM in MTP acceptance-rate tests to ensure proper shutdown - Rename Super block_reuse ids: TEP4_ADP -> DEP4_MTP_OFF, TEP4_ADP_MTP -> DEP4_MTP_ON - Move to post-merge: Super fp8_4gpus (all combos), 8gpus_mtp_custom_op, parallelism[TP4_PP2/TP8_PP1], online_eplb[TRTLLM], block_reuse[TEP4/DEP4_MTP_OFF]; Ultra online_eplb[CUTEDSL], parallelism[ADP2_PP2] Pre-merge L0 cases reduced from 22 to 7 while keeping one representative per functional dimension (MoE backends, MTP accuracy and acceptance rate, block reuse with TEP/DEP mappings, Hopper W4A16); the QA list retains broader coverage. Signed-off-by: nv-guomingz <137257613+nv-guomingz@users.noreply.github.com> --- .../defs/accuracy/test_llm_api_pytorch.py | 375 ++++-------------- .../test_lists/qa/llm_function_core.txt | 13 +- .../test_lists/test-db/l0_dgx_b200.yml | 11 +- .../test_lists/test-db/l0_dgx_h100.yml | 8 +- .../test-db/l0_gb200_multi_gpus.yml | 15 +- tests/integration/test_lists/waives.txt | 1 - 6 files changed, 82 insertions(+), 341 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 2e201782e786..f23e4f720071 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -6554,71 +6554,6 @@ class TestNemotronV3Super(LlmapiAccuracyTestHarness): EXTRA_EVALUATOR_KWARGS = dict(chat_template_kwargs=dict( enable_thinking=False)) - @skip_pre_blackwell - @pytest.mark.skip_less_device_memory(80000) - @pytest.mark.skip_less_mpi_world_size(4) - @pytest.mark.parametrize( - "tp_size, ep_size, attention_dp", - [ - (4, 4, False), - (4, 1, False), - (4, 4, True), - (4, 1, True), - ], - ) - def test_auto_dtype_4gpus(self, tp_size, ep_size, attention_dp): - - kv_cache_config = KvCacheConfig(enable_block_reuse=False, - mamba_ssm_cache_dtype="float32") - pytorch_config = dict(disable_overlap_scheduler=False, - cuda_graph_config=CudaGraphConfig( - max_batch_size=32, enable_padding=True)) - - with LLM( - f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", - kv_cache_config=kv_cache_config, - max_batch_size=32, - tensor_parallel_size=tp_size, - moe_expert_parallel_size=ep_size, - enable_attention_dp=attention_dp, - **pytorch_config, - ) as llm: - task = MMLU(self.MODEL_NAME) - task.evaluate(llm, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - - @skip_pre_blackwell - @pytest.mark.skip_less_device_memory(80000) - @pytest.mark.skip_less_mpi_world_size(4) - @parametrize_with_ids("attention_dp", [False, True]) - def test_bf16_trtllm_gen_moe_backend(self, attention_dp): - - kv_cache_config = KvCacheConfig(enable_block_reuse=False, - mamba_ssm_cache_dtype="float32") - pytorch_config = dict(disable_overlap_scheduler=False, - cuda_graph_config=CudaGraphConfig( - max_batch_size=32, enable_padding=True)) - - with LLM( - f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-BF16", - kv_cache_config=kv_cache_config, - max_batch_size=32, - tensor_parallel_size=4, - moe_expert_parallel_size=4, - enable_attention_dp=attention_dp, - moe_config=MoeConfig(backend="TRTLLM"), - **pytorch_config, - ) as llm: - task = MMLU(self.MODEL_NAME) - task.evaluate(llm, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - def _run_nvfp4_4gpus_eplb(self, moe_backend, eplb_config, model_path): kv_cache_config = KvCacheConfig( enable_block_reuse=False, @@ -6790,49 +6725,6 @@ def test_fp8_4gpus(self, attention_dp, use_cpp_mamba, monkeypatch): task.evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - @skip_pre_blackwell - @pytest.mark.skip_less_mpi_world_size(8) - @pytest.mark.parametrize("moe_backend", ["TRTLLM", "CUTLASS", "CUTEDSL"], - ids=["trtllm", "cutlass", "cutedsl"]) - @pytest.mark.parametrize( - "attention_dp", - [ - False, - True, - ], - ids=[ - "attention_dp_off", - "attention_dp_on", - ], - ) - def test_nvfp4_8gpus(self, attention_dp, moe_backend): - # Use this test to track the best performance config. - # The optimized config is still under investigation. - # Adding this test as placeholder. - with LLM( - f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4", - kv_cache_config=KvCacheConfig( - enable_block_reuse=False, - mamba_ssm_cache_dtype="float16", - free_gpu_memory_fraction=0.5, - ), - max_batch_size=32, - tensor_parallel_size=8, - moe_expert_parallel_size=8, - pipeline_parallel_size=1, - enable_attention_dp=attention_dp, - cuda_graph_config=CudaGraphConfig(max_batch_size=32, - enable_padding=True), - disable_overlap_scheduler=False, - moe_config=MoeConfig(backend=moe_backend), - ) as llm: - task = MMLU(self.MODEL_NAME) - task.evaluate(llm, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - task = GSM8K(self.MODEL_NAME) - task.evaluate(llm, - extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - @skip_pre_blackwell @pytest.mark.parametrize( "tp_size, ep_size, mamba_state_cache_interval, attention_dp, use_mtp", @@ -6843,7 +6735,7 @@ def test_nvfp4_8gpus(self, attention_dp, moe_backend): (4, 4, 256, True, False), (4, 4, 512, True, True), ], - ids=["TP1", "TP4_MTP", "TEP4", "TEP4_ADP", "TEP4_ADP_MTP"], + ids=["TP1", "TP4_MTP", "TEP4", "DEP4_MTP_OFF", "DEP4_MTP_ON"], ) def test_nvfp4_4gpus_block_reuse(self, tp_size, ep_size, mamba_state_cache_interval, attention_dp, @@ -6976,114 +6868,50 @@ def test_nvfp4_4gpu_mtp_ar(self): mamba_ssm_cache_dtype="float16", free_gpu_memory_fraction=0.5, ), - max_batch_size=4, + max_batch_size=64, enable_attention_dp=True, - cuda_graph_config=CudaGraphConfig(max_batch_size=4, + cuda_graph_config=CudaGraphConfig(max_batch_size=64, enable_padding=True), disable_overlap_scheduler=False, moe_config=MoeConfig(backend="CUTLASS"), ) - llm_spec = LLM(**llm_common_config, speculative_config=mtp_config) - - raw_prompts = [ - "The capital of France is", - "The president of the United States is", - "The future of AI is", - ] - prompts = [ - llm_spec.tokenizer.apply_chat_template( - [{ - "role": "user", - "content": p - }], - tokenize=False, - add_generation_prompt=True, - ) for p in raw_prompts - ] - tok_ids = [llm_spec.tokenizer.encode(p) for p in prompts] - - sampling_params = SamplingParams(max_tokens=128, temperature=0) - - for i in range(len(tok_ids)): - num_tokens = 0 - num_drafted = 0 - num_accepted = 0 - for output in llm_spec.generate_async(tok_ids[i], - sampling_params, - streaming=True): - new_tokens = output.outputs[0].token_ids - num_drafted += max_draft_len - num_accepted += len(new_tokens) - num_tokens - 1 - num_tokens = len(new_tokens) - - accept_rate = num_accepted / num_drafted - assert accept_rate > 0.2, \ - f"Acceptance rate too low for prompt {i}: {accept_rate:.2f}" - - @skip_pre_blackwell - @pytest.mark.skip_less_device(4) - @pytest.mark.skip_less_device_memory(80000) - def test_bf16_4gpu_mtp_ar(self): - max_draft_len = 7 - mtp_config = MTPDecodingConfig( - max_draft_len=max_draft_len, - mtp_eagle_one_model=True, - ) - model_path = f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" - llm_common_config = dict( - model=model_path, - tensor_parallel_size=4, - moe_expert_parallel_size=4, - kv_cache_config=KvCacheConfig( - enable_block_reuse=False, - mamba_ssm_cache_dtype="float16", - free_gpu_memory_fraction=0.5, - ), - max_batch_size=4, - enable_attention_dp=True, - cuda_graph_config=CudaGraphConfig(max_batch_size=4, - enable_padding=True), - disable_overlap_scheduler=False, - moe_config=MoeConfig(backend="CUTLASS"), - ) - - llm_spec = LLM(**llm_common_config, speculative_config=mtp_config) - - raw_prompts = [ - "The capital of France is", - "The president of the United States is", - "The future of AI is", - ] - prompts = [ - llm_spec.tokenizer.apply_chat_template( - [{ - "role": "user", - "content": p - }], - tokenize=False, - add_generation_prompt=True, - ) for p in raw_prompts - ] - tok_ids = [llm_spec.tokenizer.encode(p) for p in prompts] + with LLM(**llm_common_config, + speculative_config=mtp_config) as llm_spec: + raw_prompts = [ + "The capital of France is", + "The president of the United States is", + "The future of AI is", + ] + prompts = [ + llm_spec.tokenizer.apply_chat_template( + [{ + "role": "user", + "content": p + }], + tokenize=False, + add_generation_prompt=True, + ) for p in raw_prompts + ] + tok_ids = [llm_spec.tokenizer.encode(p) for p in prompts] - sampling_params = SamplingParams(max_tokens=128, temperature=0) + sampling_params = SamplingParams(max_tokens=128, temperature=0) - for i in range(len(tok_ids)): - num_tokens = 0 - num_drafted = 0 - num_accepted = 0 - for output in llm_spec.generate_async(tok_ids[i], - sampling_params, - streaming=True): - new_tokens = output.outputs[0].token_ids - num_drafted += max_draft_len - num_accepted += len(new_tokens) - num_tokens - 1 - num_tokens = len(new_tokens) + for i in range(len(tok_ids)): + num_tokens = 0 + num_drafted = 0 + num_accepted = 0 + for output in llm_spec.generate_async(tok_ids[i], + sampling_params, + streaming=True): + new_tokens = output.outputs[0].token_ids + num_drafted += max_draft_len + num_accepted += len(new_tokens) - num_tokens - 1 + num_tokens = len(new_tokens) - accept_rate = num_accepted / num_drafted - assert accept_rate > 0.2, \ - f"Acceptance rate too low for prompt {i}: {accept_rate:.2f}" + accept_rate = num_accepted / num_drafted + assert accept_rate > 0.2, \ + f"Acceptance rate too low for prompt {i}: {accept_rate:.2f}" @skip_pre_blackwell @pytest.mark.skip_less_device(4) @@ -7114,42 +6942,42 @@ def test_nvfp4_4gpu_mtp_ar_custom_op(self, monkeypatch): moe_config=MoeConfig(backend="CUTLASS"), ) - llm_spec = LLM(**llm_common_config, speculative_config=mtp_config) - - raw_prompts = [ - "The capital of France is", - "The president of the United States is", - "The future of AI is", - ] - prompts = [ - llm_spec.tokenizer.apply_chat_template( - [{ - "role": "user", - "content": p - }], - tokenize=False, - add_generation_prompt=True, - ) for p in raw_prompts - ] - tok_ids = [llm_spec.tokenizer.encode(p) for p in prompts] + with LLM(**llm_common_config, + speculative_config=mtp_config) as llm_spec: + raw_prompts = [ + "The capital of France is", + "The president of the United States is", + "The future of AI is", + ] + prompts = [ + llm_spec.tokenizer.apply_chat_template( + [{ + "role": "user", + "content": p + }], + tokenize=False, + add_generation_prompt=True, + ) for p in raw_prompts + ] + tok_ids = [llm_spec.tokenizer.encode(p) for p in prompts] - sampling_params = SamplingParams(max_tokens=128, temperature=0) + sampling_params = SamplingParams(max_tokens=128, temperature=0) - for i in range(len(tok_ids)): - num_tokens = 0 - num_drafted = 0 - num_accepted = 0 - for output in llm_spec.generate_async(tok_ids[i], - sampling_params, - streaming=True): - new_tokens = output.outputs[0].token_ids - num_drafted += max_draft_len - num_accepted += len(new_tokens) - num_tokens - 1 - num_tokens = len(new_tokens) + for i in range(len(tok_ids)): + num_tokens = 0 + num_drafted = 0 + num_accepted = 0 + for output in llm_spec.generate_async(tok_ids[i], + sampling_params, + streaming=True): + new_tokens = output.outputs[0].token_ids + num_drafted += max_draft_len + num_accepted += len(new_tokens) - num_tokens - 1 + num_tokens = len(new_tokens) - accept_rate = num_accepted / num_drafted - assert accept_rate > 0.2, \ - f"Acceptance rate too low for prompt {i}: {accept_rate:.2f}" + accept_rate = num_accepted / num_drafted + assert accept_rate > 0.2, \ + f"Acceptance rate too low for prompt {i}: {accept_rate:.2f}" @skip_pre_blackwell @pytest.mark.skip_less_mpi_world_size(8) @@ -7412,71 +7240,6 @@ def test_nvfp4_parallelism(self, tp_size, ep_size, pp_size, attention_dp): task.evaluate(llm, extra_evaluator_kwargs=self.EXTRA_EVALUATOR_KWARGS) - @skip_pre_blackwell - @pytest.mark.skip_less_device(4) - @pytest.mark.skip_less_device_memory(80000) - def test_nvfp4_4gpu_mtp_ar(self): - max_draft_len = 7 - mtp_config = MTPDecodingConfig( - max_draft_len=max_draft_len, - mtp_eagle_one_model=True, - ) - model_path = f"{llm_models_root()}/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4" - - llm_common_config = dict( - model=model_path, - tensor_parallel_size=4, - moe_expert_parallel_size=4, - kv_cache_config=KvCacheConfig( - enable_block_reuse=False, - mamba_ssm_cache_dtype="float16", - free_gpu_memory_fraction=0.5, - ), - max_batch_size=4, - enable_attention_dp=True, - cuda_graph_config=CudaGraphConfig(max_batch_size=4, - enable_padding=True), - disable_overlap_scheduler=False, - moe_config=MoeConfig(backend="CUTLASS"), - ) - - with LLM(**llm_common_config, - speculative_config=mtp_config) as llm_spec: - raw_prompts = [ - "The capital of France is", - "The president of the United States is", - "The future of AI is", - ] - prompts = [ - llm_spec.tokenizer.apply_chat_template( - [{ - "role": "user", - "content": p - }], - tokenize=False, - add_generation_prompt=True, - ) for p in raw_prompts - ] - tok_ids = [llm_spec.tokenizer.encode(p) for p in prompts] - - sampling_params = SamplingParams(max_tokens=128, temperature=0) - - for i in range(len(tok_ids)): - num_tokens = 0 - num_drafted = 0 - num_accepted = 0 - for output in llm_spec.generate_async(tok_ids[i], - sampling_params, - streaming=True): - new_tokens = output.outputs[0].token_ids - num_drafted += max_draft_len - num_accepted += len(new_tokens) - num_tokens - 1 - num_tokens = len(new_tokens) - - accept_rate = num_accepted / num_drafted - assert accept_rate > 0.2, \ - f"Acceptance rate too low for prompt {i}: {accept_rate:.2f}" - @skip_pre_hopper class TestMiniMaxM2(LlmapiAccuracyTestHarness): diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 2d152277e2f2..560573d99a25 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -171,7 +171,6 @@ accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_re accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[TP2_PP2] accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[TEP2_PP2] accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpu_mtp_ar accuracy/test_llm_api_autodeploy.py::TestQwen3_5_397B_MoE::test_nvfp4[8] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[latency] accuracy/test_llm_api_pytorch.py::TestDeepSeekR1::test_fp8_blockscale[throughput] @@ -689,11 +688,6 @@ accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[laten accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_nvfp4_4gpus[latency_moe_trtllm_eagle] accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_fp8 -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-False] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-True] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-False] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-True] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_bf16_4gpu_mtp_ar accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache] @@ -705,9 +699,6 @@ accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_e accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=TRTLLM] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-cutlass] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-trtllm] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-cutedsl] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op accuracy/test_llm_api_pytorch.py::TestNemotronV3Nano::test_nvfp4_marlin_multi_gpus[tp_size=8] @@ -718,9 +709,7 @@ accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TE accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP8_PP1] accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP8_PP1_ADP] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP_MTP] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_bf16_trtllm_gen_moe_backend[attention_dp=True] -accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_bf16_trtllm_gen_moe_backend[attention_dp=False] +accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_ON] accuracy/test_llm_api_pytorch.py::TestPhi4MiniInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen2_7BInstruct::test_auto_dtype accuracy/test_llm_api_pytorch.py::TestQwen3_235B_A22B::test_fp8[latency] diff --git a/tests/integration/test_lists/test-db/l0_dgx_b200.yml b/tests/integration/test_lists/test-db/l0_dgx_b200.yml index 3997ab6b6197..1cca270225c7 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_b200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_b200.yml @@ -169,11 +169,6 @@ l0_dgx_b200: - accuracy/test_disaggregated_serving.py::TestKimiK25::test_nvfp4 TIMEOUT (180) - accuracy/test_llm_api_pytorch.py::TestKimiK25::test_nvfp4[tp8] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp TIMEOUT (60) - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op TIMEOUT (60) - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-trtllm] TIMEOUT (60) - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-cutlass] TIMEOUT (60) - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus[attention_dp_on-cutedsl] TIMEOUT (60) - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_on-cutedsl] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_8gpus[attention_dp_off-trtllm] TIMEOUT (60) - accuracy/test_disaggregated_serving.py::TestNemotron3Super120B::test_auto_dtype[mtp_nextn=0-block_reuse=False-use_py_transceiver=True] TIMEOUT (60) @@ -254,6 +249,8 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch_multimodal.py::TestKimiK25::test_nvfp4[dep8] TIMEOUT (120) - accuracy/test_llm_api_pytorch.py::TestMistralLarge3_675B::test_fp8[latency_moe_deepgemm] TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP8_PP1] TIMEOUT (60) + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_parallelism[TP4_PP2] TIMEOUT (60) + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_8gpus_mtp_custom_op TIMEOUT (60) - test_e2e.py::test_deepseek_r1_mtp_bench TIMEOUT(60) # Cover https://nvbugs/5670108 - examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[cfg2_ulysses2_attn2d_2x1] - examples/visual_gen/test_visual_gen_multi_gpu.py::test_wan22_t2v_lpips_against_golden_multi_gpu[attn2d_2x2_ulysses2] @@ -296,10 +293,6 @@ l0_dgx_b200: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=2-pp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-low_precision_combine=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTEDSL-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_guided_decoding_4gpus[xgrammar-mtp_nextn=0] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-False] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-1-True] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-False] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_auto_dtype_4gpus[4-4-True] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpu_mtp_ar_custom_op TIMEOUT (60) - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_static_eplb[moe_backend=CUTEDSL] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_static_eplb[moe_backend=CUTLASS] diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index dc648b9b291b..8739c1851879 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -101,10 +101,6 @@ l0_dgx_h100: - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=TRTLLM-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp2pp2-fp8kv=True-attn_backend=TRTLLM-torch_compile=True] - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_guided_decoding_4gpus[xgrammar] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_hopper_w4a16 - test_e2e.py::test_ptp_quickstart_advanced_bs1 - test_e2e.py::test_ptp_quickstart_advanced_deepseek_v3_lite_4gpus_adp_balance[DeepSeek-V3-Lite-FP8-DeepSeek-V3-Lite/fp8] @@ -412,3 +408,7 @@ l0_dgx_h100: tests: # ---- FP8 per-tensor (QDQ) moved to post-merge; block-scale stays in pre ---- - unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "CUTLASS and FP8 and not FP8_BLOCK_SCALES" + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-python_mamba_cache] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_off-cpp_mamba_cache] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-python_mamba_cache] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_fp8_4gpus[attention_dp_on-cpp_mamba_cache] diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml index e917de07dd4a..ebad6367b605 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_gpus.yml @@ -30,14 +30,8 @@ l0_gb200_multi_gpus: - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=2-pp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-tp4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=True-torch_compile=False] - accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus_online_eplb[fp8kv=True-moe_backend=TRTLLM] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4_ADP_MTP] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_ON] - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_block_reuse[ADP4_MTP] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpu_mtp_ar - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL] - accuracy/test_llm_api_pytorch.py::TestQwen3_5_397B_A17B::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM] - accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_fp8_tp4[torch_compile=False] @@ -82,6 +76,11 @@ l0_gb200_multi_gpus: stage: post_merge backend: pytorch tests: + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_online_eplb[moe_backend=TRTLLM] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[TEP4] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpus_block_reuse[DEP4_MTP_OFF] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_4gpus_online_eplb[moe_backend=CUTEDSL] + - accuracy/test_llm_api_pytorch.py::TestNemotronV3Ultra::test_nvfp4_parallelism[ADP2_PP2] - accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[tep4] - accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_bf16_4gpu[dep4] - accuracy/test_llm_api_pytorch.py::TestQwen3NextInstruct::test_nvfp4[dep4-cutlass] @@ -128,6 +127,4 @@ l0_gb200_multi_gpus: - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4_4gpus[v2_kv_cache-dp4-trtllm-auto] - accuracy/test_llm_api_pytorch.py::TestGPTOSS::test_w4a16[dp4-auto] # ---- moved to post-merge (MoE CI optimization) ---- - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_bf16_trtllm_gen_moe_backend[attention_dp=True] - - accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_bf16_trtllm_gen_moe_backend[attention_dp=False] - unittest/_torch/modules/moe/test_moe_comm.py::TestMoEComm::test_moe_comm_boundary diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 4a67432029bc..347d3c69a9b0 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -196,7 +196,6 @@ full:B300/accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_auto_ full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[baseline_pp4_mtp1] SKIP (https://nvbugs/6423845) full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_nvfp4_multi_gpus[latency] SKIP (https://nvbugs/6423866) full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=True-v2_kv_cache=True] SKIP (https://nvbugs/6422343) -full:B300/accuracy/test_llm_api_pytorch.py::TestNemotronV3Super::test_nvfp4_4gpu_mtp_ar SKIP (https://nvbugs/6336990) full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[forced_chunked_prefill] SKIP (https://nvbugs/6422318) full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestExaone4_5_33B::test_auto_dtype[full_budget] SKIP (https://nvbugs/6422318) full:B300/accuracy/test_llm_api_pytorch_multimodal.py::TestQwen2_5_VL_7B::test_auto_dtype SKIP (https://nvbugs/6316983)