From 8a13e92e1e797994bed5c47d2cea2a119605f8b6 Mon Sep 17 00:00:00 2001 From: xxi Date: Thu, 6 Aug 2026 08:26:34 +0000 Subject: [PATCH] [https://nvbugs/6567403][fix] Revert #17010 to unblock DGX_H100 stage timeout This reverts commit 60e7fcaeaf491a72f18ea4b547aea377855d8eb1 (#17010), which made DGX_H100-4_GPUs-PyTorch-Others-1 time out at 244-267 min instead of the usual 67-83 min. Evidence: - PostMerge on main flipped at this commit: build 2885 (43d6fa410c) succeeded in 72 min, build 2886 (91fb4433c5) failed at 262 min. - Intersecting the main bad range (43d6fa410c..91fb4433c5) with PR 17119 bad range (5dab9bf760..e7e9da5148) leaves exactly this commit. - Across 22 builds from 14 PRs plus main, containing this commit predicts the timeout: 10 SUCCESS / 0 FAILURE without it, 0 SUCCESS / 11 FAILURE with it. - The stage stalls in test_trtllm_bench_llmapi_launch, which otherwise runs in 118-125 s; it hangs until the session is killed or fails from mpirun. Keeping a strong reference to every captured CUDA graph output tensor for the runner lifetime pins one output buffer per graph key, so the capture-time storage can no longer return to the shared graph memory pool. The nvbugs/6567403 waives are removed too: the three MoE multi-GPU cases were only collateral damage and pass in 45-76 s once the stage completes normally. The nvbugs/6525011 waives removed by #17010 are restored by this revert. Signed-off-by: xxi --- tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py | 12 ------------ tests/integration/test_lists/waives.txt | 5 ++--- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py index 3fa8edd8cdd1..07618d87a1a9 100644 --- a/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py +++ b/tensorrt_llm/_torch/pyexecutor/cuda_graph_runner.py @@ -139,10 +139,6 @@ def __init__(self, config: CUDAGraphRunnerConfig): self.graphs: Dict[KeyType, torch.cuda.CUDAGraph] = {} self.graph_outputs: Dict[KeyType, Callable[[], Optional[torch.Tensor]]] = {} - # graph_outputs holds only non-owning weak refs, so these strong refs are - # what stop the capture-time output storage from returning to the shared - # graph pool and being reused while the graph is still replayable. - self._graph_output_refs: Dict[KeyType, Any] = {} self.graph_metadata: Dict[KeyType, Dict[str, Any]] = {} self.memory_pool = config.cuda_graph_mem_pool self.padding_dummy_requests: Dict[int, LlmRequest] = {} @@ -541,7 +537,6 @@ def _setup_spec_decoding_and_forward(key: KeyType, forward_fn: Callable, saved_kv_lens_cuda) self.graphs[key] = graph - self._graph_output_refs[key] = output graph_output = make_weak_ref(output) self.graph_outputs[key] = graph_output self.memory_pool = graph.pool() @@ -850,9 +845,6 @@ def pad_batch(self, def clear(self): """Releases all captured graphs and the associated memory pool.""" - # Drop the output buffers while the pool that backs them is still alive; - # freeing them after graph.reset() trips the allocator's use_count check. - self._graph_output_refs.clear() for graph in self.graphs.values(): graph.reset() self.graphs.clear() @@ -908,8 +900,6 @@ def __init__(self, config: EncoderCUDAGraphRunnerConfig): self.graphs: Dict[EncoderKeyType, torch.cuda.CUDAGraph] = {} self.graph_outputs: Dict[EncoderKeyType, Callable[[], Optional[Any]]] = {} - # See CUDAGraphRunner._graph_output_refs. - self._graph_output_refs: Dict[EncoderKeyType, Any] = {} self.graph_metadata: Dict[EncoderKeyType, Dict[str, Any]] = {} self.memory_pool = config.cuda_graph_mem_pool @@ -1229,7 +1219,6 @@ def capture( "Encoder CUDA graph does not support nested tensor outputs. " "Disable encoder CUDA graphs for models with ragged outputs.") self.graphs[key] = graph - self._graph_output_refs[key] = output graph_output = make_weak_ref(output) self.graph_outputs[key] = graph_output self.memory_pool = graph.pool() @@ -1301,7 +1290,6 @@ def get_graph_pool(self): return self.memory_pool def clear(self): - self._graph_output_refs.clear() for graph in self.graphs.values(): graph.reset() self.graphs.clear() diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index b2971766eacd..c3ad766a084a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -153,6 +153,7 @@ full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4g full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6526186) full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_cute_dsl_nvfp4_4gpus[tp4-fp8kv=False-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True] SKIP (https://nvbugs/6474888) full:B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_4gpus_static_eplb[moe_backend=WIDEEP] SKIP (https://nvbugs/6546609) +full:B200/accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8 SKIP (https://nvbugs/6525011) full:B200/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8[fp8kv=True-attn_backend=TRTLLM-torch_compile=True] SKIP (https://nvbugs/6473161) full:B200/accuracy/test_llm_api_pytorch.py::TestLlama3_1_8BInstruct::test_fp8_4gpus[tp4-fp8kv=True-attn_backend=TRTLLM-torch_compile=True] SKIP (https://nvbugs/6473161) full:B200/accuracy/test_llm_api_pytorch.py::TestLlama3_3_70BInstruct::test_nvfp4_tp4[torch_compile=True] SKIP (https://nvbugs/6525010) @@ -186,6 +187,7 @@ full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mt full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_nvfp4_4gpus[moe_backend=CUTLASS-mtp_nextn=0-ep4-fp8kv=True-attention_dp=True-cuda_graph=True-overlap_scheduler=True-low_precision_combine=False-torch_compile=True] SKIP (https://nvbugs/6474888) full:B300/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4FlashBase::test_fp8_4gpus_static_eplb[moe_backend=WIDEEP] SKIP (https://nvbugs/6546609) full:B300/accuracy/test_llm_api_pytorch.py::TestGemma3_1BInstruct::test_fp8_prequantized[torch_compile=True] SKIP (https://nvbugs/6475346) +full:B300/accuracy/test_llm_api_pytorch.py::TestLagunaXS::test_fp8 SKIP (https://nvbugs/6525011) full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_auto_dtype[tp_size=8-ep_size=8] SKIP (https://nvbugs/6445375) full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_mxfp8[use_msa=False] SKIP (https://nvbugs/6424188) full:B300/accuracy/test_llm_api_pytorch.py::TestMiniMaxM3::test_nvfp4[use_msa=False] SKIP (https://nvbugs/6445375) @@ -198,9 +200,6 @@ full:B300/llmapi/test_llm_api_pytorch_moe_lora.py::test_qwen_moe_routed_expert_m full:B300/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) full:DGX_B200/accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] SKIP (https://nvbugs/6501837) full:DGX_B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4ProDSpark::test_gsm8k_dep8_megamoe_deepgemm SKIP (https://nvbugs/6506920) -full:DGX_H100/unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "CUTLASS and W4A16_MXFP4" SKIP (https://nvbugs/6567403) -full:DGX_H100/unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu -k "CUTLASS and W8A16" SKIP (https://nvbugs/6567403) -full:DGX_H100/unittest/_torch/modules/moe/test_moe_module.py::test_configurable_moe_multi_gpu_eplb SKIP (https://nvbugs/6567403) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/6476233) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6476233) full:GB200/accuracy/test_disaggregated_serving.py::TestLlama3_1_8BInstruct::test_auto_dtype[False-True-True-True] SKIP (https://nvbugs/6525893)