From dfe10b981c043190a2baa0b45cd8c402ddc1e85b Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sat, 1 Aug 2026 07:33:33 -0700 Subject: [PATCH 1/8] [TRTLLM-14903][fix] Free partially-allocated warmup dummy KV blocks and count spec extra tokens in warmup block estimates Two defects combined to hang LLM startup indefinitely during KV cache size estimation for Mamba-hybrid models with speculative decoding: 1. _create_warmup_request under-counted blocks_to_use: it ignored the per-sequence extra tokens (num_extra_kv_tokens, num_extra_decoding_steps, and the draft-token reserve for generation dummies) that add_dummy_requests actually allocates. With spec decoding, block-aligned multi-sequence warmup shapes (e.g. the Mamba hybrid multi-seq warmup) passed the estimate but overflowed the pool at allocation time. 2. add_dummy_requests leaked every already-registered sequence when a later add_token raised (e.g. "no free blocks left"). On the minimal KV pool built for cache-size estimation the leak left too few blocks for the estimation requests themselves, so the executor loop spun forever without scheduling them and LLM() never returned. Fix blocks_to_use to mirror the real allocation, and make add_dummy_requests remove already-registered sequences before re-raising, preserving callers' skip-on-failure semantics. Verified on a spec-decoding estimation-phase integration run on a Mamba-hybrid model (previously hung unboundedly; now completes with logits parity against a non-speculative baseline). Signed-off-by: Brian Nguyen --- .../_torch/pyexecutor/model_engine.py | 24 +++- .../_torch/pyexecutor/resource_manager.py | 130 +++++++++++------- 2 files changed, 98 insertions(+), 56 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 3fdd8190dd39..3927970e7d4f 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -2234,10 +2234,26 @@ def _create_warmup_request( if num_ctx_requests + num_gen_requests > self.batch_size: return None # Not enough batch size to fill the request - blocks_to_use = num_full_seqs * math.ceil( - max_seq_len / kv_cache_manager.tokens_per_block) + math.ceil( - num_left_over_tokens / kv_cache_manager.tokens_per_block - ) + num_gen_requests * self.max_beam_width + # Mirror add_dummy_requests' actual allocation: on top of the raw + # token count, every sequence gets num_extra_kv_tokens + + # num_extra_decoding_steps add_token calls, and generation dummies + # additionally reserve the draft-loop tokens. When a sequence length + # lands near a block boundary (e.g. spec decoding's extra tokens on + # top of an exactly block-aligned split), each of those add_token + # calls costs one extra block per sequence. Under-counting them here + # let warmup start an allocation that fails midway and, before the + # partial-allocation cleanup below existed, permanently leaked most + # of the estimation-sized KV pool (TRTLLM-14903). + tokens_per_block = kv_cache_manager.tokens_per_block + extra_ctx_tokens = (getattr(kv_cache_manager, "num_extra_kv_tokens", 0) + or 0) + num_extra_decoding_steps + extra_gen_tokens = extra_ctx_tokens + self.max_draft_loop_tokens + blocks_to_use = (num_full_seqs * math.ceil( + (max_seq_len + extra_ctx_tokens) / tokens_per_block) + (math.ceil( + (num_left_over_tokens + extra_ctx_tokens) / + tokens_per_block) if num_left_over_tokens > 0 else 0) + + num_gen_requests * self.max_beam_width * math.ceil( + (1 + extra_gen_tokens) / tokens_per_block)) if blocks_to_use > available_blocks and isinstance( kv_cache_manager, KVCacheManager): diff --git a/tensorrt_llm/_torch/pyexecutor/resource_manager.py b/tensorrt_llm/_torch/pyexecutor/resource_manager.py index 9248c1cbe87f..aab02b86ed88 100644 --- a/tensorrt_llm/_torch/pyexecutor/resource_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/resource_manager.py @@ -986,59 +986,85 @@ def add_dummy_requests( _populate_dummy_mrope_config(req, token_num, is_gen) requests.append(req) - # Use add_sequence_batch for all dummy requests, then add extra tokens. - # This must happen before is_gen state modifications below, which may - # set prompt_len to 0 and trigger assertion in setPrepopulatedPromptLen. - if batch_request_infos: - self.impl.add_sequence_batch(batch_request_infos, - batch_llm_requests) - for req_id, token_num, _ in batch_request_infos: - for _ in range(self.num_extra_kv_tokens): - self.impl.add_token(req_id) - for _ in range(num_extra_decoding_steps): - self.impl.add_token(req_id) - - if draft_batch_request_infos and draft_kv_cache_manager is not None: - draft_kv_cache_manager.impl.add_sequence_batch( - draft_batch_request_infos, draft_batch_llm_requests) - for req_id, _, _ in draft_batch_request_infos: - for _ in range(self.num_extra_kv_tokens): - draft_kv_cache_manager.impl.add_token(req_id) - - # Set is_gen state after add_sequence_batch to avoid modifying - # prompt_len before the C++ side reads it. - if is_gen: - for i, req in enumerate(requests): - token_num = token_nums[ - i] if token_nums is not None else 1 + max_num_draft_tokens - if self.mapping.has_cp_helix(): - token_num = max(token_num, 2) - req.state = LlmRequestState.GENERATION_IN_PROGRESS - req.prompt_len = token_num - 1 - req.py_prompt_len = req.prompt_len - if self.mapping.has_cp_helix(): - if self.mapping.cp_size - 1 == self.mapping.cp_rank: - req.py_helix_is_inactive_rank = False - req.prompt_len = token_num - 1 - req.py_prompt_len = req.prompt_len - req.seqlen_this_rank_cp = req.prompt_len - req.total_input_len_cp = token_num * self.mapping.cp_size - 1 - req.py_decoding_iter = 1 - else: - req.py_helix_is_inactive_rank = True - req.prompt_len = token_num - req.py_prompt_len = req.prompt_len - req.seqlen_this_rank_cp = req.prompt_len - req.total_input_len_cp = token_num * self.mapping.cp_size - 1 - req.py_decoding_iter = 1 - req.py_draft_tokens = [1] * max_num_draft_tokens - if prepare_resource: - for _ in range(_kv_draft): - self.impl.add_token(req.request_id) - if draft_kv_cache_manager is not None: + try: + # Use add_sequence_batch for all dummy requests, then add extra tokens. + # This must happen before is_gen state modifications below, which may + # set prompt_len to 0 and trigger assertion in setPrepopulatedPromptLen. + if batch_request_infos: + self.impl.add_sequence_batch(batch_request_infos, + batch_llm_requests) + for req_id, token_num, _ in batch_request_infos: + for _ in range(self.num_extra_kv_tokens): + self.impl.add_token(req_id) + for _ in range(num_extra_decoding_steps): + self.impl.add_token(req_id) + + if draft_batch_request_infos and draft_kv_cache_manager is not None: + draft_kv_cache_manager.impl.add_sequence_batch( + draft_batch_request_infos, draft_batch_llm_requests) + for req_id, _, _ in draft_batch_request_infos: + for _ in range(self.num_extra_kv_tokens): + draft_kv_cache_manager.impl.add_token(req_id) + + # Set is_gen state after add_sequence_batch to avoid modifying + # prompt_len before the C++ side reads it. + if is_gen: + for i, req in enumerate(requests): + token_num = token_nums[ + i] if token_nums is not None else 1 + max_num_draft_tokens + if self.mapping.has_cp_helix(): + token_num = max(token_num, 2) + req.state = LlmRequestState.GENERATION_IN_PROGRESS + req.prompt_len = token_num - 1 + req.py_prompt_len = req.prompt_len + if self.mapping.has_cp_helix(): + if self.mapping.cp_size - 1 == self.mapping.cp_rank: + req.py_helix_is_inactive_rank = False + req.prompt_len = token_num - 1 + req.py_prompt_len = req.prompt_len + req.seqlen_this_rank_cp = req.prompt_len + req.total_input_len_cp = token_num * self.mapping.cp_size - 1 + req.py_decoding_iter = 1 + else: + req.py_helix_is_inactive_rank = True + req.prompt_len = token_num + req.py_prompt_len = req.prompt_len + req.seqlen_this_rank_cp = req.prompt_len + req.total_input_len_cp = token_num * self.mapping.cp_size - 1 + req.py_decoding_iter = 1 + req.py_draft_tokens = [1] * max_num_draft_tokens + if prepare_resource: for _ in range(_kv_draft): - draft_kv_cache_manager.impl.add_token( - req.request_id) + self.impl.add_token(req.request_id) + if draft_kv_cache_manager is not None: + for _ in range(_kv_draft): + draft_kv_cache_manager.impl.add_token( + req.request_id) + except Exception: + # A partial allocation failure (e.g. add_token raising "no free + # blocks left" after add_sequence_batch succeeded) must not leak + # the sequences already registered. On the minimal KV pool built + # for cache-size estimation, such a leak leaves too few blocks + # for the estimation requests themselves, so the executor loop + # spins forever without ever scheduling them and LLM startup + # hangs (TRTLLM-14903). Best-effort removal, then re-raise so + # callers keep their existing skip-on-failure semantics. + for freeing_impl, freeing_requests in ( + (self.impl, batch_llm_requests), + (draft_kv_cache_manager.impl if draft_kv_cache_manager + is not None else None, draft_batch_llm_requests), + ): + if freeing_impl is None: + continue + for req in freeing_requests: + try: + freeing_impl.remove_sequence(req.py_request_id, req, + False) + except Exception: + # The sequence may never have been registered (the + # batched add itself failed); nothing to clean up. + pass + raise return requests From 70b171738b185754ec79c3955565b68aee4b313f Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Mon, 3 Aug 2026 05:52:19 -0700 Subject: [PATCH 2/8] [None][fix] Launch LMBenchmark client without PYTHONSAFEPATH multi-round-qa.py imports its sibling utils.py through the implicit script-directory sys.path entry. When the test environment sets PYTHONSAFEPATH=1, that entry is disabled and the benchmark client exits immediately with ModuleNotFoundError: No module named 'utils', failing TestServePrefixAwareScheduling tests with 'Smoke warmup failed with rc=1' while the server is still healthy. Strip the variable from the client subprocess environment. Signed-off-by: Brian Nguyen --- .../defs/kv_cache/test_prefix_aware_scheduling.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/defs/kv_cache/test_prefix_aware_scheduling.py b/tests/integration/defs/kv_cache/test_prefix_aware_scheduling.py index 074cdf99485f..56ff2a4ac813 100644 --- a/tests/integration/defs/kv_cache/test_prefix_aware_scheduling.py +++ b/tests/integration/defs/kv_cache/test_prefix_aware_scheduling.py @@ -662,6 +662,11 @@ def _run_lmbenchmark( benchmark_debug_context = _stage_debug_context(debug_context, port, server_log, output_csv) def _popen_lmbenchmark(call_args: list[str], env: Mapping[str, str]) -> subprocess.Popen: + # multi-round-qa.py imports its sibling utils.py via the implicit + # script-directory sys.path entry. PYTHONSAFEPATH=1 (set by some CI + # environments) disables that entry and the script dies at import + # time with ModuleNotFoundError: No module named 'utils'. + env = {k: v for k, v in env.items() if k != "PYTHONSAFEPATH"} return subprocess.Popen( call_args, stdout=subprocess.PIPE, From 8e53f2b0f2b40a4ebaaa816bbd05d41651fcf885 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Mon, 3 Aug 2026 14:11:49 -0700 Subject: [PATCH 3/8] [TRTLLM-14903][fix] Clean up warmup block estimate expression Signed-off-by: Brian Nguyen --- .../_torch/pyexecutor/model_engine.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 3927970e7d4f..2a408306870c 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -2237,23 +2237,24 @@ def _create_warmup_request( # Mirror add_dummy_requests' actual allocation: on top of the raw # token count, every sequence gets num_extra_kv_tokens + # num_extra_decoding_steps add_token calls, and generation dummies - # additionally reserve the draft-loop tokens. When a sequence length - # lands near a block boundary (e.g. spec decoding's extra tokens on - # top of an exactly block-aligned split), each of those add_token - # calls costs one extra block per sequence. Under-counting them here - # let warmup start an allocation that fails midway and, before the - # partial-allocation cleanup below existed, permanently leaked most - # of the estimation-sized KV pool (TRTLLM-14903). - tokens_per_block = kv_cache_manager.tokens_per_block + # additionally reserve max_draft_loop_tokens for the draft loop. + # Under-counting these let warmup start an allocation that fails + # midway and, before the partial-allocation cleanup existed, + # permanently leaked most of the estimation-sized KV pool + # (TRTLLM-14903). + def blocks_for_seq(num_tokens: int) -> int: + return math.ceil(num_tokens / kv_cache_manager.tokens_per_block) + extra_ctx_tokens = (getattr(kv_cache_manager, "num_extra_kv_tokens", 0) or 0) + num_extra_decoding_steps extra_gen_tokens = extra_ctx_tokens + self.max_draft_loop_tokens - blocks_to_use = (num_full_seqs * math.ceil( - (max_seq_len + extra_ctx_tokens) / tokens_per_block) + (math.ceil( - (num_left_over_tokens + extra_ctx_tokens) / - tokens_per_block) if num_left_over_tokens > 0 else 0) + - num_gen_requests * self.max_beam_width * math.ceil( - (1 + extra_gen_tokens) / tokens_per_block)) + blocks_to_use = num_full_seqs * blocks_for_seq(max_seq_len + + extra_ctx_tokens) + if num_left_over_tokens > 0: + blocks_to_use += blocks_for_seq(num_left_over_tokens + + extra_ctx_tokens) + blocks_to_use += (num_gen_requests * self.max_beam_width * + blocks_for_seq(1 + extra_gen_tokens)) if blocks_to_use > available_blocks and isinstance( kv_cache_manager, KVCacheManager): From 252d83f577736ad94daa71b47ca36c52bcad9e65 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Mon, 3 Aug 2026 14:16:43 -0700 Subject: [PATCH 4/8] [TRTLLM-14903][fix] Document one-engine warmup token accounting Signed-off-by: Brian Nguyen --- tensorrt_llm/_torch/pyexecutor/model_engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorrt_llm/_torch/pyexecutor/model_engine.py b/tensorrt_llm/_torch/pyexecutor/model_engine.py index 2a408306870c..fa689f4039da 100644 --- a/tensorrt_llm/_torch/pyexecutor/model_engine.py +++ b/tensorrt_llm/_torch/pyexecutor/model_engine.py @@ -2238,6 +2238,9 @@ def _create_warmup_request( # token count, every sequence gets num_extra_kv_tokens + # num_extra_decoding_steps add_token calls, and generation dummies # additionally reserve max_draft_loop_tokens for the draft loop. + # In one-engine spec modes that is (max_draft_len - 1) extra KV + # tokens plus max_draft_len draft-loop tokens per gen dummy, i.e. + # 2 * max_draft_len - 1 on top of the single prompt token. # Under-counting these let warmup start an allocation that fails # midway and, before the partial-allocation cleanup existed, # permanently leaked most of the estimation-sized KV pool From 32da14b828acb0e2d001250dd56834db7255fd44 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Tue, 4 Aug 2026 17:07:06 -0700 Subject: [PATCH 5/8] [TRTLLM-14903][fix] Attempt all warmup cleanup and re-raise cleanup failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit remove_sequence is already a no-op for request ids the failed batched add never registered, so the per-request exception handler only masked real releaseBlocks failures — which leave the KV pool poisoned, the same hang mechanism this cleanup exists to prevent. Attempt cleanup for every target and draft request, then re-raise the first cleanup failure instead of swallowing it. Signed-off-by: Brian Nguyen --- .../_torch/pyexecutor/resource_manager.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/resource_manager.py b/tensorrt_llm/_torch/pyexecutor/resource_manager.py index aab02b86ed88..eb3d4193bd8c 100644 --- a/tensorrt_llm/_torch/pyexecutor/resource_manager.py +++ b/tensorrt_llm/_torch/pyexecutor/resource_manager.py @@ -1047,8 +1047,12 @@ def add_dummy_requests( # for cache-size estimation, such a leak leaves too few blocks # for the estimation requests themselves, so the executor loop # spins forever without ever scheduling them and LLM startup - # hangs (TRTLLM-14903). Best-effort removal, then re-raise so - # callers keep their existing skip-on-failure semantics. + # hangs (TRTLLM-14903). remove_sequence is a no-op for request + # ids the failed batched add never registered, so every request + # can be removed unconditionally; attempt all target and draft + # cleanup before re-raising so one cleanup failure doesn't leak + # the remaining sequences. + cleanup_error = None for freeing_impl, freeing_requests in ( (self.impl, batch_llm_requests), (draft_kv_cache_manager.impl if draft_kv_cache_manager @@ -1060,10 +1064,13 @@ def add_dummy_requests( try: freeing_impl.remove_sequence(req.py_request_id, req, False) - except Exception: - # The sequence may never have been registered (the - # batched add itself failed); nothing to clean up. - pass + except Exception as e: + cleanup_error = cleanup_error or e + if cleanup_error is not None: + # A failed release leaves the KV pool poisoned — the same + # hang mechanism this cleanup exists to prevent — so it + # must not be masked by the allocation failure alone. + raise cleanup_error raise return requests From 01196c04d6571fc7f90233c03c54323f239f8025 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Wed, 5 Aug 2026 02:36:36 -0700 Subject: [PATCH 6/8] [TRTLLM-14903][test] Cover partial add_dummy_requests failure cleanup A 4-block pool admits both dummy sequences but runs out of blocks in the per-request draft add_token loop, exercising the partial-failure path: the exception must propagate and every already-allocated block must be freed. Fails against the pre-fix code (blocks leak), passes with the cleanup. Signed-off-by: Brian Nguyen --- .../_torch/executor/test_resource_manager.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unittest/_torch/executor/test_resource_manager.py b/tests/unittest/_torch/executor/test_resource_manager.py index 9a230973a85a..f4ab82db1d54 100644 --- a/tests/unittest/_torch/executor/test_resource_manager.py +++ b/tests/unittest/_torch/executor/test_resource_manager.py @@ -960,6 +960,45 @@ def test_batch_cache_indices_honor_requested_blocks_with_beams(self): finally: kv_cache_manager.shutdown() + def test_add_dummy_requests_failure_frees_partial_allocation(self): + """A partial add_dummy_requests failure must free every block it + allocated (TRTLLM-14903): leaked blocks on the minimal pool built for + cache-size estimation starve the estimation requests and hang startup. + """ + kv_cache_manager = KVCacheManager( + kv_cache_config=KvCacheConfig(max_tokens=256, + enable_block_reuse=False), + kv_cache_type=tensorrt_llm.bindings.internal.batch_manager. + CacheType.SELF, + num_layers=2, + num_kv_heads=2, + head_dim=128, + tokens_per_block=64, + max_seq_len=1024, + max_batch_size=2, + mapping=Mapping(), + ) + try: + total_free = kv_cache_manager.get_num_free_blocks() + self.assertEqual(total_free, 4) + # Both sequences fit in one block each, but the per-request draft + # add_token loop needs two more blocks per request: request 0 + # drains the pool and request 1's first add_token raises, after + # three of the four blocks were already allocated. + with self.assertRaises(Exception): + kv_cache_manager.add_dummy_requests([0, 1], + token_nums=[64, 64], + is_gen=True, + max_num_draft_tokens=128) + self.assertEqual(kv_cache_manager.get_num_free_blocks(), total_free) + # The freed pool must serve follow-up allocations. + requests = kv_cache_manager.add_dummy_requests([2], token_nums=[64]) + self.assertIsNotNone(requests) + kv_cache_manager.free_resources(requests[0]) + self.assertEqual(kv_cache_manager.get_num_free_blocks(), total_free) + finally: + kv_cache_manager.shutdown() + def test_kv_cache_manager_with_execution_stream(self): """ Test that KVCacheManager uses the provided execution_stream. From 1fa12afdb86962efe77a3249ad204fea5582d621 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Wed, 5 Aug 2026 08:49:52 -0700 Subject: [PATCH 7/8] [None][ci] Waive AutoDeploy MoE unit tests broken on main unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py and unittest/auto_deploy/singlegpu/transformations/library/test_moe_fusion.py fail in DGX_B200-AutoDeploy-1 for any PR rebased onto current main: identical failures on pipelines 51974 (this PR) and 51977 (PR #17225, zero file overlap). Suspect commit 89bba4cfd9 (#15297), which modifies the trtllm_moe custom op and Blackwell blockScaleMoe kernels. Waived pending an NVBug; the entries will be updated with the bug link once filed. Signed-off-by: Brian Nguyen --- tests/integration/test_lists/waives.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index b2ff70b86a17..b27230c9bdec 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -433,6 +433,8 @@ unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allred unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[ONESHOT] SKIP (https://nvbugs/6517839) unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[SYMM_MEM] SKIP (https://nvbugs/6517839) unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[TWOSHOT] SKIP (https://nvbugs/6517839) +unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py SKIP (AutoDeploy MoE unit tests broken on main since 2026-08-05; nvbug pending) +unittest/auto_deploy/singlegpu/transformations/library/test_moe_fusion.py SKIP (AutoDeploy MoE unit tests broken on main since 2026-08-05; nvbug pending) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_wait_in_progress_on_zero_timeout SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_gpu_tensor SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_multiple_chunks SKIP (https://nvbugs/6517836) From c4f06e9f5391e1b22eddddc971bf7df444e3100c Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Wed, 5 Aug 2026 10:04:51 -0700 Subject: [PATCH 8/8] [None][ci] Add nvbug link to AutoDeploy MoE unit test waives Cites nvbugs/6564714 for the two waives added in the previous commit. Signed-off-by: Brian Nguyen --- tests/integration/test_lists/waives.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index b27230c9bdec..f5813c58ff0b 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -433,8 +433,8 @@ unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allred unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[ONESHOT] SKIP (https://nvbugs/6517839) unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[SYMM_MEM] SKIP (https://nvbugs/6517839) unittest/auto_deploy/multigpu/smoke/test_ad_allreduce_strategies.py::test_allreduce_strategies[TWOSHOT] SKIP (https://nvbugs/6517839) -unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py SKIP (AutoDeploy MoE unit tests broken on main since 2026-08-05; nvbug pending) -unittest/auto_deploy/singlegpu/transformations/library/test_moe_fusion.py SKIP (AutoDeploy MoE unit tests broken on main since 2026-08-05; nvbug pending) +unittest/auto_deploy/singlegpu/custom_ops/moe/test_trtllm_moe.py SKIP (https://nvbugs/6564714) +unittest/auto_deploy/singlegpu/transformations/library/test_moe_fusion.py SKIP (https://nvbugs/6564714) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_wait_in_progress_on_zero_timeout SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_gpu_tensor SKIP (https://nvbugs/6517836) unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfer::test_mooncake_write_transfer_multiple_chunks SKIP (https://nvbugs/6517836)