From 7a289c7040ace82548c047ab7ecea3f68f3abbea Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:38:56 -0700 Subject: [PATCH] [nvbugs/6566772][fix] Don't block request fetch after shutdown sentinel PyExecutor._fetch_and_enqueue_requests chose an unbounded request_queue.get() whenever an iteration was idle, including after the shutdown sentinel had already been consumed. Once that sentinel is drained, enqueue_shutdown_request() has cleared the queue's active flag, so no further item can ever arrive to wake the get(). The executor loop therefore never reached the should_stop_processing check that ends it, and PyExecutor.shutdown() blocked forever on shutdown_event, which in turn hung the proxy's f.result() on the worker MPI future during LLM teardown. Whether the loop escaped depended on whether requests were still active when the sentinel was drained, which made the resulting teardown hang intermittent. Select a non-blocking timeout once is_shutdown is set, so the iteration completes and the loop reaches its exit check. Requests still in flight continue to drain as before. Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 6cf98b759f02..78642af31d28 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -4947,8 +4947,13 @@ def _fetch_and_enqueue_requests(self, waiting_queue: WaitingQueue, if len(self.control_requests) != 0: return - # Calculate timeout - idle = (total_num_active_requests == 0) and len(waiting_queue) == 0 + # Calculate timeout. Never wait once the shutdown sentinel has been + # consumed: no further item will arrive to wake a blocking get(), so + # blocking would keep the loop from reaching the + # `should_stop_processing` check that ends it, deadlocking shutdown() + # on `shutdown_event`. + idle = (total_num_active_requests == 0 and len(waiting_queue) == 0 + and not self.is_shutdown) if idle: # In Ray path (TLLM_DISABLE_MPI=1), use a periodic heartbeat timeout so rank 0 # reaches the broadcast path regularly to prevent trtllm-serve timeout when idle.