From 91998bd6d8d1efea385c4591e3bd57846d45aafe Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Fri, 24 Jul 2026 07:07:58 +0000 Subject: [PATCH 1/3] [None][feat] Default fabric memory KV pool for Python cache transceiver Default TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 when the Python cache transceiver (transceiver_runtime='PYTHON') is used with the C++ V1 KV cache manager, so the KV pool is allocated with fabric memory for MNNVL transfers. Explicit user settings are respected, and unsupported platforms fall back to standard allocation in C++. Also set kv_cache_bounce_size_mb=512 in the GPT-OSS disagg perf configs where the ctx/gen KV-head sharding differs (ctx TP1 -> gen TP4), which requires the bounce buffer to regroup KV heads during transfer. Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/_util.py | 25 +++++++++++++++++++ ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 ++ ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 ++ ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 ++ ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 ++ ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 ++ ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 ++ ...ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml | 2 ++ ...x1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml | 2 ++ 9 files changed, 41 insertions(+) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 577eb23301e2..224e3fc8d6c7 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -421,6 +421,31 @@ def __init__( KVCacheManagerV2) self._draft_config = draft_config self._skip_est = skip_est + self._maybe_enable_fabric_memory_for_python_transceiver() + + def _maybe_enable_fabric_memory_for_python_transceiver(self): + """Default TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 for the Python + transceiver on the C++ V1 KV cache manager. + + The Python transceiver (KvCacheTransceiverV2) transfers KV blocks + directly out of the C++ pool, so the pool should be allocated with + fabric memory to enable MNNVL transfers. This must run before any + pool allocation because the C++ env getter caches the value on first + read. Explicit user settings are respected, and platforms without + fabric memory support fall back to standard allocation in C++. + """ + if (self._cache_transceiver_config is None + or self._cache_transceiver_config.backend is None or + self._cache_transceiver_config.transceiver_runtime != "PYTHON"): + return + if not issubclass(self._kv_cache_manager_cls, KVCacheManager): + return + if os.environ.get("TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY") is None: + os.environ["TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY"] = "1" + logger.info( + "Python cache transceiver with C++ KV cache manager detected; " + "defaulting TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 (set it " + "to 0 explicitly to disable)") def _get_model_kv_cache_manager_cls( self, diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9c82b55a4a17..53736a99c780 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_1k1k_con64_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: max_tokens_in_buffer: 1024 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -88,4 +89,5 @@ worker_config: max_tokens_in_buffer: 1024 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9f4b7086060d..ffa8cdfa4522 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -75,6 +75,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: false num_postprocess_workers: 4 stream_interval: 20 @@ -103,6 +104,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index c9f73573fcf9..3f00a0063523 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -88,4 +89,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9610d4a5bdf9..edca66a96d87 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -88,4 +89,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9f4b7086060d..ffa8cdfa4522 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con1024_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -75,6 +75,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: false num_postprocess_workers: 4 stream_interval: 20 @@ -103,6 +104,7 @@ worker_config: max_tokens_in_buffer: 9216 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 9ed015cd6cda..afa141b6ae82 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con128_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -99,4 +100,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml index 5476fae34c0e..c8dffadebfa7 100644 --- a/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_gpt-oss-120b-fp4_8k1k_con4_ctx1_tp1_gen1_tp4_eplb0_mtp0_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true num_postprocess_workers: 4 stream_interval: 20 @@ -99,4 +100,5 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true diff --git a/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml index 0701596a0bad..9e9dd0877c90 100644 --- a/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_stress-gpt-oss-120b-fp4_8k1k_ctx1_tp1_gen1_tp4_eplb0_eagle3_ccb-NIXL.yaml @@ -80,6 +80,7 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true speculative_config: &id001 decoding_type: Eagle @@ -110,5 +111,6 @@ worker_config: max_tokens_in_buffer: 8448 backend: NIXL transceiver_runtime: PYTHON + kv_cache_bounce_size_mb: 512 disable_overlap_scheduler: true speculative_config: *id001 From 524d48b248df81ed15fa2eb5904c9e4844af09e5 Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Mon, 27 Jul 2026 06:27:21 +0000 Subject: [PATCH 2/3] unwaive some case on gb300 Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 03c6fc71e3d3..32c87d9bc6e4 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -323,7 +323,6 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-glm5_fp4_blackwell-glm5_fp4_tep8_ perf/test_perf_sanity.py::test_e2e[aggr_upload-k25_thinking_fp4_2_nodes_grace_blackwell-k25_thinking_fp4_tep8_32k8k] SKIP (https://nvbugs/6422339) perf/test_perf_sanity.py::test_e2e[aggr_upload-super_ad_blackwell-super_ad_ws1_1k1k] SKIP (https://nvbugs/6153575) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6426890) -perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] SKIP (https://nvbugs/6517846) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6490049) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6478615) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6490049) @@ -345,8 +344,6 @@ perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro- perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con4301_ctx12_dep4_gen1_dep8_eplb384_mtp1_ccb-NIXL] SKIP (https://nvbugs/6517846) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con666_ctx6_dep4_gen1_dep16_eplb384_mtp3_ccb-NIXL] SKIP (https://nvbugs/6517846) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6517846) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con1024_ctx1_dep2_gen1_dep8_eplb256_mtp1_ccb-NIXL] SKIP (https://nvbugs/6517846) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_glm-5-fp4_8k1k_con512_ctx1_dep2_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6517846) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb416_mtp3_ccb-NIXL] SKIP (https://nvbugs/6490049) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6490049) test_doc.py::test_url_validity SKIP (https://nvbugs/6215684) From 66471a3fe1a9ab62dd44279c708b842f5a9c7dae Mon Sep 17 00:00:00 2001 From: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> Date: Wed, 29 Jul 2026 01:55:27 +0000 Subject: [PATCH 3/3] [None][chore] Add return annotation to fabric memory helper Signed-off-by: Chuang Zhu <111838961+chuangz0@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 224e3fc8d6c7..40ab2e3a64ed 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -423,7 +423,7 @@ def __init__( self._skip_est = skip_est self._maybe_enable_fabric_memory_for_python_transceiver() - def _maybe_enable_fabric_memory_for_python_transceiver(self): + def _maybe_enable_fabric_memory_for_python_transceiver(self) -> None: """Default TRTLLM_KVCACHE_POOL_USE_FABRIC_MEMORY=1 for the Python transceiver on the C++ V1 KV cache manager.