diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv3.py b/tensorrt_llm/_torch/models/modeling_deepseekv3.py index c95aac6c8147..235e7e9a0a74 100755 --- a/tensorrt_llm/_torch/models/modeling_deepseekv3.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv3.py @@ -1897,25 +1897,14 @@ class DeepseekV3ForCausalLM(SpecDecOneEngineForCausalLM[DeepseekV3Model, def get_preferred_transceiver_runtime(cls, pretrained_config: Any = None ) -> Optional[Literal["PYTHON"]]: - """GLM-5 family checkpoints default to the Python (v2) KV-cache transceiver. - - This implementation class is shared by DeepSeek-V3/V3.2 and the GLM-5 family — both - GLM-5 and GLM-5.2 declare ``GlmMoeDsaForCausalLM`` / ``glm_moe_dsa`` — so the preference - is differentiated per checkpoint: only GLM checkpoints opt into the Python transceiver. - The MLA backbone transfers a large latent KV, which the Python transceiver handles better - in disaggregated serving. This is only adopted when the user leaves - ``cache_transceiver_config.transceiver_runtime`` at 'auto' and the effective backend is - NIXL; otherwise the C++ transceiver is used. + """``DeepseekV3ForCausalLM``, ``DeepseekV32ForCausalLM``, and ``GlmMoeDsaForCausalLM`` all prefer the Python (v2) KV-cache transceiver. + + All three use MLA attention (``DeepseekV3Attention`` and ``DeepseekV32Attention`` both + extend ``MLA``), which transfers a large latent KV that the Python transceiver handles + better in disaggregated serving. Applied only when ``transceiver_runtime`` is 'auto' + and the backend is NIXL. """ - if pretrained_config is None: - return None - architectures = getattr(pretrained_config, 'architectures', None) or [] - # model_type is checked as a fallback: it is 'glm_moe_dsa' on GLM - # checkpoints until __init__ rewrites it to 'deepseek_v32'. - if ("GlmMoeDsaForCausalLM" in architectures or getattr( - pretrained_config, 'model_type', None) == 'glm_moe_dsa'): - return "PYTHON" - return None + return "PYTHON" def __init__(self, model_config: ModelConfig[PretrainedConfig]): self.mapping_with_cp = None diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index 92e9e36230a3..8b76516d6a3f 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -64,6 +64,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: CPP kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -91,6 +92,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: CPP kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index b37e2855173b..f5d27a87811b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -63,6 +63,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: CPP disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -90,5 +91,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: CPP disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index f3d9e4d5d8b7..f5d31aea2da2 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_128k8k_con256_ctx1_pp4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -75,6 +75,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: CPP kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: &id001 @@ -102,6 +103,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 131104 backend: NIXL + transceiver_runtime: CPP kv_transfer_timeout_ms: 600000 disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml index 1e35e7e6d1a4..dfb7a1f7e5f6 100644 --- a/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb300_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-NIXL.yaml @@ -74,6 +74,7 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: CPP disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP @@ -101,5 +102,6 @@ worker_config: cache_transceiver_config: max_tokens_in_buffer: 16384 backend: NIXL + transceiver_runtime: CPP disable_overlap_scheduler: true speculative_config: *id001 diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index 183481805aed..d9dc658837f8 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -3760,13 +3760,8 @@ def test_resolve_default_backend_env_priority(self, monkeypatch): backend="UCX")._resolve_default_backend() == ("UCX", None) -class TestGlm5TransceiverPreference: - """GLM-5 defaults to the Python KV-cache transceiver in disagg. - - DeepseekV3ForCausalLM is shared by DeepSeek-V3/V3.2 and GLM-5 - (GlmMoeDsaForCausalLM); the preference must apply to GLM checkpoints - only. - """ +class TestDeepseekTransceiverPreference: + """DeepseekV3ForCausalLM, DeepseekV32ForCausalLM, and GlmMoeDsaForCausalLM all prefer the Python KV-cache transceiver.""" @staticmethod def _pretrained_config(architectures, model_type): @@ -3775,43 +3770,34 @@ def _pretrained_config(architectures, model_type): cfg.model_type = model_type return cfg - @pytest.mark.parametrize( - "architectures,model_type,expected", - [ - (["GlmMoeDsaForCausalLM"], "glm_moe_dsa", "PYTHON"), - (["DeepseekV3ForCausalLM"], "deepseek_v3", None), - (["DeepseekV32ForCausalLM"], "deepseek_v32", None), - # Each predicate in isolation: the architecture match and the - # model_type fallback must each suffice on their own. - (["GlmMoeDsaForCausalLM"], "deepseek_v32", "PYTHON"), - (["DeepseekV32ForCausalLM"], "glm_moe_dsa", "PYTHON"), - ]) - def test_preference_per_architecture(self, architectures, model_type, - expected): + @pytest.mark.parametrize("architectures,model_type", [ + (["GlmMoeDsaForCausalLM"], "glm_moe_dsa"), + (["DeepseekV3ForCausalLM"], "deepseek_v3"), + (["DeepseekV32ForCausalLM"], "deepseek_v32"), + ]) + def test_preference_per_architecture(self, architectures: list[str], + model_type: str) -> None: from tensorrt_llm._torch.models.modeling_deepseekv3 import \ DeepseekV3ForCausalLM cfg = self._pretrained_config(architectures, model_type) assert DeepseekV3ForCausalLM.get_preferred_transceiver_runtime( - cfg) == expected + cfg) == "PYTHON" - def test_no_config_defers_to_cpp(self): - """Without a pretrained config the class defers to the C++ default.""" + def test_prefers_python_without_config(self) -> None: + """Preference is unconditional without a pretrained config.""" from tensorrt_llm._torch.models.modeling_deepseekv3 import \ DeepseekV3ForCausalLM - assert DeepseekV3ForCausalLM.get_preferred_transceiver_runtime() is None - - def test_glm5_resolves_auto_to_python_on_nixl(self): - """GLM-5 on NIXL adopts the Python transceiver from 'auto'. + assert DeepseekV3ForCausalLM.get_preferred_transceiver_runtime( + ) == "PYTHON" - End-to-end through _resolve_transceiver_runtime_auto with the real - model class and a GLM pretrained config. - """ + def test_deepseek_resolves_auto_to_python_on_nixl(self) -> None: + """DeepseekV3ForCausalLM on NIXL adopts the Python transceiver from 'auto'.""" from tensorrt_llm._torch.models.modeling_deepseekv3 import \ DeepseekV3ForCausalLM args = TorchLlmArgs( model="/tmp/dummy_model", cache_transceiver_config=CacheTransceiverConfig(backend="NIXL"), ) - cfg = self._pretrained_config(["GlmMoeDsaForCausalLM"], "glm_moe_dsa") + cfg = self._pretrained_config(["DeepseekV3ForCausalLM"], "deepseek_v3") _resolve_transceiver_runtime_auto(args, DeepseekV3ForCausalLM, cfg) assert args.cache_transceiver_config.transceiver_runtime == "PYTHON"