diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml index 901ea87ea..7f2de9e42 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b200-fp4-mtp/agentic.yaml @@ -5,7 +5,7 @@ base: name: minimaxm3-fp4-b200-vllm-agentic model: path: hf:nvidia/MiniMax-M3-NVFP4 - container: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + container: vllm/vllm-openai:nightly-29468dde8b515031dc6d4d9d06bf0a2fa0442098 precision: fp4 resources: gpu_type: b200 @@ -30,10 +30,11 @@ base: workers: 1 args: served-model-name: nvidia/MiniMax-M3-NVFP4 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 block-size: 128 language-model-only: true enable-prefix-caching: true + enable-chunked-prefill: true no-enable-flashinfer-autotune: true reasoning-parser: minimax_m3 tool-call-parser: minimax_m3 @@ -47,7 +48,7 @@ base: trust-remote-code: true # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to # synthetic rejection at the golden acceptance length. - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' env: PYTHONNOUSERSITE: '1' VLLM_ENGINE_READY_TIMEOUT_S: '3600' @@ -61,13 +62,15 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. DRAM points give SimpleCPUOffload the whole host budget -# (TOTAL_CPU_DRAM_GB GiB) with lazy offload. +# (TOTAL_CPU_DRAM_GB decimal GB) with lazy offload. override_tp8_c1: roles: agg: gpus: 8 args: tensor-parallel-size: 8 + max-num-seqs: 2 + max-cudagraph-capture-size: 8 benchmark: env: CONC: '1' @@ -79,6 +82,8 @@ override_tp4_c1: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 2 + max-cudagraph-capture-size: 8 benchmark: env: CONC: '1' @@ -90,6 +95,8 @@ override_tp4_c5: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 10 + max-cudagraph-capture-size: 40 benchmark: env: CONC: '5' @@ -101,6 +108,8 @@ override_tp4_c10: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 20 + max-cudagraph-capture-size: 80 benchmark: env: CONC: '10' @@ -112,6 +121,8 @@ override_tp4_c15: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 30 + max-cudagraph-capture-size: 120 benchmark: env: CONC: '15' @@ -123,6 +134,8 @@ override_tp4_c20: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 40 + max-cudagraph-capture-size: 160 benchmark: env: CONC: '20' @@ -134,7 +147,9 @@ override_tp4_c15_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 30 + max-cudagraph-capture-size: 120 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -149,7 +164,9 @@ override_tp4_c20_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 40 + max-cudagraph-capture-size: 160 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -164,7 +181,9 @@ override_tp4_c25_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 50 + max-cudagraph-capture-size: 200 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -179,7 +198,9 @@ override_tp4_c30_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 60 + max-cudagraph-capture-size: 240 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -194,7 +215,9 @@ override_tp4_c32_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 64 + max-cudagraph-capture-size: 256 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -209,7 +232,9 @@ override_tp4_c34_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 68 + max-cudagraph-capture-size: 272 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -224,7 +249,9 @@ override_tp4_c36_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 72 + max-cudagraph-capture-size: 288 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -239,7 +266,9 @@ override_tp4_c38_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 76 + max-cudagraph-capture-size: 304 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -254,7 +283,9 @@ override_tp4_c40_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1161788653568,"lazy_offload":true}}' + max-num-seqs: 80 + max-cudagraph-capture-size: 320 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1082000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: diff --git a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml index d77552482..531850164 100644 --- a/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml @@ -5,7 +5,7 @@ base: name: minimaxm3-fp4-b300-vllm-agentic model: path: hf:nvidia/MiniMax-M3-NVFP4 - container: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + container: vllm/vllm-openai:nightly-29468dde8b515031dc6d4d9d06bf0a2fa0442098 precision: fp4 resources: gpu_type: b300 @@ -30,10 +30,11 @@ base: workers: 1 args: served-model-name: nvidia/MiniMax-M3-NVFP4 - gpu-memory-utilization: 0.9 + gpu-memory-utilization: 0.95 block-size: 128 language-model-only: true enable-prefix-caching: true + enable-chunked-prefill: true no-enable-flashinfer-autotune: true reasoning-parser: minimax_m3 tool-call-parser: minimax_m3 @@ -42,12 +43,12 @@ base: attention-config: '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' kv-cache-dtype: fp8 max-cudagraph-capture-size: 512 - max-num-batched-tokens: 16384 + max-num-batched-tokens: 32768 stream-interval: 20 trust-remote-code: true # Three-token EAGLE3 with the GQA draft head. Throughput runs switch to # synthetic rejection at the golden acceptance length. - speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASHINFER"}' env: PYTHONNOUSERSITE: '1' VLLM_ENGINE_READY_TIMEOUT_S: '3600' @@ -61,13 +62,15 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'vllm:' # One variant per point. DRAM points give SimpleCPUOffload the whole host budget -# (TOTAL_CPU_DRAM_GB GiB) with lazy offload. +# (TOTAL_CPU_DRAM_GB decimal GB) with lazy offload. override_tp8_c1: roles: agg: gpus: 8 args: tensor-parallel-size: 8 + max-num-seqs: 2 + max-cudagraph-capture-size: 8 benchmark: env: CONC: '1' @@ -79,6 +82,8 @@ override_tp4_c1: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 2 + max-cudagraph-capture-size: 8 benchmark: env: CONC: '1' @@ -90,6 +95,8 @@ override_tp4_c5: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 10 + max-cudagraph-capture-size: 40 benchmark: env: CONC: '5' @@ -101,6 +108,8 @@ override_tp4_c10: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 20 + max-cudagraph-capture-size: 80 benchmark: env: CONC: '10' @@ -112,6 +121,8 @@ override_tp4_c15: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 30 + max-cudagraph-capture-size: 120 benchmark: env: CONC: '15' @@ -123,6 +134,8 @@ override_tp4_c20: gpus: 4 args: tensor-parallel-size: 4 + max-num-seqs: 40 + max-cudagraph-capture-size: 160 benchmark: env: CONC: '20' @@ -134,14 +147,16 @@ override_tp4_c30_dram: gpus: 4 args: tensor-parallel-size: 4 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1099511627776,"lazy_offload":true}}' + max-num-seqs: 60 + max-cudagraph-capture-size: 240 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: env: CONC: '30' KV_OFFLOADING: dram - TOTAL_CPU_DRAM_GB: '1024' + TOTAL_CPU_DRAM_GB: '1499' override_tp2_c24_dram: roles: @@ -149,7 +164,10 @@ override_tp2_c24_dram: gpus: 2 args: tensor-parallel-size: 2 - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":804232626176,"lazy_offload":true}}' + gpu-memory-utilization: 0.92 + max-num-seqs: 48 + max-cudagraph-capture-size: 192 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":749000000000,"lazy_offload":true}}' env: VLLM_USE_SIMPLE_KV_OFFLOAD: '1' benchmark: @@ -157,3 +175,89 @@ override_tp2_c24_dram: CONC: '24' KV_OFFLOADING: dram TOTAL_CPU_DRAM_GB: '749' + +override_tp4_c40_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 80 + max-cudagraph-capture-size: 320 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1499' + +override_tp4_c48_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 96 + max-cudagraph-capture-size: 384 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '48' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1499' + +override_tp4_c64_dram: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + max-num-seqs: 128 + max-cudagraph-capture-size: 512 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":1499000000000,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '64' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '1499' + +override_tp2_c32_dram: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + gpu-memory-utilization: 0.92 + max-num-seqs: 64 + max-cudagraph-capture-size: 256 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":749000000000,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '32' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '749' +override_tp2_c40_dram: + roles: + agg: + gpus: 2 + args: + tensor-parallel-size: 2 + gpu-memory-utilization: 0.92 + max-num-seqs: 80 + max-cudagraph-capture-size: 320 + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":749000000000,"lazy_offload":true}}' + env: + VLLM_USE_SIMPLE_KV_OFFLOAD: '1' + benchmark: + env: + CONC: '40' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '749' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1ecb68d8b..54b06aac5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -5508,7 +5508,7 @@ qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp: additional-settings: - "CONFIG_FILE=recipes/qwen3.5/sglang/gb200-fp4/agentx/agg-variants.yaml:override_tp2ep2_hicache_cap48" minimaxm3-fp4-b300-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + image: vllm/vllm-openai:nightly-29468dde8b515031dc6d4d9d06bf0a2fa0442098 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b300-dsxe @@ -5517,14 +5517,14 @@ minimaxm3-fp4-b300-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.683 + - dram-utilization: 1.0 search-space: - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 5, 10, 15, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } - - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [30, 40, 48, 64], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } - dram-utilization: 1.0 search-space: - - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } + - { tp: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [24, 32, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/b300-fp4-mtp/agentic.yaml } minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 @@ -5636,33 +5636,34 @@ minimaxm3-fp4-b300-trtllm-agentic-mtp: # MiniMax-M3's only active B200 configuration -- the 8k1k scenario that carried # the previous B200 curves was removed in #2493. # -# TP4-only. The B300 sibling also runs a TP2 arm, but TP2 does not fit on B200: +# TP4-only until a new TP2 arm is qualified. At 0.90 HBM utilization TP2 +# does not fit on B200: # the ~250 GB NVFP4 checkpoint leaves a 31.72 GiB KV pool at # --gpu-memory-utilization 0.9, and one request at the model's 1,048,576-token # max_model_len needs 37.62 GiB, so vLLM raises in _check_enough_kv_cache_memory # and the engine never reaches serving (run 31141913741 -- both TP2 cells, c1 -# and c2, died identically at init; every TP4 cell passed). This is structural, -# not a concurrency cliff: the estimated max model length at TP2 is 883,840, -# below the corpus context, so no conc-list would have made the arm run. +# and c2, died identically at init; every TP4 cell passed). This is a startup +# capacity limit at 0.90, not a concurrency cliff: the estimated max model +# length at TP2 is 883,840, below the corpus context. The new 0.95 limit may +# clear that check, but needs a live full-context run. # -# Concurrency on the surviving arms still differs from B300, whose TP4 pool is -# ~787 GB against B200's ~398 GB: +# Concurrency on the surviving arms still differs from B300; at the prior +# 0.90 HBM limit, its TP4 KV pool was ~787 GB against B200's ~398 GB: # TP4 B300 runs 1-20; B200 samples 5-15 densely because the cliff lands # inside that range at roughly half the KV, and retains 20 past it. # TP4 + SimpleCPU picks up at 20 rather than B300's 30, since the host tier # has to start absorbing the working set one step earlier, and stops at -# 40 rather than following B300 to 75. Past 40 the host tier is +# 40 rather than following B300's new sweep to 64. Past 40 the host tier is # absorbing a working set that never fit in B200's ~398 GB TP4 pool to # begin with, so those points cost four engine starts each to trace a # tail that is offload bandwidth, not the SKU. # -# dram-utilization 0.683 is the B300 value verbatim, and it resolves to the same -# 1,024 GB engine-level CPU KV budget at TP4: b200-nscale and b300-nv both report -# more installed DRAM than MAX_AGENTIC_AVAILABLE_CPU_DRAM_MIB, so both clamp to -# the same 3 TB AgentX ceiling before the proportional-GPU rule is applied. -# GPU-resident points receive a zero budget. +# B200's 1.0 offload arm uses the full declared host budget: 2,063,920 MiB +# times TP4/8, or about 1,082 GB for this server. GPU-resident points receive +# no host KV budget. B300 has a higher, 3 TB capped node budget (1,499 GB for +# TP4, 749 GB for TP2 after whole-GB rounding). minimaxm3-fp4-b200-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-1dc464d42681d22f38caf1fdc1eb632dc4421c45 + image: vllm/vllm-openai:nightly-29468dde8b515031dc6d4d9d06bf0a2fa0442098 model: nvidia/MiniMax-M3-NVFP4 model-prefix: minimaxm3 runner: cluster:b200-nscale diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0f361fd6d..a9e710931 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8962,3 +8962,57 @@ - "旧镜像切出于 2026-09-16，新镜像领先 441 个提交，包含当前曲线所不含的三项 Kimi-K3 正确性修复：vllm-project/vllm#51483 不再将无状态的首个 chunk 当作 decode，vllm-project/vllm#57098 修复 Kimi-K3 reasoning parser，vllm-project/vllm#57430 支持 routed expert 量化。新镜像还包含本臂在并发大于 4 时均会使用的 ROCm CPU KV 卸载改动，包括 vllm-project/vllm#57160（ROCm CPU KV 卸载改用私有 pinned 张量）与 vllm-project/vllm#50045（卸载背压检测）。" - "This image bump does not change the DSpark draft model data type. Only the image: line changes and kimik3_fp4_mi355x_mtp.sh is unchanged; the draft loads unmodified from the published Inferact/Kimi-K3-DSpark checkpoint via --speculative-config (model=Inferact/Kimi-K3-DSpark, method=dspark). The only dtype in that speculative-config is kv_cache_dtype=fp8, which sets the draft KV-cache storage precision, not the draft weights. No flag overrides or re-quantizes the draft-model weights, so the draft dtype is preserved from its checkpoint across this re-sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3419 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Pin MiniMax-M3 B200/B300 AgentX to vLLM nightly-29468dde8b515031dc6d4d9d06bf0a2fa0442098 and remove the obsolete SimpleCPU KV patch." + - "Increase HBM and B300 host KV budgets, tune serving limits, expand the B300 sweep, and correct host-capacity units." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3435 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use BF16 EAGLE3 draft KV cache to avoid the FA4 FP8 descale startup failure." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3435 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use FlashInfer with FP8 EAGLE3 draft KV cache on B200 and B300." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3435 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use Triton attention with FP8 EAGLE3 draft KV cache." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3435 + +- config-keys: + - minimaxm3-fp4-b200-vllm-agentic-mtp + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Port the settings to active srt-slurm recipes and use FlashInfer draft attention with inherited FP8 KV." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3435 + +- config-keys: + - minimaxm3-fp4-b300-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Set B300 TP2 offload to 0.92 GPU memory utilization to reserve MSA prefill workspace." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3435