diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh index e53578124..8e0c67aef 100755 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh @@ -71,22 +71,16 @@ while (( CAPTURE_SIZE < GRAPH_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) done -# The sparse-attention indexer and its companion per-rank buffers scale with -# --max-num-batched-tokens at roughly 4.4 MiB per token, measured on gfx950, so -# a smaller prefill chunk buys KV room. TP=2 starts from half the per-rank space -# and is the arm that runs short: at the upstream 16384 it holds 7.84M KV -# tokens, 122K per request at c64, where run 35574132719 fell to a 17.6% prefix -# cache hit rate, 187 s TTFT and 150 tok/s against 94.8%, 1.3 s and 957 tok/s at -# c32. Every point that held had 232K per request or more, so keep the upstream -# chunk through c32 (245K at TP=2) and trade it away only above that. B300 runs -# 8192 at TP=4 and the Blackwell TP=2 arms run 4096 (#3320, #3321). -if (( CONC >= 128 )); then - (( TP == 2 )) && BATCHED_TOKENS=4096 || BATCHED_TOKENS=8192 -elif (( TP == 2 && CONC >= 64 )); then - BATCHED_TOKENS=8192 -else - BATCHED_TOKENS=16384 -fi +# ATOM's TP2/TP4 AgentX recipe for this model (ROCm/ATOM +# dsv41flash_fp4_mi355x_atom_mtp.sh) runs the upstream 16384 chunk unconditionally +# at every concurrency, including TP=2 c64/c128 where this recipe previously +# traded chunk size for KV room (see the superseded comment this replaces: the +# sparse-attention indexer's per-rank buffers scale with --max-num-batched-tokens +# at ~4.4 MiB/token on gfx950, and run 35574132719 saw TP=2 c64 fall to 17.6% +# prefix cache hit rate at the smaller 8192 chunk before this change). Matching +# ATOM's fixed value here is unvalidated at c64/c128 on this SKU -- confirm KV +# headroom does not regress at those points before taking this out of draft. +BATCHED_TOKENS=16384 # DSpark verifies 1+5 tokens per sequence, so a decode batch of max_num_seqs # needs six times that many token slots. The MI355X API-server default of 1024 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 543c40458..102950f18 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1409,8 +1409,9 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: # Two GPUs per server doubles the servers per node and is the layout that # decides whether DSv4.1-Flash is throughput- or interactivity-bound here. dsv41flash-fp4-mi355x-vllm-agentic-dspark: - # ROCm 10.0 nightly channel, shared with kimik3-fp4-mi355x-vllm-agentic-mtp. - image: vllm/vllm-openai-rocm:nightly-rocm100-29468dde8b515031dc6d4d9d06bf0a2fa0442098 + # TODO: repin once the fixed 16384 prefill chunk above (matching ATOM) has + # been validated for KV headroom at TP=2 c64/c128 and TP=4 c128 on this SKU. + image: TBD model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1bc132cd3..7840d4084 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8950,3 +8950,10 @@ description: - "Update SGLang image from v0.5.19-cu130 to v0.5.20-cu130 and pass --cuda-graph-max-bs-decode after the alias removal." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3334 + +- config-keys: + - dsv41flash-fp4-mi355x-vllm-agentic-dspark + description: + - "Match ATOM's fixed 16384 --max-num-batched-tokens (prefill chunk) at every concurrency instead of reducing to 4096/8192 at TP=2 c64+/c128+ and TP=4 c128; unpin image to TBD pending KV-headroom validation at those points on this SKU." + - "将 --max-num-batched-tokens(预填充块大小)在所有并发下固定为与 ATOM 一致的 16384,不再在 TP=2 c64+/c128+ 与 TP=4 c128 处降为 4096/8192;镜像取消固定为 TBD,待在该 SKU 上验证这些并发点的 KV 显存余量后再恢复固定。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3451