From ca8c4ae8a08b00e17bf2b3bf4256691219c6704f Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Thu, 24 Sep 2026 21:31:28 -0700 Subject: [PATCH 1/4] perf: update B300 DSV4 performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新 B300 DSV4 性能。 --- .../agentic/dsv4_fp4_b300_sglang_mtp.sh | 52 +++++++++++++++---- configs/nvidia-master.yaml | 14 +++++ perf-changelog.yaml | 9 ++++ 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh index 7adf2c573b..966274c8b0 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -2,7 +2,7 @@ set -eo pipefail set -x -# DeepSeek-V4-Pro-0813 FP4 on B300 with SGLang DSpark K=6. +# DeepSeek-V4-Pro FP4 with EAGLE, or Pro-0813 with DSpark K=6, on B300. # KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -68,6 +68,9 @@ if require_agentic_kv_offload_backend hicache; then else HICACHE_RATIO=8 fi + if [ "$MODEL" = "deepseek-ai/DeepSeek-V4-Pro" ]; then + HICACHE_RATIO=2 + fi HICACHE_WRITE_POLICY="write_back" HICACHE_IO_BACKEND="direct" HICACHE_MEM_LAYOUT="page_first_direct" @@ -81,12 +84,13 @@ if require_agentic_kv_offload_backend hicache; then # AIPerf owns the AgentX warmup; SGLang's per-DP warmup can time out after # the API is already healthy. WARMUP_ARGS=(--skip-server-warmup) - echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" + echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, cpu_budget=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" fi USE_SGLANG_ROUTER=false SGLANG_BACKEND_PORT="$PORT" ROUTER_LOG="$RESULT_DIR/router.log" +ROUTER_ARGS=() if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true @@ -100,13 +104,16 @@ METRICS_ARGS=(--enable-metrics --enable-cache-report) MEM_FRACTION_STATIC=0.88 CHUNKED_PREFILL_SIZE=8192 if [ "$DP_ATTENTION" = "true" ]; then + PREFILL_DECODE_INTERVAL=20 + if [ "$MODEL" = "deepseek-ai/DeepSeek-V4-Pro" ]; then + PREFILL_DECODE_INTERVAL=32 + fi PARALLEL_ARGS+=( --dp "$TP" --tokenizer-worker-num "$TP" --enable-prefill-delayer - --prefill-decode-interval 20 + --prefill-decode-interval "$PREFILL_DECODE_INTERVAL" --enable-dp-attention - --enable-dp-lm-head --enable-dp-attention-local-control-broadcast --incremental-streaming-output --stream-interval 20 @@ -140,6 +147,14 @@ if [ "$DP_ATTENTION" = "true" ]; then # Scale it so every DEP shape gets 8192 per rank; 16384/rank exceeds # MegaMoE's per-rank token cap (startup ValueError). CHUNKED_PREFILL_SIZE=$((8192 * TP)) + if [ "$MODEL" = "deepseek-ai/DeepSeek-V4-Pro" ]; then + MEM_FRACTION_STATIC=0.84 + PARALLEL_ARGS+=(--enable-mixed-chunk --schedule-policy shortest-prefill-first) + export SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION=1 + ROUTER_ARGS+=(--disable-circuit-breaker) + else + PARALLEL_ARGS+=(--enable-dp-lm-head) + fi else PARALLEL_ARGS+=( --moe-runner-backend flashinfer_mxfp4 @@ -152,6 +167,9 @@ MODEL_ARGS=( --page-size 256 --disable-shared-experts-fusion ) +if [ "$MODEL" = "deepseek-ai/DeepSeek-V4-Pro" ]; then + MODEL_ARGS=(--attention-backend dsv4 --page-size 256 --disable-shared-experts-fusion) +fi # AgentX concurrency counts live session trees, not individual requests. # Allow subagent fan-out to exceed CONC without clipping request bursts. @@ -192,8 +210,25 @@ if [ "$DP_ATTENTION" = "true" ]; then # extra 128 is headroom over the exact-fit boundary. export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 fi +SPEC_ARGS=( + --speculative-algorithm DSPARK + --speculative-dspark-block-size 6 + --speculative-num-steps 1 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 7 +) +SYNTHETIC_ACC_LEN=3.77 +if [ "$MODEL" = "deepseek-ai/DeepSeek-V4-Pro" ]; then + SPEC_ARGS=( + --speculative-algorithm EAGLE + --speculative-num-steps 3 + --speculative-eagle-topk 1 + --speculative-num-draft-tokens 4 + ) + SYNTHETIC_ACC_LEN=2.49 +fi if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.77 + export SGLANG_SIMULATE_ACC_LEN="$SYNTHETIC_ACC_LEN" export SGLANG_SIMULATE_ACC_METHOD=match-expected export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token fi @@ -224,11 +259,7 @@ SGLANG_CMD=( --reasoning-parser deepseek-v4 --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" --watchdog-timeout 1800 - --speculative-algorithm DSPARK - --speculative-dspark-block-size 6 - --speculative-num-steps 1 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 7 + "${SPEC_ARGS[@]}" "${MODEL_ARGS[@]}" "${METRICS_ARGS[@]}" "${CACHE_ARGS[@]}" @@ -277,6 +308,7 @@ if [ "$USE_SGLANG_ROUTER" = "true" ]; then --connect-timeout-secs 900 \ --request-timeout-secs 14400 \ --disable-health-check \ + "${ROUTER_ARGS[@]}" \ `# A single transient router->engine send failure would otherwise` \ `# surface as a 500, and AgentX aborts the whole run when a root` \ `# warmup request fails ("ProfileAborted"). Measured at conc 512:` \ diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e38ba4bc9e..69865ed9c6 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1145,6 +1145,20 @@ dsv4-fp4-b300-sglang-agentic-hicache-mtp: # Both paths share DSpark K=6 (1 step, 7 draft tokens) and # max-running-requests 2*CONC. +dsv4-fp4-b300-sglang-agentic-hicache-eagle: + image: lmsysorg/sglang:dev-cu13-nightly-09242@sha256:c47bbe7448050608b4e08669c742df4835a45a95bf6b19f2c4e51a7f6609e9d5 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:b300-dsxe + precision: fp4 + framework: sglang + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.95 + search-space: + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: sglang-router, version: "0.3.2" } } + qwen3.5-fp8-b200-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1bc132cd37..1d61229ce1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8950,3 +8950,12 @@ description: - "Update SGLang image from v0.5.19-cu130 to v0.5.20-cu130 and pass --cuda-graph-max-bs-decode after the alias removal." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3334 + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-eagle + scenario-type: + - agentic-coding + description: + - "Update B300 DSV4 performance." + - "更新 B300 DSV4 性能。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From d47e631d967c25a300439778c5690bb44d95ae9f Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Thu, 24 Sep 2026 22:17:13 -0700 Subject: [PATCH 2/4] perf: include B300 DSV4 low-latency points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补充 B300 DSV4 TP8 低延迟并发点 1、2、4、8、16、32。 --- configs/nvidia-master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 69865ed9c6..a4ad60f7dd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1157,6 +1157,7 @@ dsv4-fp4-b300-sglang-agentic-hicache-eagle: agentic-coding: - dram-utilization: 0.95 search-space: + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: sglang-router, version: "0.3.2" } } qwen3.5-fp8-b200-sglang: From 86882167bf00f5e7e72e71a8930a340d9dd06df9 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Thu, 24 Sep 2026 22:26:47 -0700 Subject: [PATCH 3/4] fix: use explicit decode graph flag for DSV4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DSV4 低延迟配置使用明确的 decode CUDA graph 参数，修复新镜像的 CLI 歧义错误。 --- benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh index 966274c8b0..12e48370fb 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh @@ -179,9 +179,10 @@ MAX_RUNNING_REQUESTS=$((2 * CONC)) CUDA_GRAPH_MAX_BS=$((CONC * 4)) [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 -# --cuda-graph-max-bs is an alias whose dest is cuda_graph_max_bs_decode, so the -# two forms below are the same knob and must not both be passed. CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") +if [ "$MODEL" = "deepseek-ai/DeepSeek-V4-Pro" ]; then + CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS") +fi SWA_FULL_TOKENS_RATIO=0.1 if [ "$DP_ATTENTION" = "true" ]; then # Decode graphs must cover the padded speculative batch across all DP ranks, which From 2e46f34fe78627d0b0b108301dbacca77f360062 Mon Sep 17 00:00:00 2001 From: Yuwei An Date: Fri, 25 Sep 2026 13:27:20 -0700 Subject: [PATCH 4/4] perf: add B300 DSV4 TP4 low-latency points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加 B300 DSV4 TP4 低延迟测试点，并保留现有 TP8 和 DP8 配置。 --- configs/nvidia-master.yaml | 1 + perf-changelog.yaml | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a4ad60f7dd..8aa31b2cae 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1157,6 +1157,7 @@ dsv4-fp4-b300-sglang-agentic-hicache-eagle: agentic-coding: - dram-utilization: 0.95 search-space: + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: sglang-router, version: "0.3.2" } } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1d61229ce1..ab63defc92 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8959,3 +8959,12 @@ - "Update B300 DSV4 performance." - "更新 B300 DSV4 性能。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + +- config-keys: + - dsv4-fp4-b300-sglang-agentic-hicache-eagle + scenario-type: + - agentic-coding + description: + - "Add B300 DSV4 TP4 low-latency points." + - "添加 B300 DSV4 TP4 低延迟测试点。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3426