diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml index 749c76977..8ce6b46cd 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -5,7 +5,7 @@ base: name: qwen3.5-fp4-mi355x-sglang-agentic model: path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 precision: fp4 resources: gpu_type: mi355x @@ -65,7 +65,8 @@ base: AIPERF_APPLY_CHAT_TEMPLATE: 'true' # One variant per point. Admission is 2x CONC and the decode graph batch follows -# it up to 128; TP4 adds tokenizer workers. HiCache holds 1.5x the device KV pool. +# it up to 128; TP4 adds tokenizer workers. HiCache pins a 253 GB host pool +# (hicache-size overrides hicache-ratio). override_tp4_c1: roles: agg: @@ -188,7 +189,7 @@ override_tp2_c12: CONC: '12' KV_OFFLOADING: none -override_tp2_c16: +override_tp2_c16_hicache: roles: agg: gpus: 2 @@ -196,23 +197,16 @@ override_tp2_c16: tensor-parallel-size: 2 max-running-requests: 32 cuda-graph-max-bs-decode: 32 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-size: 253 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first benchmark: env: CONC: '16' - KV_OFFLOADING: none - -override_tp2_c20: - roles: - agg: - gpus: 2 - args: - tensor-parallel-size: 2 - max-running-requests: 40 - cuda-graph-max-bs-decode: 40 - benchmark: - env: - CONC: '20' - KV_OFFLOADING: none + KV_OFFLOADING: dram override_tp2_c20_hicache: roles: @@ -224,6 +218,7 @@ override_tp2_c20_hicache: cuda-graph-max-bs-decode: 40 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -242,6 +237,7 @@ override_tp2_c24_hicache: cuda-graph-max-bs-decode: 48 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -260,6 +256,7 @@ override_tp2_c28_hicache: cuda-graph-max-bs-decode: 56 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -278,6 +275,7 @@ override_tp2_c32_hicache: cuda-graph-max-bs-decode: 64 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -296,6 +294,7 @@ override_tp2_c36_hicache: cuda-graph-max-bs-decode: 72 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -314,6 +313,7 @@ override_tp2_c40_hicache: cuda-graph-max-bs-decode: 80 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 6eeba9f6d..89cc9f10f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -409,7 +409,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml qwen3.5-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: cluster:mi355x-amds @@ -421,8 +421,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.80 search-space: - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0f361fd6d..084688455 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8962,3 +8962,11 @@ - "旧镜像切出于 2026-09-16，新镜像领先 441 个提交，包含当前曲线所不含的三项 Kimi-K3 正确性修复：vllm-project/vllm#51483 不再将无状态的首个 chunk 当作 decode，vllm-project/vllm#57098 修复 Kimi-K3 reasoning parser，vllm-project/vllm#57430 支持 routed expert 量化。新镜像还包含本臂在并发大于 4 时均会使用的 ROCm CPU KV 卸载改动，包括 vllm-project/vllm#57160（ROCm CPU KV 卸载改用私有 pinned 张量）与 vllm-project/vllm#50045（卸载背压检测）。" - "This image bump does not change the DSpark draft model data type. Only the image: line changes and kimik3_fp4_mi355x_mtp.sh is unchanged; the draft loads unmodified from the published Inferact/Kimi-K3-DSpark checkpoint via --speculative-config (model=Inferact/Kimi-K3-DSpark, method=dspark). The only dtype in that speculative-config is kv_cache_dtype=fp8, which sets the draft KV-cache storage precision, not the draft weights. No flag overrides or re-quantizes the draft-model weights, so the draft dtype is preserved from its checkpoint across this re-sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3419 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + description: + - "Update SGLang ROCm image from v0.5.19-rocm720-mi35x-20260915 to v0.5.20-rocm720-mi35x-20260924 (latest nightly)" + - "Port the #3364 tuning onto the srt-slurm recipe: move TP2 concurrency 16 from GPU-resident KV to HiCache and drop the TP2 GPU-resident concurrency-20 point, giving TP2 grids resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]; TP4 stays [1, 4, 8, 12, 16]." + - "Pin the HiCache host pool with hicache-size 253 (overrides hicache-ratio)." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3455