From 4835978c34f7cb4331b412065ec636df2371929b Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Sat, 26 Sep 2026 12:30:14 +0000 Subject: [PATCH 1/3] [AMD][Qwen3.5] Bump MI355X AgentX image to v0.5.20-rocm720-mi35x-20260924 Move the TP2 conc-16 point onto HiCache and pin hicache-size 253 on the srt-slurm recipe. --- .../sglang/mi355x-fp4-mtp/agentic.yaml | 34 +++++++++---------- configs/amd-master.yaml | 6 ++-- perf-changelog.yaml | 10 ++++++ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml index 749c76977..8ce6b46cd 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -5,7 +5,7 @@ base: name: qwen3.5-fp4-mi355x-sglang-agentic model: path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 precision: fp4 resources: gpu_type: mi355x @@ -65,7 +65,8 @@ base: AIPERF_APPLY_CHAT_TEMPLATE: 'true' # One variant per point. Admission is 2x CONC and the decode graph batch follows -# it up to 128; TP4 adds tokenizer workers. HiCache holds 1.5x the device KV pool. +# it up to 128; TP4 adds tokenizer workers. HiCache pins a 253 GB host pool +# (hicache-size overrides hicache-ratio). override_tp4_c1: roles: agg: @@ -188,7 +189,7 @@ override_tp2_c12: CONC: '12' KV_OFFLOADING: none -override_tp2_c16: +override_tp2_c16_hicache: roles: agg: gpus: 2 @@ -196,23 +197,16 @@ override_tp2_c16: tensor-parallel-size: 2 max-running-requests: 32 cuda-graph-max-bs-decode: 32 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-size: 253 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first benchmark: env: CONC: '16' - KV_OFFLOADING: none - -override_tp2_c20: - roles: - agg: - gpus: 2 - args: - tensor-parallel-size: 2 - max-running-requests: 40 - cuda-graph-max-bs-decode: 40 - benchmark: - env: - CONC: '20' - KV_OFFLOADING: none + KV_OFFLOADING: dram override_tp2_c20_hicache: roles: @@ -224,6 +218,7 @@ override_tp2_c20_hicache: cuda-graph-max-bs-decode: 40 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -242,6 +237,7 @@ override_tp2_c24_hicache: cuda-graph-max-bs-decode: 48 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -260,6 +256,7 @@ override_tp2_c28_hicache: cuda-graph-max-bs-decode: 56 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -278,6 +275,7 @@ override_tp2_c32_hicache: cuda-graph-max-bs-decode: 64 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -296,6 +294,7 @@ override_tp2_c36_hicache: cuda-graph-max-bs-decode: 72 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -314,6 +313,7 @@ override_tp2_c40_hicache: cuda-graph-max-bs-decode: 80 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e08e74208..aed6bff8f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -409,7 +409,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml qwen3.5-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: cluster:mi355x-amds @@ -421,8 +421,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.80 search-space: - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 0f361fd6d..6327973b1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8962,3 +8962,13 @@ - "旧镜像切出于 2026-09-16，新镜像领先 441 个提交，包含当前曲线所不含的三项 Kimi-K3 正确性修复：vllm-project/vllm#51483 不再将无状态的首个 chunk 当作 decode，vllm-project/vllm#57098 修复 Kimi-K3 reasoning parser，vllm-project/vllm#57430 支持 routed expert 量化。新镜像还包含本臂在并发大于 4 时均会使用的 ROCm CPU KV 卸载改动，包括 vllm-project/vllm#57160（ROCm CPU KV 卸载改用私有 pinned 张量）与 vllm-project/vllm#50045（卸载背压检测）。" - "This image bump does not change the DSpark draft model data type. Only the image: line changes and kimik3_fp4_mi355x_mtp.sh is unchanged; the draft loads unmodified from the published Inferact/Kimi-K3-DSpark checkpoint via --speculative-config (model=Inferact/Kimi-K3-DSpark, method=dspark). The only dtype in that speculative-config is kv_cache_dtype=fp8, which sets the draft KV-cache storage precision, not the draft weights. No flag overrides or re-quantizes the draft-model weights, so the draft dtype is preserved from its checkpoint across this re-sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3419 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 (Docker Hub digest sha256:baadaba198e23c46c1dd651b5edefae8000bd5430943bd10b246b563905b3ecf) in configs/amd-master.yaml and the srt-slurm AgentX recipe." + - "Move TP2 concurrency 16 from GPU-resident KV to HiCache and remove the TP2 GPU-resident concurrency-20 point. Keep TP4 GPU-resident [1, 4, 8, 12, 16]. The resulting TP2 grids are resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]." + - "Pin the HiCache host pool with hicache-size 253 (overrides hicache-ratio) on every TP2 HiCache point." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 From fd5cd97deb063a14c764096f053fde21b78e5443 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Sat, 26 Sep 2026 12:30:58 +0000 Subject: [PATCH 2/3] [AMD][Qwen3.5] Point the AgentX changelog entry at pull 3478 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6327973b1..84e514e6b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8971,4 +8971,4 @@ - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 (Docker Hub digest sha256:baadaba198e23c46c1dd651b5edefae8000bd5430943bd10b246b563905b3ecf) in configs/amd-master.yaml and the srt-slurm AgentX recipe." - "Move TP2 concurrency 16 from GPU-resident KV to HiCache and remove the TP2 GPU-resident concurrency-20 point. Keep TP4 GPU-resident [1, 4, 8, 12, 16]. The resulting TP2 grids are resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]." - "Pin the HiCache host pool with hicache-size 253 (overrides hicache-ratio) on every TP2 HiCache point." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3478 From abda81c96a6b7179422edafdaf3f09e250bfd678 Mon Sep 17 00:00:00 2001 From: "jacky.cheng" Date: Sat, 26 Sep 2026 12:44:05 +0000 Subject: [PATCH 3/3] [AMD][Qwen3.5] Bump MI355X AgentX image to v0.5.20-rocm720-mi35x-20260926 --- .../qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml | 2 +- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml index 8ce6b46cd..8b81fdcdd 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -5,7 +5,7 @@ base: name: qwen3.5-fp4-mi355x-sglang-agentic model: path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260926 precision: fp4 resources: gpu_type: mi355x diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index aed6bff8f..aac062bcd 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -409,7 +409,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml qwen3.5-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 + image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260926 model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 84e514e6b..7bb3439b2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8968,7 +8968,7 @@ scenario-type: - agentic-coding description: - - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 (Docker Hub digest sha256:baadaba198e23c46c1dd651b5edefae8000bd5430943bd10b246b563905b3ecf) in configs/amd-master.yaml and the srt-slurm AgentX recipe." + - "Bump the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 to lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260926 (Docker Hub digest sha256:f71b2a13a23132b9efdd39df58d13f310a59c2d101ec72271457295ed48016a9) in configs/amd-master.yaml and the srt-slurm AgentX recipe." - "Move TP2 concurrency 16 from GPU-resident KV to HiCache and remove the TP2 GPU-resident concurrency-20 point. Keep TP4 GPU-resident [1, 4, 8, 12, 16]. The resulting TP2 grids are resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]." - "Pin the HiCache host pool with hicache-size 253 (overrides hicache-ratio) on every TP2 HiCache point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3478