From 0c10a6661d553a3f8247f1a4f3b26a4a4b5db96b Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 26 Sep 2026 01:24:32 -0400 Subject: [PATCH 1/2] [Klaud Cold] Update qwen3.5-fp4-mi355x-sglang-agentic-mtp SGLang ROCm image to v0.5.20-rocm720-mi35x-20260924 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml | 2 +- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml index 749c76977..57a656d7f 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -5,7 +5,7 @@ base: name: qwen3.5-fp4-mi355x-sglang-agentic model: path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 - container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 precision: fp4 resources: gpu_type: mi355x diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ce2e4e0bc..639e21aa9 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -409,7 +409,7 @@ qwen3.5-fp4-mi355x-sglang-mtp: srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml qwen3.5-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915 + image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924 model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1bc132cd3..62d63f545 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8950,3 +8950,9 @@ description: - "Update SGLang image from v0.5.19-cu130 to v0.5.20-cu130 and pass --cuda-graph-max-bs-decode after the alias removal." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3334 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + description: + - "Update SGLang ROCm image from v0.5.19-rocm720-mi35x-20260915 to v0.5.20-rocm720-mi35x-20260924 (latest nightly)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3455 From afc4a0bc076a6d61b974bb3a9657408edb546f43 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 26 Sep 2026 01:27:58 -0400 Subject: [PATCH 2/2] Port #3364 TP2 conc grid and hicache-size 253 onto the srt-slurm recipe TP2 GPU-resident KV now covers [1, 4, 8, 12] and TP2 HiCache [16, 20, 24, 28, 32, 36, 40]; every HiCache variant pins hicache-size 253. Drops the now-unused tp2_c16/tp2_c20 resident variants and adds tp2_c16_hicache so each matrix point resolves to exactly one recipe variant. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../sglang/mi355x-fp4-mtp/agentic.yaml | 32 +++++++++---------- configs/amd-master.yaml | 4 +-- perf-changelog.yaml | 2 ++ 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml index 57a656d7f..8ce6b46cd 100644 --- a/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml +++ b/benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml @@ -65,7 +65,8 @@ base: AIPERF_APPLY_CHAT_TEMPLATE: 'true' # One variant per point. Admission is 2x CONC and the decode graph batch follows -# it up to 128; TP4 adds tokenizer workers. HiCache holds 1.5x the device KV pool. +# it up to 128; TP4 adds tokenizer workers. HiCache pins a 253 GB host pool +# (hicache-size overrides hicache-ratio). override_tp4_c1: roles: agg: @@ -188,7 +189,7 @@ override_tp2_c12: CONC: '12' KV_OFFLOADING: none -override_tp2_c16: +override_tp2_c16_hicache: roles: agg: gpus: 2 @@ -196,23 +197,16 @@ override_tp2_c16: tensor-parallel-size: 2 max-running-requests: 32 cuda-graph-max-bs-decode: 32 + enable-hierarchical-cache: true + hicache-ratio: 1.5 + hicache-size: 253 + hicache-write-policy: write_through + hicache-io-backend: kernel + hicache-mem-layout: page_first benchmark: env: CONC: '16' - KV_OFFLOADING: none - -override_tp2_c20: - roles: - agg: - gpus: 2 - args: - tensor-parallel-size: 2 - max-running-requests: 40 - cuda-graph-max-bs-decode: 40 - benchmark: - env: - CONC: '20' - KV_OFFLOADING: none + KV_OFFLOADING: dram override_tp2_c20_hicache: roles: @@ -224,6 +218,7 @@ override_tp2_c20_hicache: cuda-graph-max-bs-decode: 40 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -242,6 +237,7 @@ override_tp2_c24_hicache: cuda-graph-max-bs-decode: 48 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -260,6 +256,7 @@ override_tp2_c28_hicache: cuda-graph-max-bs-decode: 56 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -278,6 +275,7 @@ override_tp2_c32_hicache: cuda-graph-max-bs-decode: 64 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -296,6 +294,7 @@ override_tp2_c36_hicache: cuda-graph-max-bs-decode: 72 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first @@ -314,6 +313,7 @@ override_tp2_c40_hicache: cuda-graph-max-bs-decode: 80 enable-hierarchical-cache: true hicache-ratio: 1.5 + hicache-size: 253 hicache-write-policy: write_through hicache-io-backend: kernel hicache-mem-layout: page_first diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 639e21aa9..d6fb5c820 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -421,8 +421,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - dram-utilization: 0.80 search-space: - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 62d63f545..db2d37325 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8955,4 +8955,6 @@ - qwen3.5-fp4-mi355x-sglang-agentic-mtp description: - "Update SGLang ROCm image from v0.5.19-rocm720-mi35x-20260915 to v0.5.20-rocm720-mi35x-20260924 (latest nightly)" + - "Port the #3364 tuning onto the srt-slurm recipe: move TP2 concurrency 16 from GPU-resident KV to HiCache and drop the TP2 GPU-resident concurrency-20 point, giving TP2 grids resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]; TP4 stays [1, 4, 8, 12, 16]." + - "Pin the HiCache host pool with hicache-size 253 (overrides hicache-ratio)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3455