[Klaud Cold] Update qwen3.5-fp4-mi355x-sglang-agentic-mtp to v0.5.20-rocm720-mi35x-20260924 and port #3364 TP2 conc / hicache-size tuning

benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml

@@ -5,7 +5,7 @@ base:
   name: qwen3.5-fp4-mi355x-sglang-agentic
   model:
     path: hf:amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2
-    container: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915
+    container: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924
     precision: fp4
   resources:
     gpu_type: mi355x
@@ -65,7 +65,8 @@ base:
       AIPERF_APPLY_CHAT_TEMPLATE: 'true'
 
 # One variant per point. Admission is 2x CONC and the decode graph batch follows
-# it up to 128; TP4 adds tokenizer workers. HiCache holds 1.5x the device KV pool.
+# it up to 128; TP4 adds tokenizer workers. HiCache pins a 253 GB host pool
+# (hicache-size overrides hicache-ratio).
 override_tp4_c1:
   roles:
     agg:
@@ -188,31 +189,24 @@ override_tp2_c12:
       CONC: '12'
       KV_OFFLOADING: none
 
-override_tp2_c16:
+override_tp2_c16_hicache:
   roles:
     agg:
       gpus: 2
       args:
         tensor-parallel-size: 2
         max-running-requests: 32
         cuda-graph-max-bs-decode: 32
+        enable-hierarchical-cache: true
+        hicache-ratio: 1.5
+        hicache-size: 253
+        hicache-write-policy: write_through
+        hicache-io-backend: kernel
+        hicache-mem-layout: page_first
   benchmark:
     env:
       CONC: '16'
-      KV_OFFLOADING: none
-
-override_tp2_c20:
-  roles:
-    agg:
-      gpus: 2
-      args:
-        tensor-parallel-size: 2
-        max-running-requests: 40
-        cuda-graph-max-bs-decode: 40
-  benchmark:
-    env:
-      CONC: '20'
-      KV_OFFLOADING: none
+      KV_OFFLOADING: dram
 
 override_tp2_c20_hicache:
   roles:
@@ -224,6 +218,7 @@ override_tp2_c20_hicache:
         cuda-graph-max-bs-decode: 40
         enable-hierarchical-cache: true
         hicache-ratio: 1.5
+        hicache-size: 253
         hicache-write-policy: write_through
         hicache-io-backend: kernel
         hicache-mem-layout: page_first
@@ -242,6 +237,7 @@ override_tp2_c24_hicache:
         cuda-graph-max-bs-decode: 48
         enable-hierarchical-cache: true
         hicache-ratio: 1.5
+        hicache-size: 253
         hicache-write-policy: write_through
         hicache-io-backend: kernel
         hicache-mem-layout: page_first
@@ -260,6 +256,7 @@ override_tp2_c28_hicache:
         cuda-graph-max-bs-decode: 56
         enable-hierarchical-cache: true
         hicache-ratio: 1.5
+        hicache-size: 253
         hicache-write-policy: write_through
         hicache-io-backend: kernel
         hicache-mem-layout: page_first
@@ -278,6 +275,7 @@ override_tp2_c32_hicache:
         cuda-graph-max-bs-decode: 64
         enable-hierarchical-cache: true
         hicache-ratio: 1.5
+        hicache-size: 253
         hicache-write-policy: write_through
         hicache-io-backend: kernel
         hicache-mem-layout: page_first
@@ -296,6 +294,7 @@ override_tp2_c36_hicache:
         cuda-graph-max-bs-decode: 72
         enable-hierarchical-cache: true
         hicache-ratio: 1.5
+        hicache-size: 253
         hicache-write-policy: write_through
         hicache-io-backend: kernel
         hicache-mem-layout: page_first
@@ -314,6 +313,7 @@ override_tp2_c40_hicache:
         cuda-graph-max-bs-decode: 80
         enable-hierarchical-cache: true
         hicache-ratio: 1.5
+        hicache-size: 253
         hicache-write-policy: write_through
         hicache-io-backend: kernel
         hicache-mem-layout: page_first

configs/amd-master.yaml

@@ -409,7 +409,7 @@ qwen3.5-fp4-mi355x-sglang-mtp:
         srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/8k1k.yaml
 
 qwen3.5-fp4-mi355x-sglang-agentic-mtp:
-  image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260915
+  image: lmsysorg/sglang-rocm:v0.5.20-rocm720-mi35x-20260924
   model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2
   model-prefix: qwen3.5
   runner: cluster:mi355x-amds
@@ -421,8 +421,8 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp:
     - dram-utilization: 0.80
       search-space:
       - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml }
-      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml }
+      - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 20, 24, 28, 32, 36, 40], srt-recipe: benchmarks/single_node/srt-slurm-recipes/qwen3.5/sglang/mi355x-fp4-mtp/agentic.yaml }
 
 qwen3.5-fp4-mi355x-sglang-disagg:
   image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809

perf-changelog.yaml

@@ -8962,3 +8962,11 @@
     - "旧镜像切出于 2026-09-16，新镜像领先 441 个提交，包含当前曲线所不含的三项 Kimi-K3 正确性修复：vllm-project/vllm#51483 不再将无状态的首个 chunk 当作 decode，vllm-project/vllm#57098 修复 Kimi-K3 reasoning parser，vllm-project/vllm#57430 支持 routed expert 量化。新镜像还包含本臂在并发大于 4 时均会使用的 ROCm CPU KV 卸载改动，包括 vllm-project/vllm#57160（ROCm CPU KV 卸载改用私有 pinned 张量）与 vllm-project/vllm#50045（卸载背压检测）。"
     - "This image bump does not change the DSpark draft model data type. Only the image: line changes and kimik3_fp4_mi355x_mtp.sh is unchanged; the draft loads unmodified from the published Inferact/Kimi-K3-DSpark checkpoint via --speculative-config (model=Inferact/Kimi-K3-DSpark, method=dspark). The only dtype in that speculative-config is kv_cache_dtype=fp8, which sets the draft KV-cache storage precision, not the draft weights. No flag overrides or re-quantizes the draft-model weights, so the draft dtype is preserved from its checkpoint across this re-sweep."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3419
+
+- config-keys:
+    - qwen3.5-fp4-mi355x-sglang-agentic-mtp
+  description:
+    - "Update SGLang ROCm image from v0.5.19-rocm720-mi35x-20260915 to v0.5.20-rocm720-mi35x-20260924 (latest nightly)"
+    - "Port the #3364 tuning onto the srt-slurm recipe: move TP2 concurrency 16 from GPU-resident KV to HiCache and drop the TP2 GPU-resident concurrency-20 point, giving TP2 grids resident [1, 4, 8, 12] and HiCache [16, 20, 24, 28, 32, 36, 40]; TP4 stays [1, 4, 8, 12, 16]."
+    - "Pin the HiCache host pool with hicache-size 253 (overrides hicache-ratio)."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3455