feat(srt): run AgentX on native srt-slurm

benchmarks/multi_node/amd_utils/trace_replay.sh

@@ -73,7 +73,7 @@ PORT="${ROUTER_PORT}"
 check_env_vars DURATION RESULT_FILENAME FLUSH_DRAIN_TIMEOUT CLEAR_CACHE_BETWEEN_CONC
 export MODEL DURATION MAX_MODEL_LEN
 # The workflow guard / upload steps expect one "${RESULT_FILENAME}_conc<N>.json" per
-# concurrency, so each conc below is suffixed with _conc<N> (as agentic_srt.sh does).
+# concurrency, so each conc below is suffixed with _conc<N> (as srt_agentic.sh does).
 RESULT_FILENAME_BASE="${RESULT_FILENAME}"
 
 mkdir -p "$RESULT_DIR"
@@ -103,7 +103,7 @@ for max_concurrency in "${chosen_concurrencies[@]}"; do
 
     # benchmark-multinode-tmpl.yml expects the per-conc nesting (LOGS/agentic/conc_*/...)
     # even though CI runs one concurrency per job; nesting also keeps local multi-conc
-    # sweeps from overwriting each other (same layout as agentic_srt.sh).
+    # sweeps from overwriting each other (same layout as srt_agentic.sh).
     CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}"
     mkdir -p "$CONC_RESULT_DIR"
 

benchmarks/multi_node/srt-slurm-recipes/RECIPES.md

@@ -15,14 +15,14 @@ Store every recipe at `<model-prefix>/<engine>/<gpu>-<precision>/<workload>/<rec
 ```text
 dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml
 glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml
-qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml
+qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml
 ```
 
 - Use the master config's `model-prefix` and `precision` labels. Engines are `sglang`, `vllm`, `trtllm`, and `tilert`; frontend selection remains explicit inside the recipe. Hardware directories use GPU types such as `b200` and `gb300`, rather than cluster names.
 - Workloads are `1k1k`, `8k1k`, or `agentx`. Existing bundles spanning several fixed sequence lengths use `fixed-seq-len`; keep their override selectors intact.
 - Use lowercase, hyphen-separated filenames beginning with `agg` or `disagg`. Include topology and the settings that distinguish sibling recipes, such as parallelism, batch size, concurrency, MTP, offload, or cache configuration. Avoid dates, numbered latency/throughput labels, and repeating the model or hardware already in the path.
 - In topology names, `1p4d` denotes prefill/decode worker counts, not necessarily physical nodes. Role-qualified `p-tp4` and `d-tp8` identify prefill/decode TP; `b` denotes batch size and `c` concurrency. The YAML is authoritative for runtime settings.
-- Name override bundles `*-variants.yaml`. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction.
+- Name override bundles `*-variants.yaml`. Multi-node AgentX recipes that differ only per configuration share one bundle per master-config entry, usually `agg-variants.yaml` or `disagg-variants.yaml`: `base` holds the shared settings and each former recipe becomes a named `override_<name>` block holding only its differences (plain overrides, not `zip_override_*`). Master entries select one with `CONFIG_FILE=recipes/<dir>/<bundle>.yaml:override_<name>`. Recipes read as text by a launcher, such as power recipes with top-level `telemetry:`, stay standalone. Keep distinct sweep entry files separate even when their contents match: recipe paths participate in eval grouping. The Qwen3.5 `*-stp-sweep.yaml` and `*-mtp-sweep.yaml` pair preserves that existing distinction.
 - Update `CONFIG_FILE` and `EVAL_CONFIG_FILE` references in active and deprecated master configs, launcher path rules, workflow filters, and local documentation together when moving a file. Preserve upstream source URLs as provenance and leave historical performance-changelog entries unchanged. No aliases for the old layout are provided.
 
 Shared runtime assets stay under `configs/` beside the model directories; they are not standalone recipes. The four files in `configs/dsv4-moe-load-balancer-configs/` are copied verbatim from NVIDIA/srt-slurm commit `deb1dfd9934398664f92d194169c183e009da83b`, preserving the EPLB initial expert assignments used by 17 DSV4 TRT recipes. `setup_srt_slurm()` stages them into the job checkout's `configs/` directory for the recipes' bind mounts. Keeping a recipe in this tree does not activate it; the master configs determine the benchmark matrix.

benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md

@@ -15,14 +15,14 @@ InferenceX 要求 srt-slurm 2.0 或更新版本，且配置必须声明 `schema:
 ```text
 dsr1/sglang/b200-fp4/8k1k/disagg-stp-mtp-variants.yaml
 glm5.2/sglang/h200-fp8/agentx/disagg-1p1d-pcp8-tp8-dp8-mtp6-hicache.yaml
-qwen3.5/trtllm/gb300-fp4/agentx/disagg-1p7d-dep4-tep8-c7-b1-mtp-kvoffload.yaml
+qwen3.5/trtllm/gb300-fp4/agentx/disagg-variants.yaml
 ```
 
 - 使用主配置中的 `model-prefix` 和 `precision` 标签。引擎目录为 `sglang`、`vllm`、`trtllm` 或 `tilert`；前端仍在配置内显式声明。硬件目录使用 `b200`、`gb300` 等 GPU 型号，不使用集群名称。
 - 工作负载目录为 `1k1k`、`8k1k` 或 `agentx`。已有的跨序列长度配置集合放在 `fixed-seq-len` 下，保留其覆盖项选择器。
 - 文件名使用小写字母和连字符，以 `agg` 或 `disagg` 开头。包含拓扑及用于区分同目录配置的关键参数，例如并行方式、批大小、并发数、MTP、卸载或缓存设置。避免日期、带序号的延迟/吞吐量标签，以及重复目录中已有的模型或硬件信息。
 - 拓扑名中的 `1p4d` 表示预填充/解码 worker 数，不一定等于物理节点数。`p-tp4` 和 `d-tp8` 分别标识预填充和解码 TP；`b` 表示批大小，`c` 表示并发数。运行参数以 YAML 为准。
-- 覆盖项集合使用 `*-variants.yaml` 命名。即使内容相同，也保留独立扫描入口：配置路径参与评估分组。Qwen3.5 的 `*-stp-sweep.yaml` 和 `*-mtp-sweep.yaml` 保留了这一既有区别。
+- 覆盖项集合使用 `*-variants.yaml` 命名。仅在各配置间存在差异的多节点 AgentX 配置，按主配置条目合并为一个集合，通常为 `agg-variants.yaml` 或 `disagg-variants.yaml`：`base` 保存共享设置，每个原配置成为一个具名 `override_<name>` 块，只包含其差异（使用普通覆盖项，而非 `zip_override_*`）。主配置通过 `CONFIG_FILE=recipes/<dir>/<bundle>.yaml:override_<name>` 选择其一。启动器以文本方式读取的配置（例如带顶层 `telemetry:` 的功耗配置）保持独立文件。即使内容相同，也保留独立扫描入口：配置路径参与评估分组。Qwen3.5 的 `*-stp-sweep.yaml` 和 `*-mtp-sweep.yaml` 保留了这一既有区别。
 - 移动文件时，同步更新当前及已弃用主配置中的 `CONFIG_FILE`、`EVAL_CONFIG_FILE`，以及启动器路径规则、工作流过滤器和本地文档。保留上游来源 URL，并保持历史性能变更日志不变。不为旧目录结构提供别名。
 
 共享运行时资源保留在模型目录旁的 `configs/` 中，不属于独立基准测试配置。`configs/dsv4-moe-load-balancer-configs/` 中的四个文件原样取自 NVIDIA/srt-slurm 提交 `deb1dfd9934398664f92d194169c183e009da83b`，保留了 17 个 DSV4 TRT 配置使用的 EPLB 初始专家分配。`setup_srt_slurm()` 将这些文件复制到作业仓库的 `configs/` 目录，供配置中的绑定挂载使用。将配置文件放入本目录不会启用该配置；实际基准测试矩阵由主配置决定。

benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h100-block32.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Install the measured H100 DeepSeek-V4.1-Flash block-32 tilings into the worker's SGLang.
+set -euo pipefail
+agentic=/infmax-workspace/benchmarks/single_node/agentic
+python3 "$agentic/install_h100_block32_configs.py" "$agentic/kernel_configs/h100_dsv41_block32" /logs

benchmarks/multi_node/srt-slurm-recipes/configs/dsv41flash-h200-block32.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Install the measured H200 DeepSeek-V4.1-Flash block-32 tilings into the worker's SGLang.
+set -euo pipefail
+agentic=/infmax-workspace/benchmarks/single_node/agentic
+python3 "$agentic/install_h200_block32_configs.py" "$agentic/kernel_configs/h200_dsv41_block32" /logs "$DSV41_BLOCK32_TP"

benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-atom-dspark-draft.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Stage the Kimi-K3 DSpark draft before ATOM starts: with an uncached repo id
+# every rank pulls the same 7 GB at once. Shared-cache downloads can hit
+# transient stale handles, hence the retries.
+set -euo pipefail
+unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy
+for attempt in 1 2 3 4 5; do
+    hf download Inferact/Kimi-K3-DSpark && exit 0
+    echo "hf download attempt $attempt failed; retrying in 60s" >&2
+    sleep 60
+done
+echo "hf download of Inferact/Kimi-K3-DSpark failed after 5 attempts" >&2
+exit 1

benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-b300-mooncake.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Pin the worker's Mooncake client and point its store at one active RDMA rail.
+set -euo pipefail
+pip_install=(python3 -m pip install)
+if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then
+    pip_install+=(--break-system-packages)
+fi
+"${pip_install[@]}" --quiet --no-cache-dir --no-deps --force-reinstall \
+    mooncake-transfer-engine-cuda13==0.3.11.post1
+python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
+
+# Rail-isolated nodes: two RNICs cannot reach each other even within a node, so
+# every rank uses one rail. mlx5_0 is down on some nodes, and topology discovery
+# then finds no HCA, so take the first active rail at runtime. DSXE nodes name
+# their rails rdmap*.
+rail=""
+for device in mlx5_0 mlx5_1 mlx5_2 mlx5_3 mlx5_4 mlx5_5 mlx5_8 mlx5_9 \
+              mlx5_10 mlx5_11 mlx5_16 mlx5_17 mlx5_20 mlx5_21 mlx5_22 mlx5_23 \
+              $(ls /sys/class/infiniband 2>/dev/null | grep '^rdmap' | sort -V); do
+    if grep -q ACTIVE "/sys/class/infiniband/$device/ports/1/state" 2>/dev/null; then
+        rail="$device"
+        break
+    fi
+done
+if [[ -z "$rail" ]]; then
+    echo "Error: no active RDMA rail on $(hostname); Mooncake cannot initialise" >&2
+    for state in /sys/class/infiniband/*/ports/*/state; do
+        echo "$state: $(cat "$state" 2>&1)" >&2
+    done
+    exit 1
+fi
+config="${MOONCAKE_CONFIG_PATH:-/logs/mooncake_store_config.json}"
+python3 - "$config" "$rail" <<'PY'
+import json, sys
+path, rail = sys.argv[1:]
+with open(path) as handle:
+    config = json.load(handle)
+config["device_name"] = rail
+with open(path, "w") as handle:
+    json.dump(config, handle, indent=2)
+PY
+echo "Mooncake rail: $rail"

benchmarks/multi_node/srt-slurm-recipes/configs/lmcache-mp-rocm.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Start one LMCache MP server per TP rank in the worker container before vLLM,
+# as the legacy MI300X MiniMax-M3 AgentX script did. A variant opts in with
+# LMCACHE_SHARDS and LMCACHE_L1_SHARD_GB; its kv-transfer-config lists
+# tcp://127.0.0.1:5555 through 5555 + LMCACHE_SHARDS - 1.
+set -euo pipefail
+[[ -n "${LMCACHE_SHARDS:-}" ]] || exit 0
+: "${LMCACHE_L1_SHARD_GB:?}"
+version=0.5.3
+pip_install=(python3 -m pip install)
+if python3 -m pip install --help 2>/dev/null | grep -q -- --break-system-packages; then
+    pip_install+=(--break-system-packages)
+fi
+"${pip_install[@]}" --quiet --no-cache-dir --no-deps \
+    "sortedcontainers==2.4.0" \
+    "opentelemetry-exporter-prometheus==0.61b0" \
+    "cupy-rocm-7-0==14.1.1" \
+    "lmcache==${version}" \
+    --find-links "https://github.com/LMCache/LMCache/releases/expanded_assets/v${version}-rocm"
+python3 -c "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus"
+
+pids=()
+for ((shard = 0; shard < LMCACHE_SHARDS; shard++)); do
+    # Detached so the servers outlive this preamble and serve the vLLM step.
+    setsid lmcache server \
+        --host 127.0.0.1 --port $((5555 + shard)) \
+        --http-host 127.0.0.1 --http-port $((8080 + shard)) \
+        --l1-size-gb "$LMCACHE_L1_SHARD_GB" --l1-init-size-gb 10 \
+        --l1-read-ttl-seconds 7200 --chunk-size 256 --max-workers 2 \
+        --eviction-policy LRU --supported-transfer-mode lmcache_driven \
+        > "/logs/lmcache_server_${shard}.log" 2>&1 < /dev/null &
+    pids+=($!)
+done
+for ((shard = 0; shard < LMCACHE_SHARDS; shard++)); do
+    for ((attempt = 0; ; attempt++)); do
+        python3 -c 'import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=2)' \
+            "http://127.0.0.1:$((8080 + shard))/healthcheck" 2> /dev/null && break
+        if ! kill -0 "${pids[$shard]}" 2>/dev/null || (( attempt >= 600 )); then
+            echo "ERROR: LMCache server $shard did not become ready" >&2
+            tail -n 50 "/logs/lmcache_server_${shard}.log" >&2 || true
+            exit 1
+        fi
+        sleep 1
+    done
+done
+echo "LMCache: ${LMCACHE_SHARDS} servers ready, ${LMCACHE_L1_SHARD_GB} GB L1 each"