refactor(amd): share one fixed-sequence client across multi-node recipes

benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml

@@ -157,50 +157,15 @@ base:
     interval_seconds: 5
   benchmark:
     type: custom
-    command: |
-      set -eo pipefail
-      source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only
-      check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP USE_CHAT_TEMPLATE BENCHMARK_VARIANT
-      result_dir=/logs/sa-bench_isl_8192_osl_1024
-      mkdir -p "${result_dir}"
-      ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP))
-      gen=$((DECODE_NUM_WORKERS * DECODE_TP))
-      chat_args=()
-      if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then
-        chat_args+=(--use-chat-template)
-      fi
-      for concurrency in ${CONC_LIST}; do
-        num_prompts=$((concurrency * 10))
-        if (( num_prompts < 16 )); then num_prompts=16; fi
-        python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
-          --backend openai \
-          --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
-          --endpoint /v1/completions \
-          --model amd/DeepSeek-R1-0528-MXFP4-v2 \
-          --tokenizer /model \
-          --trust-remote-code \
-          --dataset-name random \
-          --random-input-len 8192 \
-          --random-output-len 1024 \
-          --random-range-ratio 1.0 \
-          --random-num-workers 1 \
-          "${chat_args[@]}" \
-          --num-warmups "$((concurrency * 2))" \
-          --num-prompts "${num_prompts}" \
-          --max-concurrency "${concurrency}" \
-          --request-rate inf \
-          --ignore-eos \
-          --disable-tqdm \
-          --save-result \
-          --result-dir "${result_dir}" \
-          --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json"
-      done
+    command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh
     env:
+      TOKENIZER: /model
+      ISL: '8192'
+      OSL: '1024'
       HF_HOME: /hf_hub_cache
       HF_HUB_CACHE: /hf_hub_cache/hub
       HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub
       BENCHMARK_VARIANT: stp-1p1d-tp8
-      USE_CHAT_TEMPLATE: '0'
 override_stp_1p1d_tp8:
   name: mi355x-dsr1-fp4-stp-1p1d-tp8-fixed-seq
   benchmark:
@@ -395,7 +360,6 @@ override_mtp3_1p1d_tp8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp3-1p1d-tp8
-      USE_CHAT_TEMPLATE: '1'
 override_mtp3_1p2d_tp8_wide:
   name: mi355x-dsr1-fp4-mtp3-1p2d-tp8-wide-fixed-seq
   resources: {}
@@ -425,7 +389,6 @@ override_mtp3_1p2d_tp8_wide:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp3-1p2d-tp8-wide
-      USE_CHAT_TEMPLATE: '1'
 override_mtp3_1p2d_tp8_narrow:
   name: mi355x-dsr1-fp4-mtp3-1p2d-tp8-narrow-fixed-seq
   resources: {}
@@ -455,7 +418,6 @@ override_mtp3_1p2d_tp8_narrow:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp3-1p2d-tp8-narrow
-      USE_CHAT_TEMPLATE: '1'
 override_mtp2_1p2d_tp8:
   name: mi355x-dsr1-fp4-mtp2-1p2d-tp8-fixed-seq
   resources: {}
@@ -485,7 +447,6 @@ override_mtp2_1p2d_tp8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp2-1p2d-tp8
-      USE_CHAT_TEMPLATE: '1'
 override_mtp3_1p1d_dep8:
   name: mi355x-dsr1-fp4-mtp3-1p1d-dep8-fixed-seq
   roles:
@@ -563,7 +524,6 @@ override_mtp3_1p1d_dep8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp3-1p1d-dep8
-      USE_CHAT_TEMPLATE: '1'
 override_mtp1_1p1d_dep8:
   name: mi355x-dsr1-fp4-mtp1-1p1d-dep8-fixed-seq
   roles:
@@ -641,7 +601,6 @@ override_mtp1_1p1d_dep8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp1-1p1d-dep8
-      USE_CHAT_TEMPLATE: '1'
 override_mtp1_2p1d_dep8:
   name: mi355x-dsr1-fp4-mtp1-2p1d-dep8-fixed-seq
   resources: {}
@@ -724,4 +683,3 @@ override_mtp1_2p1d_dep8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp1-2p1d-dep8
-      USE_CHAT_TEMPLATE: '1'

benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml

@@ -147,50 +147,14 @@ base:
     interval_seconds: 5
   benchmark:
     type: custom
-    command: |
-      set -eo pipefail
-      source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only
-      check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP USE_CHAT_TEMPLATE BENCHMARK_VARIANT
-      result_dir=/logs/sa-bench_isl_8192_osl_1024
-      mkdir -p "${result_dir}"
-      ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP))
-      gen=$((DECODE_NUM_WORKERS * DECODE_TP))
-      chat_args=()
-      if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then
-        chat_args+=(--use-chat-template)
-      fi
-      for concurrency in ${CONC_LIST}; do
-        num_prompts=$((concurrency * 10))
-        if (( num_prompts < 16 )); then num_prompts=16; fi
-        python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
-          --backend openai \
-          --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
-          --endpoint /v1/completions \
-          --model deepseek-ai/DeepSeek-R1-0528 \
-          --tokenizer deepseek-ai/DeepSeek-R1-0528 \
-          --trust-remote-code \
-          --dataset-name random \
-          --random-input-len 8192 \
-          --random-output-len 1024 \
-          --random-range-ratio 1.0 \
-          --random-num-workers 1 \
-          "${chat_args[@]}" \
-          --num-warmups "$((concurrency * 2))" \
-          --num-prompts "${num_prompts}" \
-          --max-concurrency "${concurrency}" \
-          --request-rate inf \
-          --ignore-eos \
-          --disable-tqdm \
-          --save-result \
-          --result-dir "${result_dir}" \
-          --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json"
-      done
+    command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh
     env:
+      ISL: '8192'
+      OSL: '1024'
       HF_HOME: /hf_hub_cache
       HF_HUB_CACHE: /hf_hub_cache/hub
       HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub
       BENCHMARK_VARIANT: base
-      USE_CHAT_TEMPLATE: '0'
 override_stp_2p1d_dep8:
   name: mi355x-dsr1-fp8-stp-2p1d-dep8-fixed-seq
   resources: {}
@@ -327,7 +291,6 @@ override_mtp_2p1d_dep8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp1-2p1d-dep8
-      USE_CHAT_TEMPLATE: '1'
 override_mtp_1p2d_tp8:
   name: mi355x-dsr1-fp8-mtp2-1p2d-tp8-fixed-seq
   resources: {}
@@ -353,7 +316,6 @@ override_mtp_1p2d_tp8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp2-1p2d-tp8
-      USE_CHAT_TEMPLATE: '1'
 override_mtp_1p1d_tp4_tp8:
   name: mi355x-dsr1-fp8-mtp2-1p1d-tp4-tp8-fixed-seq
   resources: {}
@@ -382,4 +344,3 @@ override_mtp_1p1d_tp4_tp8:
   benchmark:
     env:
       BENCHMARK_VARIANT: mtp2-1p1d-tp4-tp8
-      USE_CHAT_TEMPLATE: '1'

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml

@@ -114,41 +114,10 @@ health_check:
 
 benchmark:
   type: custom
-  command: |
-    set -eo pipefail
-    source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only
-    check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP
-    result_dir=/logs/sa-bench_isl_8192_osl_1024
-    mkdir -p "${result_dir}"
-    ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP))
-    gen=$((DECODE_NUM_WORKERS * DECODE_TP))
-    for concurrency in ${CONC_LIST}; do
-      num_prompts=$((concurrency * 10))
-      if ((num_prompts < 16)); then
-        num_prompts=16
-      fi
-      python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
-        --backend openai-chat \
-        --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
-        --endpoint /v1/chat/completions \
-        --model Qwen/Qwen3.5-397B-A17B-FP8 \
-        --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \
-        --dataset-name random \
-        --random-input-len 8192 \
-        --random-output-len 1024 \
-        --random-range-ratio 1.0 \
-        --random-num-workers 1 \
-        --num-warmups "$((concurrency * 2))" \
-        --num-prompts "${num_prompts}" \
-        --max-concurrency "${concurrency}" \
-        --request-rate inf \
-        --ignore-eos \
-        --disable-tqdm \
-        --save-result \
-        --result-dir "${result_dir}" \
-        --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json"
-    done
+  command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh
   env:
+    ISL: '8192'
+    OSL: '1024'
     HF_HOME: /hf_hub_cache
     HF_HUB_CACHE: /hf_hub_cache/hub
     HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub

benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml

@@ -108,41 +108,10 @@ health_check:
 
 benchmark:
   type: custom
-  command: |
-    set -eo pipefail
-    source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only
-    check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP
-    result_dir=/logs/sa-bench_isl_8192_osl_1024
-    mkdir -p "${result_dir}"
-    ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP))
-    gen=$((DECODE_NUM_WORKERS * DECODE_TP))
-    for concurrency in ${CONC_LIST}; do
-      num_prompts=$((concurrency * 10))
-      if ((num_prompts < 16)); then
-        num_prompts=16
-      fi
-      python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \
-        --backend openai-chat \
-        --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
-        --endpoint /v1/chat/completions \
-        --model amd/Qwen3.5-397B-A17B-MXFP4 \
-        --tokenizer amd/Qwen3.5-397B-A17B-MXFP4 \
-        --dataset-name random \
-        --random-input-len 8192 \
-        --random-output-len 1024 \
-        --random-range-ratio 1.0 \
-        --random-num-workers 1 \
-        --num-warmups "$((concurrency * 2))" \
-        --num-prompts "${num_prompts}" \
-        --max-concurrency "${concurrency}" \
-        --request-rate inf \
-        --ignore-eos \
-        --disable-tqdm \
-        --save-result \
-        --result-dir "${result_dir}" \
-        --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json"
-    done
+  command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh
   env:
+    ISL: '8192'
+    OSL: '1024'
     HF_HOME: /hf_hub_cache
     HF_HUB_CACHE: /hf_hub_cache/hub
     HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub

benchmarks/multi_node/srt_fixed_sequence.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Fixed-sequence client for multi-node srt-slurm recipes: SRT owns the servers;
+# this runs the InferenceX client once per concurrency and writes the result
+# layout that copy_fixed_sequence_results collects.
+set -eo pipefail
+source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" --validation-only
+check_env_vars ISL OSL SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST \
+    PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP
+CLIENT_ARGS=(--trust-remote-code)
+case "${CLIENT_BACKEND:=openai}" in
+    openai) endpoint=/v1/completions ;;
+    openai-chat) endpoint=/v1/chat/completions ;;
+    *) echo "ERROR: unsupported CLIENT_BACKEND: $CLIENT_BACKEND" >&2; exit 1 ;;
+esac
+case "${USE_CHAT_TEMPLATE:=true}" in
+    true) CLIENT_ARGS+=(--use-chat-template) ;;
+    false) ;;
+    *) echo "ERROR: USE_CHAT_TEMPLATE must be true or false" >&2; exit 1 ;;
+esac
+
+repo_root="$(dirname "${BASH_SOURCE[0]}")/../.."
+# Request the name the workers registered; the workflow's MODEL is the HF id, which can differ.
+model=$(curl -sf "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}/v1/models" |
+    python3 -c 'import json, sys; print(json.load(sys.stdin)["data"][0]["id"])')
+result_dir="/logs/sa-bench_isl_${ISL}_osl_${OSL}"
+mkdir -p "$result_dir"
+ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP))
+gen=$((DECODE_NUM_WORKERS * DECODE_TP))
+for concurrency in $CONC_LIST; do
+    result="results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json"
+    python3 "$repo_root/utils/bench_serving/benchmark_serving.py" \
+        --backend "$CLIENT_BACKEND" \
+        --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \
+        --endpoint "$endpoint" \
+        --model "$model" \
+        --tokenizer "${TOKENIZER:-$model}" \
+        --dataset-name random \
+        --random-input-len "$ISL" \
+        --random-output-len "$OSL" \
+        --random-range-ratio "${RANDOM_RANGE_RATIO:-0.8}" \
+        --random-num-workers 1 \
+        --num-warmups "$((concurrency * 2))" \
+        --num-prompts "$((concurrency * 10))" \
+        --max-concurrency "$concurrency" \
+        --request-rate inf \
+        --ignore-eos \
+        --disable-tqdm \
+        --save-result \
+        --result-dir "$result_dir" \
+        --result-filename "$result" \
+        "${CLIENT_ARGS[@]}"
+    # Power lanes: tell srt-slurm which interval this concurrency's result measured.
+    if [[ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]]; then
+        PYTHONPATH="$repo_root" python3 -m infx.results.power.window "$result_dir/$result" "$concurrency"
+    fi
+done

infx/results/power/window.py

@@ -0,0 +1,51 @@
+"""Write srt-slurm's power measurement window for one fixed-sequence result.
+
+srt-slurm samples GPU power for the whole job; the window tells it which
+interval belongs to one concurrency point, and which result it measured.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class MeasurementWindow:
+    """One completed ``custom`` benchmark window in srt-slurm's schema version 1."""
+
+    result_path: str
+    concurrency: int
+    benchmark_start_time_unix: float
+    benchmark_end_time_unix: float
+    duration: float
+    schema_version: int = 1
+    benchmark_type: str = "custom"
+    clock_source: str = "head_node_unix_clock"
+    status: str = "completed"
+    reason: str | None = None
+
+
+def write_window(result: Path, concurrency: int, windows: Path) -> None:
+    """Publish the client's measured boundary as a ``custom`` benchmark window."""
+    data = json.loads(result.read_text())
+    window = MeasurementWindow(
+        # srt-slurm resolves this against the log directory, <log dir>/<power dir>/windows/../..
+        result_path=result.relative_to(windows.parent.parent).as_posix(),
+        concurrency=concurrency,
+        benchmark_start_time_unix=data["benchmark_start_time_unix"],
+        benchmark_end_time_unix=data["benchmark_end_time_unix"],
+        duration=data["duration"],
+    )
+    temporary = windows / f".{result.stem}.json.tmp"
+    temporary.write_text(json.dumps(asdict(window), indent=2))
+    temporary.replace(windows / f"{result.stem}.json")
+
+
+if __name__ == "__main__":
+    write_window(
+        Path(sys.argv[1]), int(sys.argv[2]), Path(os.environ["SRT_MEASUREMENT_WINDOW_DIR"])
+    )