diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml index 5189e3f052..48a966a8e5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml @@ -157,50 +157,15 @@ base: interval_seconds: 5 benchmark: type: custom - command: | - set -eo pipefail - source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only - check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP USE_CHAT_TEMPLATE BENCHMARK_VARIANT - result_dir=/logs/sa-bench_isl_8192_osl_1024 - mkdir -p "${result_dir}" - ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) - gen=$((DECODE_NUM_WORKERS * DECODE_TP)) - chat_args=() - if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then - chat_args+=(--use-chat-template) - fi - for concurrency in ${CONC_LIST}; do - num_prompts=$((concurrency * 10)) - if (( num_prompts < 16 )); then num_prompts=16; fi - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/completions \ - --model amd/DeepSeek-R1-0528-MXFP4-v2 \ - --tokenizer /model \ - --trust-remote-code \ - --dataset-name random \ - --random-input-len 8192 \ - --random-output-len 1024 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - "${chat_args[@]}" \ - --num-warmups "$((concurrency * 2))" \ - --num-prompts "${num_prompts}" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_dir}" \ - --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" - done + command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh env: + TOKENIZER: /model + ISL: '8192' + OSL: '1024' HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache/hub HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub BENCHMARK_VARIANT: stp-1p1d-tp8 - USE_CHAT_TEMPLATE: '0' override_stp_1p1d_tp8: name: mi355x-dsr1-fp4-stp-1p1d-tp8-fixed-seq benchmark: @@ -395,7 +360,6 @@ override_mtp3_1p1d_tp8: benchmark: env: BENCHMARK_VARIANT: mtp3-1p1d-tp8 - USE_CHAT_TEMPLATE: '1' override_mtp3_1p2d_tp8_wide: name: mi355x-dsr1-fp4-mtp3-1p2d-tp8-wide-fixed-seq resources: {} @@ -425,7 +389,6 @@ override_mtp3_1p2d_tp8_wide: benchmark: env: BENCHMARK_VARIANT: mtp3-1p2d-tp8-wide - USE_CHAT_TEMPLATE: '1' override_mtp3_1p2d_tp8_narrow: name: mi355x-dsr1-fp4-mtp3-1p2d-tp8-narrow-fixed-seq resources: {} @@ -455,7 +418,6 @@ override_mtp3_1p2d_tp8_narrow: benchmark: env: BENCHMARK_VARIANT: mtp3-1p2d-tp8-narrow - USE_CHAT_TEMPLATE: '1' override_mtp2_1p2d_tp8: name: mi355x-dsr1-fp4-mtp2-1p2d-tp8-fixed-seq resources: {} @@ -485,7 +447,6 @@ override_mtp2_1p2d_tp8: benchmark: env: BENCHMARK_VARIANT: mtp2-1p2d-tp8 - USE_CHAT_TEMPLATE: '1' override_mtp3_1p1d_dep8: name: mi355x-dsr1-fp4-mtp3-1p1d-dep8-fixed-seq roles: @@ -563,7 +524,6 @@ override_mtp3_1p1d_dep8: benchmark: env: BENCHMARK_VARIANT: mtp3-1p1d-dep8 - USE_CHAT_TEMPLATE: '1' override_mtp1_1p1d_dep8: name: mi355x-dsr1-fp4-mtp1-1p1d-dep8-fixed-seq roles: @@ -641,7 +601,6 @@ override_mtp1_1p1d_dep8: benchmark: env: BENCHMARK_VARIANT: mtp1-1p1d-dep8 - USE_CHAT_TEMPLATE: '1' override_mtp1_2p1d_dep8: name: mi355x-dsr1-fp4-mtp1-2p1d-dep8-fixed-seq resources: {} @@ -724,4 +683,3 @@ override_mtp1_2p1d_dep8: benchmark: env: BENCHMARK_VARIANT: mtp1-2p1d-dep8 - USE_CHAT_TEMPLATE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml index cd3d9d16f0..028dc7f619 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml @@ -147,50 +147,14 @@ base: interval_seconds: 5 benchmark: type: custom - command: | - set -eo pipefail - source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only - check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP USE_CHAT_TEMPLATE BENCHMARK_VARIANT - result_dir=/logs/sa-bench_isl_8192_osl_1024 - mkdir -p "${result_dir}" - ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) - gen=$((DECODE_NUM_WORKERS * DECODE_TP)) - chat_args=() - if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then - chat_args+=(--use-chat-template) - fi - for concurrency in ${CONC_LIST}; do - num_prompts=$((concurrency * 10)) - if (( num_prompts < 16 )); then num_prompts=16; fi - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/completions \ - --model deepseek-ai/DeepSeek-R1-0528 \ - --tokenizer deepseek-ai/DeepSeek-R1-0528 \ - --trust-remote-code \ - --dataset-name random \ - --random-input-len 8192 \ - --random-output-len 1024 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - "${chat_args[@]}" \ - --num-warmups "$((concurrency * 2))" \ - --num-prompts "${num_prompts}" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_dir}" \ - --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" - done + command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh env: + ISL: '8192' + OSL: '1024' HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache/hub HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub BENCHMARK_VARIANT: base - USE_CHAT_TEMPLATE: '0' override_stp_2p1d_dep8: name: mi355x-dsr1-fp8-stp-2p1d-dep8-fixed-seq resources: {} @@ -327,7 +291,6 @@ override_mtp_2p1d_dep8: benchmark: env: BENCHMARK_VARIANT: mtp1-2p1d-dep8 - USE_CHAT_TEMPLATE: '1' override_mtp_1p2d_tp8: name: mi355x-dsr1-fp8-mtp2-1p2d-tp8-fixed-seq resources: {} @@ -353,7 +316,6 @@ override_mtp_1p2d_tp8: benchmark: env: BENCHMARK_VARIANT: mtp2-1p2d-tp8 - USE_CHAT_TEMPLATE: '1' override_mtp_1p1d_tp4_tp8: name: mi355x-dsr1-fp8-mtp2-1p1d-tp4-tp8-fixed-seq resources: {} @@ -382,4 +344,3 @@ override_mtp_1p1d_tp4_tp8: benchmark: env: BENCHMARK_VARIANT: mtp2-1p1d-tp4-tp8 - USE_CHAT_TEMPLATE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml index 9f05cdddfa..dffc2c267a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml @@ -114,41 +114,10 @@ health_check: benchmark: type: custom - command: | - set -eo pipefail - source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only - check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP - result_dir=/logs/sa-bench_isl_8192_osl_1024 - mkdir -p "${result_dir}" - ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) - gen=$((DECODE_NUM_WORKERS * DECODE_TP)) - for concurrency in ${CONC_LIST}; do - num_prompts=$((concurrency * 10)) - if ((num_prompts < 16)); then - num_prompts=16 - fi - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai-chat \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/chat/completions \ - --model Qwen/Qwen3.5-397B-A17B-FP8 \ - --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \ - --dataset-name random \ - --random-input-len 8192 \ - --random-output-len 1024 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - --num-warmups "$((concurrency * 2))" \ - --num-prompts "${num_prompts}" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_dir}" \ - --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" - done + command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh env: + ISL: '8192' + OSL: '1024' HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache/hub HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml index 8693f14fc1..2afa1c227a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml @@ -108,41 +108,10 @@ health_check: benchmark: type: custom - command: | - set -eo pipefail - source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only - check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP - result_dir=/logs/sa-bench_isl_8192_osl_1024 - mkdir -p "${result_dir}" - ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) - gen=$((DECODE_NUM_WORKERS * DECODE_TP)) - for concurrency in ${CONC_LIST}; do - num_prompts=$((concurrency * 10)) - if ((num_prompts < 16)); then - num_prompts=16 - fi - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai-chat \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/chat/completions \ - --model amd/Qwen3.5-397B-A17B-MXFP4 \ - --tokenizer amd/Qwen3.5-397B-A17B-MXFP4 \ - --dataset-name random \ - --random-input-len 8192 \ - --random-output-len 1024 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - --num-warmups "$((concurrency * 2))" \ - --num-prompts "${num_prompts}" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_dir}" \ - --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" - done + command: bash /infmax-workspace/benchmarks/multi_node/srt_fixed_sequence.sh env: + ISL: '8192' + OSL: '1024' HF_HOME: /hf_hub_cache HF_HUB_CACHE: /hf_hub_cache/hub HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt_fixed_sequence.sh b/benchmarks/multi_node/srt_fixed_sequence.sh new file mode 100644 index 0000000000..965462b163 --- /dev/null +++ b/benchmarks/multi_node/srt_fixed_sequence.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Fixed-sequence client for multi-node srt-slurm recipes: SRT owns the servers; +# this runs the InferenceX client once per concurrency and writes the result +# layout that copy_fixed_sequence_results collects. +set -eo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" --validation-only +check_env_vars ISL OSL SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST \ + PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP +CLIENT_ARGS=(--trust-remote-code) +case "${CLIENT_BACKEND:=openai}" in + openai) endpoint=/v1/completions ;; + openai-chat) endpoint=/v1/chat/completions ;; + *) echo "ERROR: unsupported CLIENT_BACKEND: $CLIENT_BACKEND" >&2; exit 1 ;; +esac +case "${USE_CHAT_TEMPLATE:=true}" in + true) CLIENT_ARGS+=(--use-chat-template) ;; + false) ;; + *) echo "ERROR: USE_CHAT_TEMPLATE must be true or false" >&2; exit 1 ;; +esac + +repo_root="$(dirname "${BASH_SOURCE[0]}")/../.." +# Request the name the workers registered; the workflow's MODEL is the HF id, which can differ. +model=$(curl -sf "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}/v1/models" | + python3 -c 'import json, sys; print(json.load(sys.stdin)["data"][0]["id"])') +result_dir="/logs/sa-bench_isl_${ISL}_osl_${OSL}" +mkdir -p "$result_dir" +ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) +gen=$((DECODE_NUM_WORKERS * DECODE_TP)) +for concurrency in $CONC_LIST; do + result="results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" + python3 "$repo_root/utils/bench_serving/benchmark_serving.py" \ + --backend "$CLIENT_BACKEND" \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint "$endpoint" \ + --model "$model" \ + --tokenizer "${TOKENIZER:-$model}" \ + --dataset-name random \ + --random-input-len "$ISL" \ + --random-output-len "$OSL" \ + --random-range-ratio "${RANDOM_RANGE_RATIO:-0.8}" \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "$((concurrency * 10))" \ + --max-concurrency "$concurrency" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "$result_dir" \ + --result-filename "$result" \ + "${CLIENT_ARGS[@]}" + # Power lanes: tell srt-slurm which interval this concurrency's result measured. + if [[ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]]; then + PYTHONPATH="$repo_root" python3 -m infx.results.power.window "$result_dir/$result" "$concurrency" + fi +done diff --git a/infx/results/power/window.py b/infx/results/power/window.py new file mode 100644 index 0000000000..956f95528d --- /dev/null +++ b/infx/results/power/window.py @@ -0,0 +1,51 @@ +"""Write srt-slurm's power measurement window for one fixed-sequence result. + +srt-slurm samples GPU power for the whole job; the window tells it which +interval belongs to one concurrency point, and which result it measured. +""" + +from __future__ import annotations + +import json +import os +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class MeasurementWindow: + """One completed ``custom`` benchmark window in srt-slurm's schema version 1.""" + + result_path: str + concurrency: int + benchmark_start_time_unix: float + benchmark_end_time_unix: float + duration: float + schema_version: int = 1 + benchmark_type: str = "custom" + clock_source: str = "head_node_unix_clock" + status: str = "completed" + reason: str | None = None + + +def write_window(result: Path, concurrency: int, windows: Path) -> None: + """Publish the client's measured boundary as a ``custom`` benchmark window.""" + data = json.loads(result.read_text()) + window = MeasurementWindow( + # srt-slurm resolves this against the log directory, //windows/../.. + result_path=result.relative_to(windows.parent.parent).as_posix(), + concurrency=concurrency, + benchmark_start_time_unix=data["benchmark_start_time_unix"], + benchmark_end_time_unix=data["benchmark_end_time_unix"], + duration=data["duration"], + ) + temporary = windows / f".{result.stem}.json.tmp" + temporary.write_text(json.dumps(asdict(window), indent=2)) + temporary.replace(windows / f"{result.stem}.json") + + +if __name__ == "__main__": + write_window( + Path(sys.argv[1]), int(sys.argv[2]), Path(os.environ["SRT_MEASUREMENT_WINDOW_DIR"]) + )