From 552462a0c8e6c5d62072d3b260daa16b0312d064 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 17:02:24 -0500 Subject: [PATCH 1/6] refactor: standardize srt-slurm cluster setup hooks Per-allocation host checks live in runners/srt-slurm/hooks//setup.sh and run through the cluster profile's default_host_setup. Shared host-check functions move to hooks/common.sh. MI355X gets its RDMA/QoS preflight, GPU drain gate, and hugepage reclaim. --- AGENTS.md | 9 ++++ benchmarks/benchmark_lib.sh | 29 +--------- docs/configuration-procedures.md | 26 +++++++++ runners/launch_mi355x-amds.sh | 2 +- runners/srt-slurm/hooks/common.sh | 26 +++++++++ .../srt-slurm/hooks/mi355x-amds/check-rdma.sh | 54 +++++++++++++++++++ runners/srt-slurm/hooks/mi355x-amds/setup.sh | 51 ++++++++++++++++++ runners/srt-slurm/mi355x-amds.yaml | 7 +++ utils/test_srt_fixed_sequence.py | 3 ++ 9 files changed, 179 insertions(+), 28 deletions(-) create mode 100644 runners/srt-slurm/hooks/common.sh create mode 100755 runners/srt-slurm/hooks/mi355x-amds/check-rdma.sh create mode 100755 runners/srt-slurm/hooks/mi355x-amds/setup.sh diff --git a/AGENTS.md b/AGENTS.md index 26e91ccbac..20d2291b81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,6 +66,15 @@ check_env_vars IS_MULTINODE MODEL_NAME PRECISION - Do not add launcher-name aliases to `runners/runtime_settings.sh` or elsewhere for scripts that no runner resolves to. A launcher without a pool is dead code; a pool without a launcher fails at job start. - When a pool is retired, delete its launcher in the same PR rather than keeping it as a fallback for another pool. +## SRT Slurm cluster hooks + +- Put reusable host-check functions in `runners/srt-slurm/hooks/common.sh`. Sourcing it must only define functions, without running checks, changing environment variables, or initializing benchmarks. Cluster-only helpers stay beside their setup script. +- Keep cluster-specific host prerequisites in `runners/srt-slurm/hooks//setup.sh`, invoked explicitly by the matching cluster profile's `default_host_setup`. These run after allocation, before services and workers start. +- Hooks are only for checks and setup required by that cluster's hosts or fabric. Keep them small, workload-independent, and safe to run repeatedly. Prefer native srt-slurm configuration whenever it can express the requirement. +- Do not put benchmark execution, model selection, engine flags, concurrency tuning, evaluation, result collection, or job orchestration in hooks. Those belong in recipes, benchmark scripts, or the existing orchestration layer. +- Do not use hooks to patch engines or containers, bypass failed checks, or hide runtime bugs behind retries and ad hoc workarounds. Fix problems in the component that owns them. +- Pass settings explicitly from the cluster profile. Scope mutations to the allocated nodes, preserve other jobs' resources, and register teardown for temporary state that needs restoring. See [cluster profiles](docs/configuration-procedures.md#cluster-profiles). + ## SRT Slurm synthetic acceptance - **Do not hard-code synthetic acceptance lengths in SRT recipes, master configs, or launchers.** InferenceX automatically selects the measured value from [`golden_al_distribution/`](golden_al_distribution/) for speculative AgentX throughput runs. Do not add manual `SYNTHETIC_ACCEPTANCE_LENGTH`, vLLM `synthetic_acceptance_length`, SGLang `SGLANG_SIMULATE_ACC_LEN`, or TRT-LLM `TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS` settings. diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index ed708ede8e..b29ad3686e 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -558,33 +558,8 @@ _write_amd_smi_sidecar() { fi } -# Poll rocm-smi VRAM% every 10s for up to 15 min until the busiest GPU is at or -# below the threshold percent (default 10); return 1 otherwise so the caller -# aborts instead of starting on GPUs still draining the previous job. -# Pass a stricter threshold when the run sizes its KV cache from device-wide free -# memory (torch.cuda.mem_get_info): on 288 GB parts the 10% gate admits ~28.8 GB -# of residual, which the engine folds into non_torch and subtracts from the KV -# pool, so the pool drifts run to run. -wait_for_amd_gpu_clean() { - local threshold="${1:-10}" - local gpu_clean=false vram_max i - for i in $(seq 1 90); do - vram_max=$(rocm-smi --showmemuse 2>/dev/null \ - | grep -oE "GPU Memory Allocated \(VRAM%\): [0-9]+" \ - | awk '{if ($NF > m) m = $NF} END {print m+0}') - if [ "${vram_max:-0}" -le "$threshold" ]; then - echo "GPUs clean (vram%max=$vram_max <= $threshold after $((i * 10))s)" - gpu_clean=true - break - fi - echo "waiting for prior-job GPU memory reclaim: vram%max=$vram_max (target <= $threshold)" - sleep 10 - done - if [ "$gpu_clean" != "true" ]; then - echo "Error: GPUs still draining prior job's memory after 15min" >&2 - return 1 - fi -} +# shellcheck source=runners/srt-slurm/hooks/common.sh +source "$(dirname "${BASH_SOURCE[0]}")/../runners/srt-slurm/hooks/common.sh" || return 1 # Return success only while a PID exists and is not a zombie waiting to be # reaped. `kill -0` alone treats zombies as live processes. diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 9f083eae9e..24f958976e 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -65,6 +65,32 @@ parsed YAML scalars so quotes and punctuation remain data, not YAML or shell syn Keep model selection, cache preparation, and workload-dependent time limits in the launcher. Do not add profiles for non-srt-slurm launchers or change their routing here. +Put per-allocation host checks and setup in +`runners/srt-slurm/hooks//setup.sh`, with cluster-specific helpers beside it. +The directory name matches the cluster profile's filename stem. Invoke the script +explicitly through `default_host_setup.commands` in that profile; scripts are not +auto-discovered. srt-slurm runs them on the selected allocated nodes, outside containers, +before starting services and workers. A failed check stops startup by default. +Pass configuration explicitly from the profile. If setup needs an undo step, keep it +in `teardown.sh` beside `setup.sh` and register it in `default_host_setup.teardown`. +These are job-owned hooks, not administrator-installed Slurm Prolog/Epilog scripts. +Only add hooks for clusters that need them; do not add empty scripts for every profile. + +Put reusable host-check functions in `runners/srt-slurm/hooks/common.sh`; keep +cluster-only helpers beside `setup.sh`. The common file only defines functions: +sourcing it must not run checks, change environment variables, or initialize +benchmarks. Both setup hooks and benchmark scripts can reuse these functions +without importing benchmark initialization into host setup. + +Hooks inject **cluster-specific host prerequisites only**, such as fabric checks or +required host-state preparation. Keep them small, workload-independent, and safe to +run repeatedly. Prefer native srt-slurm settings over shell code where possible. +Benchmark execution, model selection, engine flags, concurrency tuning, evaluation, +result collection, and job orchestration do not belong here. Do not patch engines or +containers, bypass failed checks, or hide runtime bugs with retries and ad hoc +workarounds; fix the owning component instead. Limit host changes to allocated nodes +and preserve resources used by other jobs. + ## Procedure index 1. [Prepare a worktree](#prepare-a-worktree) diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 8971a58f6b..90682aeb5d 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -19,7 +19,7 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then export SALLOC_TIME_LIMIT=500 export SRT_SRUN_OPTIONS='{"container-remap-root":"", "container-writable":""}' SRT_SQUASH_FILE="/var/lib/squash/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - launch_srt_single_node mi355x-amds + launch_srt_single_node mi355x-amds --var GITHUB_WORKSPACE "$GITHUB_WORKSPACE" exit $? fi diff --git a/runners/srt-slurm/hooks/common.sh b/runners/srt-slurm/hooks/common.sh new file mode 100644 index 0000000000..5dbc215dfd --- /dev/null +++ b/runners/srt-slurm/hooks/common.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# Shared host checks. Sourcing this file only defines functions. + +# Poll VRAM usage every 10s for up to 15 minutes. A stricter threshold is useful +# when the engine sizes its KV cache from device-wide free memory. +wait_for_amd_gpu_clean() { + local threshold="${1:-10}" + local gpu_clean=false vram_max i + for i in $(seq 1 90); do + vram_max=$(rocm-smi --showmemuse 2>/dev/null \ + | grep -oE "GPU Memory Allocated \(VRAM%\): [0-9]+" \ + | awk '{if ($NF > m) m = $NF} END {print m+0}') + if [ "${vram_max:-0}" -le "$threshold" ]; then + echo "GPUs clean (vram%max=$vram_max <= $threshold after $((i * 10))s)" + gpu_clean=true + break + fi + echo "waiting for prior-job GPU memory reclaim: vram%max=$vram_max (target <= $threshold)" + sleep 10 + done + if [ "$gpu_clean" != "true" ]; then + echo "Error: GPUs still draining prior job's memory after 15min" >&2 + return 1 + fi +} diff --git a/runners/srt-slurm/hooks/mi355x-amds/check-rdma.sh b/runners/srt-slurm/hooks/mi355x-amds/check-rdma.sh new file mode 100755 index 0000000000..e1f7c68b6b --- /dev/null +++ b/runners/srt-slurm/hooks/mi355x-amds/check-rdma.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Fast, per-node fabric preflight for MI355X srt-slurm allocations. This keeps +# the meaningful QoS/DCQCN gate from the retired amd_utils launcher without its +# Docker or job-control plumbing. + +log() { printf '[%s] %s\n' "$(hostname -s)" "$*"; } +fail() { log "RDMA preflight failed: $*" >&2; exit 1; } + +source "$(dirname "${BASH_SOURCE[0]}")/../../../../benchmarks/benchmark_lib.sh" --validation-only +check_env_vars IBDEVICES +expected_devices="$IBDEVICES" +IFS=',' read -r -a devices <<< "$expected_devices" +for device in "${devices[@]}"; do + [[ -d "/sys/class/infiniband/${device}" ]] || fail "missing device ${device}" +done +log "found all ${#devices[@]} expected RDMA devices: ${expected_devices}" + +if ! command -v nicctl >/dev/null 2>&1; then + log "nicctl is unavailable; device presence passed, QoS/DCQCN checks skipped" + exit 0 +fi + +probe=$(sudo -n nicctl show version firmware 2>&1 || true) +if grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC' <<< "$probe"; then + fail "nicctl cannot access the AMD NICs" +fi + +qos=$(sudo -n nicctl show qos 2>/dev/null) || fail "nicctl show qos failed" +classification=$(awk '/Classification type/ {print $NF; exit}' <<< "$qos") +[[ "$classification" == "DSCP" ]] || fail "classification is ${classification:-unset}, expected DSCP" + +priorities=$(awk '/PFC no-drop priorities/ {print $NF; exit}' <<< "$qos") +bitmap=$(awk '/PFC priority bitmap/ {print $NF; exit}' <<< "$qos") +[[ -n "$priorities" ]] || fail "PFC no-drop priorities are missing" +[[ -n "$bitmap" && "$bitmap" != "0x0" ]] || fail "PFC is disabled" +IFS=',' read -r -a priority_values <<< "$priorities" +for priority in "${priority_values[@]}"; do + priority="${priority//[^0-9]/}" + [[ -n "$priority" ]] || fail "invalid PFC priority list: ${priorities}" + (( bitmap & (1 << priority) )) || fail "PFC bitmap ${bitmap} does not cover priority ${priority}" +done + +dcqcn=$(sudo -n nicctl show dcqcn 2>/dev/null) || fail "nicctl show dcqcn failed" +device_count=$(grep -c 'ROCE device' <<< "$dcqcn" || true) +(( device_count > 0 )) || fail "no RoCE devices reported by nicctl" +if grep 'Status' <<< "$dcqcn" | grep -qv 'Enabled'; then + fail "DCQCN is disabled on at least one RoCE device" +fi +cnp_count=$(awk '/DSCP value used for CNP/ {print $NF}' <<< "$dcqcn" | sort -u | grep -c . || true) +(( cnp_count == 1 )) || fail "CNP DSCP is inconsistent across NICs" + +log "RDMA QoS/DCQCN preflight passed" diff --git a/runners/srt-slurm/hooks/mi355x-amds/setup.sh b/runners/srt-slurm/hooks/mi355x-amds/setup.sh new file mode 100755 index 0000000000..c5170d9ac5 --- /dev/null +++ b/runners/srt-slurm/hooks/mi355x-amds/setup.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -eo pipefail + +bash "$(dirname "${BASH_SOURCE[0]}")/check-rdma.sh" + +# Preserve the legacy bare-process GPU drain gate. Slurm owns the node, but a +# process left outside the prior job's container can still retain VRAM and make +# the next model load fail much later with a misleading OOM. +# shellcheck source=runners/srt-slurm/hooks/common.sh +source "$(dirname "${BASH_SOURCE[0]}")/../common.sh" +wait_for_amd_gpu_clean + +# Some MI355X experiments reserve large 2 MiB HugeTLB pools and leave the +# reservation behind after their Slurm allocation exits. Those free hugepages +# remain unavailable to ordinary host allocations, which can make a later +# unchanged SGLang HiCache recipe fail even on a 3 TiB node. Reclaim only free +# pages; pages currently used or reserved by host services are preserved. +meminfo=/proc/meminfo +nr_hugepages=/proc/sys/vm/nr_hugepages + +read_hugepage_value() { + local key="$1" + awk -v key="${key}:" '$1 == key {print $2}' "$meminfo" +} + +total=$(read_hugepage_value HugePages_Total) +free=$(read_hugepage_value HugePages_Free) +reserved=$(read_hugepage_value HugePages_Rsvd) +used=$((total - free)) +target=$((used + reserved)) + +echo "MI355X host memory before preparation:" +grep -E '^(MemAvailable|HugePages_Total|HugePages_Free|HugePages_Rsvd|HugePages_Surp|Hugetlb):' "$meminfo" + +if (( target < total )); then + printf '%s\n' "$target" | sudo -n tee "$nr_hugepages" >/dev/null +fi + +after_total=$(read_hugepage_value HugePages_Total) +after_free=$(read_hugepage_value HugePages_Free) +echo "MI355X host memory after preparation:" +grep -E '^(MemAvailable|HugePages_Total|HugePages_Free|HugePages_Rsvd|HugePages_Surp|Hugetlb):' "$meminfo" + +if (( after_total - after_free < used )); then + echo "Host preparation released hugepages that were in use" >&2 + exit 1 +fi +if (( after_free > reserved )); then + echo "Host preparation could not reclaim all unused hugepages" >&2 + exit 1 +fi diff --git a/runners/srt-slurm/mi355x-amds.yaml b/runners/srt-slurm/mi355x-amds.yaml index f6a0df40e8..f29f1fabdc 100644 --- a/runners/srt-slurm/mi355x-amds.yaml +++ b/runners/srt-slurm/mi355x-amds.yaml @@ -12,3 +12,10 @@ default_sbatch_directives: use_gpus_per_node_directive: true use_segment_sbatch_directive: false use_exclusive_sbatch_directive: true +default_host_setup: + # The GPU-drain check can take 15 minutes; leave room for the fabric check + # and host-memory preparation. + timeout_seconds: 1200 + commands: + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/srt-slurm/hooks/mi355x-amds/setup.sh" + nodes: all diff --git a/utils/test_srt_fixed_sequence.py b/utils/test_srt_fixed_sequence.py index 84575b1db2..14ec741421 100644 --- a/utils/test_srt_fixed_sequence.py +++ b/utils/test_srt_fixed_sequence.py @@ -172,6 +172,9 @@ def test_native_post_eval_preserves_results_topology_and_failure(client_environm scripts = workspace / "benchmarks/single_node" scripts.mkdir(parents=True) shutil.copyfile(ROOT / "benchmarks/benchmark_lib.sh", scripts.parent / "benchmark_lib.sh") + hooks = workspace / "runners/srt-slurm/hooks" + hooks.mkdir(parents=True) + shutil.copyfile(ROOT / "runners/srt-slurm/hooks/common.sh", hooks / "common.sh") shutil.copyfile(ROOT / "benchmarks/single_node/srt_eval.sh", scripts / "srt_eval.sh") python = tmp_path / "bin/python3" python.write_text( From 0b572740cd08fb30d2225e5472dce74f13232adb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 17:04:32 -0500 Subject: [PATCH 2/6] feat(amd): add disaggregated SGLang and vLLM to synthetic acceptance --- infx/srt_slurm/synthetic_acceptance.py | 2 ++ utils/test_synthetic_acceptance.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/infx/srt_slurm/synthetic_acceptance.py b/infx/srt_slurm/synthetic_acceptance.py index 0a9aa05403..be7232eb12 100644 --- a/infx/srt_slurm/synthetic_acceptance.py +++ b/infx/srt_slurm/synthetic_acceptance.py @@ -20,6 +20,8 @@ GOLDEN_DIR = Path(__file__).resolve().parents[2] / "golden_al_distribution" ENGINES = { "sglang": "sglang", + "sglang-disagg": "sglang", + "vllm-disagg": "vllm", "vllm": "vllm", "dynamo-vllm": "vllm", "dynamo-sglang": "sglang", diff --git a/utils/test_synthetic_acceptance.py b/utils/test_synthetic_acceptance.py index 0c26b9fbc6..ffa8b01aa0 100644 --- a/utils/test_synthetic_acceptance.py +++ b/utils/test_synthetic_acceptance.py @@ -163,7 +163,7 @@ def test_kimi_curve_requires_an_explicit_supported_sampler( ("framework", "args", "environment", "expected"), [ ( - "dynamo-sglang", + "sglang-disagg", { "speculative-algorithm": "DSpark", "speculative-dspark-block-size": 3, @@ -229,7 +229,7 @@ def test_engine_token_selection_and_environment( "environment", [{"EVAL_ONLY": "true"}, {"IS_AGENTIC": "0"}, {"SPEC_DECODING": "none"}], ) -@pytest.mark.parametrize("framework", ["vllm", "dynamo-sglang", "trt"]) +@pytest.mark.parametrize("framework", ["vllm", "vllm-disagg", "sglang", "dynamo-sglang", "trt"]) def test_real_runs_clear_synthetic_without_a_curve( tmp_path: Path, framework: str, environment: dict[str, str] ) -> None: @@ -251,7 +251,7 @@ def test_real_runs_clear_synthetic_without_a_curve( build_overrides(recipe, framework, env, golden_dir=tmp_path / "absent"), ) role = result["roles"]["agg"] - if framework == "vllm": + if framework in {"vllm", "vllm-disagg"}: assert json.loads(role["args"]["speculative-config"]) == { "method": "dspark", "num_speculative_tokens": 3, From 087aae5467eb3fe36d3d870ff9d70624db415bc4 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 16:50:42 -0500 Subject: [PATCH 3/6] feat(amd): run MI355X multi-node recipes through srt-slurm Multi-node jobs with a CONFIG_FILE use the shared native path: srt-slurm owns allocation, serving, and post-eval; results and eval artifacts are collected with the shared helpers. The MI355X profile adds the multi-node fabric, model, and cache settings. --- runners/launch_mi355x-amds.sh | 59 ++++++++++++++++++++++++++++++ runners/srt-slurm/mi355x-amds.yaml | 34 +++++++++++++---- 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 90682aeb5d..2395cb9288 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -23,6 +23,65 @@ if [[ "$EXECUTION_PATH" == native-single-node ]]; then exit $? fi +# Multi-node srt-slurm recipes use the shared native path: srt-slurm owns the +# allocation, serving, and post-eval; the recipe owns the workload. +if [[ "$EXECUTION_PATH" == multinode && -n "${CONFIG_FILE:-}" ]]; then + check_env_vars GITHUB_WORKSPACE IMAGE FRAMEWORK RESULT_FILENAME + source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 + SRT_SHARED_BASE=/it-share/gharunners2/srt-slurm + SRTCTL_ROOT="$GITHUB_WORKSPACE/srt-slurm" + rm -rf "$SRTCTL_ROOT" + setup_srt_slurm "$SRTCTL_ROOT" "$FRAMEWORK" 0 || exit 1 + if ! command -v uv >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh + source "$HOME/.local/bin/env" + fi + uv venv .venv + source .venv/bin/activate + uv pip install -e . + export PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" + + # Reuse a provisioned image when one exists; otherwise Pyxis imports it. + SQUASH_FILE="$SRT_SHARED_BASE/containers/$(printf '%s' "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + [[ -f "$SQUASH_FILE" ]] || SQUASH_FILE="$IMAGE" + SLURM_ACCOUNT="$USER" SLURM_PARTITION=compute NGINX_SQUASH_FILE=nginx:1.27.4 \ + write_srt_cluster_config mi355x-amds srtslurm.yaml 0 \ + --var SRT_DEFAULT_TIME_LIMIT 01:00:00 --var GITHUB_WORKSPACE "$GITHUB_WORKSPACE" \ + --container "$IMAGE" "$SQUASH_FILE" \ + --mount /it-share/aiperf-cache /aiperf_mmap_cache || exit 1 + make setup ARCH=x86_64 + export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" + + SRT_JOB_ID="" + trap '[[ -n "$SRT_JOB_ID" ]] && slurm_job_is_active "$SRT_JOB_ID" && scancel "$SRT_JOB_ID"' EXIT + apply_srt_recipe "$CONFIG_FILE" "$FRAMEWORK" "${SRTCTL_EVAL_ARGS[@]}" \ + -f "$CONFIG_FILE" --json --yes > "$GITHUB_WORKSPACE/srt-submission.json" || { + cat "$GITHUB_WORKSPACE/srt-submission.json" >&2 + exit 1 + } + python3 -m infx.srt_slurm.single_node submission "$GITHUB_WORKSPACE/srt-submission.json" \ + > srt-submission-fields || exit 1 + mapfile -t SRT_SUBMISSION < srt-submission-fields + SRT_JOB_ID="${SRT_SUBMISSION[0]}" + LOGS_DIR="${SRT_SUBMISSION[1]}/logs" + + job_rc=0 + stream_slurm_job_log "$SRT_JOB_ID" "$LOGS_DIR/sweep_${SRT_JOB_ID}.log" || job_rc=$? + verify_slurm_job_status "$SRT_JOB_ID" || job_rc=$? + tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$LOGS_DIR" . || job_rc=1 + if [[ "$EVAL_ONLY" != true ]]; then + if [[ "$IS_AGENTIC" == 1 ]]; then + copy_agentic_results "$INFMAX_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" || job_rc=1 + else + copy_fixed_sequence_results "$LOGS_DIR" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" || job_rc=1 + fi + fi + if [[ "$RUN_EVAL" == true || "$EVAL_ONLY" == true ]]; then + cp "$LOGS_DIR"/eval_results/* "$GITHUB_WORKSPACE/" || job_rc=1 + fi + exit "$job_rc" +fi + scancel_sync() { local jobid=$1 local timeout=${2:-600} diff --git a/runners/srt-slurm/mi355x-amds.yaml b/runners/srt-slurm/mi355x-amds.yaml index f29f1fabdc..b49b37548e 100644 --- a/runners/srt-slurm/mi355x-amds.yaml +++ b/runners/srt-slurm/mi355x-amds.yaml @@ -1,17 +1,37 @@ +# srt-slurm cluster profile for the MI355X AMD Slurm cluster. The login and +# compute nodes share /it-share, so source, output, image, and result paths do +# not need node-local transport. + +cluster: mi355x-amds default_partition: compute default_time_limit: ${SRT_DEFAULT_TIME_LIMIT} +output_dir: /it-share/gharunners2/srt-slurm/outputs +srtctl_root: ${SRTCTL_ROOT} + gpus_per_node: 8 -network_interface: '' visible_devices_env: ROCR_VISIBLE_DEVICES -srtctl_root: ${SRTCTL_ROOT} -default_mounts: - /dev/kfd: /dev/kfd - /dev/dri: /dev/dri -default_sbatch_directives: - cpus-per-task: '128' +default_gpu_exporter: null +network_interface: eno0 + use_gpus_per_node_directive: true use_segment_sbatch_directive: false use_exclusive_sbatch_directive: true + +default_sbatch_directives: + cpus-per-task: '128' + +model_paths: + DeepSeek-V4-Pro-0813: /it-share/data/DeepSeek-V4-Pro-0813 + +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + # This host directory is already the Hub cache, not HF_HOME. + /it-share/hf-hub-cache: /hf_hub_cache/hub + /it-share/hf_home: /it-share/hf_home + +nginx_raise_ulimit: false + default_host_setup: # The GPU-drain check can take 15 minutes; leave room for the fabric check # and host-memory preparation. From e836794342b9b911e27702e06f2dc80d502946d0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 17:43:30 -0500 Subject: [PATCH 4/6] feat(amd): run MI355X multi-node fixed-sequence configs on srt-slurm Move the Qwen3.5 FP8/MXFP4 and DeepSeek-R1 FP8/MXFP4 disaggregated configs to srt-slurm recipes. Recipes write results in the shared sa-bench layout, and eval runs drop fake expert dispatch. The legacy multi-node path now only serves AgentX. --- benchmarks/multi_node/amd_utils/bench.sh | 105 --- benchmarks/multi_node/amd_utils/env_atom.sh | 40 - .../multi_node/amd_utils/models_atom.yaml | 30 - .../multi_node/amd_utils/models_vllm.yaml | 25 - .../multi_node/amd_utils/server_atom.sh | 532 ------------- .../multi_node/amd_utils/server_vllm.sh | 485 ------------ .../dsr1_fp4_mi355x_sglang-disagg.sh | 78 -- .../dsr1_fp8_mi355x_sglang-disagg.sh | 78 -- .../qwen3.5_fp4_mi355x_sglang-disagg.sh | 79 -- .../qwen3.5_fp8_mi355x_sglang-disagg.sh | 79 -- .../dsr1/mi355x/fp4-disagg-fixed-seq.yaml | 727 ++++++++++++++++++ .../dsr1/mi355x/fp8-disagg-fixed-seq.yaml | 385 ++++++++++ .../disagg-1p1d-tp4p-tp8d-fixed-seq.yaml | 154 ++++ .../disagg-1p1d-tp8-mxfp4-fixed-seq.yaml | 148 ++++ configs/amd-master.yaml | 416 ++-------- runners/launch_mi355x-amds.sh | 65 +- 16 files changed, 1502 insertions(+), 1924 deletions(-) delete mode 100755 benchmarks/multi_node/amd_utils/bench.sh delete mode 100644 benchmarks/multi_node/amd_utils/env_atom.sh delete mode 100644 benchmarks/multi_node/amd_utils/models_atom.yaml delete mode 100644 benchmarks/multi_node/amd_utils/models_vllm.yaml delete mode 100755 benchmarks/multi_node/amd_utils/server_atom.sh delete mode 100755 benchmarks/multi_node/amd_utils/server_vllm.sh delete mode 100644 benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh delete mode 100644 benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh delete mode 100755 benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh delete mode 100755 benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml diff --git a/benchmarks/multi_node/amd_utils/bench.sh b/benchmarks/multi_node/amd_utils/bench.sh deleted file mode 100755 index f889d26fb2..0000000000 --- a/benchmarks/multi_node/amd_utils/bench.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -# Disaggregated fixed-seq-len benchmark runner; writes JSON results via -# benchmark_serving.py for the CI pipeline. -# -# Usage: bash bench.sh \ -# \ -# - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars ENGINE MODEL_PATH MODEL_NAME ROUTER_PORT -if [[ $# -ne 13 ]]; then - echo "Error: bench.sh requires 13 positional arguments" >&2 - exit 1 -fi - -n_prefill=$1 -n_decode=$2 -prefill_gpus=$3 -decode_gpus=$4 -model_path=$5 -model_name=$6 -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - BENCH_MODEL="${MODEL_NAME}" -else - BENCH_MODEL="${MODEL_PATH}" -fi -log_path=$7 - -chosen_isl=${8} -chosen_osl=${9} -concurrency_list=${10} -chosen_req_rate=${11} -random_range_ratio=${12} -num_prompts_multiplier=${13} - -IFS='x' read -r -a chosen_concurrencies <<< "$concurrency_list" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" - -profile_folder="${log_path}/${ENGINE}_isl_${chosen_isl}_osl_${chosen_osl}" -mkdir -p "$profile_folder" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -for max_concurrency in "${chosen_concurrencies[@]}"; do - - export_file="${profile_folder}/concurrency_${max_concurrency}_req_rate_${chosen_req_rate}_gpus_$((prefill_gpus+decode_gpus))_ctx_${prefill_gpus}_gen_${decode_gpus}" - - num_prompts=$(( max_concurrency * num_prompts_multiplier )) - if [[ "$num_prompts" -lt 16 ]]; then - num_prompts=16 - fi - - echo "profile_folder: $profile_folder" - echo "max_concurrency: $max_concurrency" - echo "chosen_req_rate: $chosen_req_rate" - echo "MODEL_PATH: $MODEL_PATH" - echo "ROUTER_PORT: $ROUTER_PORT" - echo "chosen_isl: $chosen_isl" - echo "chosen_osl: $chosen_osl" - echo "num_prompts: $num_prompts" - echo "export_file: $export_file" - - extra_flags="" - if [[ "$ENGINE" == "vllm-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - elif [[ "$ENGINE" == "atom-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - if [ "$IS_MTP" = "true" ]; then - # just override extra_flags as dsv3 use different tokenizer path - extra_flags="--use-chat-template" - fi - else - if [ "$IS_MTP" = "true" ]; then - extra_flags="--use-chat-template" - fi - fi - - run_benchmark_serving \ - --bench-serving-dir "$REPO_ROOT" \ - --model "$BENCH_MODEL" \ - --port "$ROUTER_PORT" \ - --backend openai \ - --input-len "$chosen_isl" \ - --output-len "$chosen_osl" \ - --random-range-ratio "$random_range_ratio" \ - --num-prompts "$num_prompts" \ - --max-concurrency "$max_concurrency" \ - --result-filename "$export_file" \ - --result-dir /workspace/ \ - $extra_flags - - echo "-----------------------------------------" - - if [[ "$ENGINE" == "vllm-disagg" ]]; then - echo "[BENCH] Cooldown: waiting 10s for idle KV block reaper..." - sleep 10 - fi -done diff --git a/benchmarks/multi_node/amd_utils/env_atom.sh b/benchmarks/multi_node/amd_utils/env_atom.sh deleted file mode 100644 index 71cbdf06ff..0000000000 --- a/benchmarks/multi_node/amd_utils/env_atom.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# ATOM/mooncake environment, sourced by server_atom.sh in place of env.sh. -# IBDEVICES: RDMA device names (e.g. ionic_0,ionic_1,...), set by the runner or -# auto-detected. - -set -x - -export PYTHONUNBUFFERED=1 -export PYTHONDONTWRITEBYTECODE=1 - - -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - # ATOM passes no IB device to the server (mooncake picks its own RDMA device via - # proxy_ip/handshake_port), so a missing IBDEVICES is non-fatal here. - echo "[WARN] Unable to detect RDMA devices via ibv_devinfo; IBDEVICES unset (non-fatal for ATOM/mooncake)" >&2 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - - -export LD_LIBRARY_PATH=/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib:${LD_LIBRARY_PATH:-} - -export SAFETENSORS_FAST_GPU=1 - -export VLLM_LOG_LEVEL=WARNING -export ATOM_LOG_LEVEL=WARNING -export AITER_LOG_LEVEL=WARNING -export LOG_LEVEL=WARNING -export LOGLEVEL=WARNING - -set +x - -echo "[INFO] ATOM env: IBDEVICES=$IBDEVICES LD_LIBRARY_PATH includes mooncake" \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/models_atom.yaml b/benchmarks/multi_node/amd_utils/models_atom.yaml deleted file mode 100644 index d3232c01da..0000000000 --- a/benchmarks/multi_node/amd_utils/models_atom.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model-specific ATOM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# env: str # Space-separated KEY=VALUE pairs exported unconditionally -# tp_dp_flags: str # Shared TP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_tp_dp_flags: str # TP+DPA flags for prefill only (overrides tp_dp_flags) -# decode_tp_dp_flags: str # TP+DPA flags for decode only (overrides tp_dp_flags) -# tp_dp_env: str # Space-separated KEY=VALUE pairs exported only in TP+DPA mode -# ep_dp_flags: str # Shared EP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_ep_dp_flags: str # EP+DPA flags for prefill only (overrides ep_dp_flags) -# decode_ep_dp_flags: str # EP+DPA flags for decode only (overrides ep_dp_flags) -# ep_dp_env: str # Space-separated KEY=VALUE pairs exported only in EP+DPA mode -# mtp_flags: str # Flags passed to SPEC_ARGS before $DECODE_MTP_SIZE (e.g. "--method mtp --num-speculative-tokens") -# kv_cache_flags: str # Full --kv_cache_dtype flag string (e.g. "--kv_cache_dtype fp8", or "" for none) -# online_quant_config: str # JSON string passed to --online_quant_config (used when DPA is disabled) -# online_quant_dpa_config: str # JSON string passed to --online_quant_config when DPA is enabled (falls back to online_quant_config) -# block_size: str # --block-size value (overrides server_atom.sh default of 16) -# mem_frac_static: str # --gpu-memory-utilization value (overrides default of 0.85) -# max_model_len: str # --max-model-len value (overrides default of unset) -# max_num_seqs: str # --max-num-seqs value (overrides default of 256) -# max_num_batched_tokens: str # --max-num-batched-tokens value (overrides default of unset) -# scheduler_delay_factor: str # --scheduler-delay-factor value (overrides default of unset) -# All registered model/scenario combinations are retired; see deprecated/models_atom.yaml. -{} diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml deleted file mode 100644 index 515e5caf83..0000000000 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Model-specific vLLM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the model identifier -# used in amd-master.yaml and the directory/HF-cache name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# prefill_flags: str # vLLM CLI flags for prefill workers -# decode_flags: str # vLLM CLI flags for decode workers -# env: str # Space-separated KEY=VALUE pairs exported before vllm serve -# hf_dir: str # (optional) On-disk directory name if it differs from the key -# # e.g. HF cache layout: models--org--checkpoint - -Llama-3.1-405B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" - -amd-Llama-3.3-70B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" diff --git a/benchmarks/multi_node/amd_utils/server_atom.sh b/benchmarks/multi_node/amd_utils/server_atom.sh deleted file mode 100755 index e08b1cba5b..0000000000 --- a/benchmarks/multi_node/amd_utils/server_atom.sh +++ /dev/null @@ -1,532 +0,0 @@ -#!/bin/bash -# ATOM disaggregated launcher: mooncake RDMA KV transfer and atomesh routing. - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars \ - MODEL_NAME ROUTER_PORT PREFILL_PORT DECODE_PORT HANDSHAKE_PORT \ - MEM_FRAC_STATIC BLOCK_SIZE MAX_NUM_SEQS WAIT_SERVER_TIMEOUT - -check_env_vars \ - NODE0_ADDR NODE_RANK xP yD IPADDRS \ - PREFILL_TP_SIZE DECODE_TP_SIZE PREFILL_ENABLE_EP PREFILL_ENABLE_DP DECODE_ENABLE_EP \ - DECODE_ENABLE_DP DECODE_MTP_SIZE BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_RANDOM_RANGE_RATIO \ - BENCH_REQUEST_RATE BENCH_NUM_PROMPTS_MULTIPLIER BENCH_MAX_CONCURRENCY DRY_RUN GPUS_PER_NODE \ - RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK BENCHMARK_LOGS_DIR MODEL_DIR \ - ATOM_WS_PATH - -EXTRA_SERVER_ARGS="${EXTRA_SERVER_ARGS:-}" - -source $ATOM_WS_PATH/setup_deps.sh -source $ATOM_WS_PATH/env_atom.sh - -# lm-eval with high num_concurrent exhausts the default 1024 FD limit. -ulimit -n 65536 2>/dev/null || ulimit -n 8192 2>/dev/null || true -echo "ulimit -n (open files): $(ulimit -n)" - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -if [[ -z "$host_ip" ]]; then - host_ip=$(hostname -I 2>/dev/null | awk '{print $1}') -fi -host_name=$(hostname) - -set -x -_yaml_tmp=$(mktemp) -python3 << PYEOF > "$_yaml_tmp" -import yaml -with open('${ATOM_WS_PATH}/models_atom.yaml') as f: - m = yaml.safe_load(f).get('${MODEL_NAME}', {}) -def sh(v): return v.replace("'", "'\\''") -print(f"MODEL_ENVS='{sh(m.get('env', ''))}'") -_tp_dp = m.get('tp_dp_flags', '') -print(f"PREFILL_MODEL_TP_DP_FLAGS='{sh(m.get('prefill_tp_dp_flags', _tp_dp))}'") -print(f"DECODE_MODEL_TP_DP_FLAGS='{sh(m.get('decode_tp_dp_flags', _tp_dp))}'") -_ep_dp = m.get('ep_dp_flags', '') -print(f"PREFILL_MODEL_EP_DP_FLAGS='{sh(m.get('prefill_ep_dp_flags', _ep_dp))}'") -print(f"DECODE_MODEL_EP_DP_FLAGS='{sh(m.get('decode_ep_dp_flags', _ep_dp))}'") -print(f"MODEL_TP_DP_ENV='{sh(m.get('tp_dp_env', ''))}'") -print(f"MODEL_EP_DP_ENV='{sh(m.get('ep_dp_env', ''))}'") -print(f"MODEL_MTP_FLAGS='{sh(m.get('mtp_flags', ''))}'") -print(f"MODEL_KV_ARG='{sh(m.get('kv_cache_flags', ''))}'") -print(f"_ONLINE_QUANT_CONFIG='{sh(m.get('online_quant_config', ''))}'") -print(f"_ONLINE_QUANT_DPA_CONFIG='{sh(m.get('online_quant_dpa_config', m.get('online_quant_config', '')))}'") -print(f"_YAML_BLOCK_SIZE='{sh(m.get('block_size', ''))}'") -print(f"_YAML_MEM_FRAC_STATIC='{sh(m.get('mem_frac_static', ''))}'") -print(f"_YAML_MAX_MODEL_LEN='{sh(m.get('max_model_len', ''))}'") -print(f"_YAML_MAX_NUM_SEQS='{sh(m.get('max_num_seqs', ''))}'") -print(f"_YAML_MAX_NUM_BATCHED_TOKENS='{sh(m.get('max_num_batched_tokens', ''))}'") -print(f"_YAML_SCHEDULER_DELAY_FACTOR='{sh(m.get('scheduler_delay_factor', ''))}'") -PYEOF -# shellcheck source=/dev/null -source "$_yaml_tmp" -rm -f "$_yaml_tmp" -unset _yaml_tmp - -# Model YAML overrides the caller-provided server tuning. -BLOCK_SIZE="${_YAML_BLOCK_SIZE:-${BLOCK_SIZE}}" -MEM_FRAC_STATIC="${_YAML_MEM_FRAC_STATIC:-${MEM_FRAC_STATIC}}" -MAX_MODEL_LEN="${_YAML_MAX_MODEL_LEN:-${MAX_MODEL_LEN:-}}" -MAX_NUM_SEQS="${_YAML_MAX_NUM_SEQS:-${MAX_NUM_SEQS}}" -MAX_NUM_BATCHED_TOKENS="${_YAML_MAX_NUM_BATCHED_TOKENS:-${MAX_NUM_BATCHED_TOKENS:-}}" -SCHEDULER_DELAY_FACTOR="${_YAML_SCHEDULER_DELAY_FACTOR:-${SCHEDULER_DELAY_FACTOR:-}}" -unset _YAML_BLOCK_SIZE _YAML_MEM_FRAC_STATIC _YAML_MAX_MODEL_LEN _YAML_MAX_NUM_SEQS _YAML_MAX_NUM_BATCHED_TOKENS _YAML_SCHEDULER_DELAY_FACTOR - -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -PREFILL_ARGS="" -PREFILL_IPS=() -for i in $(seq 0 $((xP - 1))); do - idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_IPS[$i]="${IP_ARRAY[$idx]}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$idx]}:${PREFILL_PORT}" -done - -DECODE_ARGS="" -DECODE_IPS=() -for i in $(seq 0 $((yD - 1))); do - idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_IPS[$i]="${IP_ARRAY[$idx]}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$idx]}:${DECODE_PORT}" -done - -PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE") #TP -ONLINE_QUANT_ARG="" -if [ "$PREFILL_ENABLE_DP" = "true" ]; then - if [ "$PREFILL_ENABLE_EP" = "true" ]; then #EP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi - if [[ -n "$_ONLINE_QUANT_DPA_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_DPA_CONFIG}'" - fi -else - if [[ -n "$_ONLINE_QUANT_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_CONFIG}'" - fi -fi - -DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE") #TP -if [ "$DECODE_ENABLE_DP" = "true" ]; then - if [ "$DECODE_ENABLE_EP" = "true" ]; then #EP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi -fi -unset _dp_env_pair -unset _ONLINE_QUANT_CONFIG _ONLINE_QUANT_DPA_CONFIG - -for _env_pair in ${MODEL_ENVS}; do - export "$_env_pair" -done -unset _env_pair - -SPEC_ARGS=() -if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE}" -gt 0 ]]; then - SPEC_ARGS=(${MODEL_MTP_FLAGS} "$DECODE_MTP_SIZE") -fi - -KV_CACHE_ARG="${MODEL_KV_ARG}" - -MODEL_LEN_ARGS="" -if [[ -n "$MAX_MODEL_LEN" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-model-len ${MAX_MODEL_LEN}" -fi -if [[ -n "$MAX_NUM_BATCHED_TOKENS" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS}" -fi -if [[ -n "$SCHEDULER_DELAY_FACTOR" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --scheduler-delay-factor ${SCHEDULER_DELAY_FACTOR}" -fi - -cat < prefill node 0 + router; 1..NODE_OFFSET-1 -> prefill; -# NODE_OFFSET.. -> decode. -if [ "$NODE_RANK" -eq 0 ]; then - echo "NODE INFO =======================================" - echo "${host_name}:${host_ip} is Prefill Node 0 + Router" - echo "Prefill TP=${PREFILL_TP_SIZE}, Decode TP=${DECODE_TP_SIZE}" - echo "Prefill servers: ${PREFILL_ARGS}" - echo "Decode servers: ${DECODE_ARGS}" - echo "================================================" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill0_${host_name}.log & - set +x - prefill0_pid=$! - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for all servers to be up (timeout=${WAIT_SERVER_TIMEOUT}s)..." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for prefill/decode /health endpoints" - else - _deadline=$(( $(date +%s) + WAIT_SERVER_TIMEOUT )) - for _ip in "${PREFILL_IPS[@]}"; do - echo "[wait] prefill http://${_ip}:${PREFILL_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${PREFILL_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] prefill ${_ip}:${PREFILL_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] prefill ${_ip}:${PREFILL_PORT} ready" - done - for _ip in "${DECODE_IPS[@]}"; do - echo "[wait] decode http://${_ip}:${DECODE_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${DECODE_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] decode ${_ip}:${DECODE_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] decode ${_ip}:${DECODE_PORT} ready" - done - fi - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "All servers up. Starting atomesh router..." - - ROUTER_CMD="/usr/local/bin/atomesh launch \ - --host 0.0.0.0 --port ${ROUTER_PORT} \ - --pd-disaggregation \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS} \ - --policy random \ - --backend atom \ - --log-level info \ - --disable-health-check \ - --disable-circuit-breaker \ - --prometheus-port 29100" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/tmp/slurm_job-${SLURM_JOB_ID}_router_${host_name}.log" - set -x - eval "$ROUTER_CMD" 2>&1 | tee "$ROUTER_LOG_FILE" & - set +x - proxy_pid=$! - - check_env_vars WAIT_LOCAL_ROUTER_TIMEOUT - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-$WAIT_LOCAL_ROUTER_TIMEOUT}" - echo "[wait] router http://0.0.0.0:${ROUTER_PORT}/v1/models (timeout=${WAIT_ROUTER_TIMEOUT}s)" - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/v1/models" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${ROUTER_PORT}/v1/models not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router /v1/models ready" - - echo "Router is ready for benchmarking" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - cd $ATOM_WS_PATH - - export IS_MTP="false" - if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE}" -gt 0 ]]; then - export IS_MTP="true" - fi - - BENCH_CMD="bash $ATOM_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - if [[ "${RUN_EVAL}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --port ${ROUTER_PORT} (framework=${EVAL_FRAMEWORK}, conc=${EVAL_CONCURRENT_REQUESTS})" - else - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" run_eval --port "${ROUTER_PORT}" - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - export DECODE_NUM_WORKERS="${yD}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" append_lm_eval_summary - - fi - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - if stage_eval_artifacts \ - "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then - echo "Eval artifacts staged in $EVAL_COPY_DIR" - else - echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2 - EVAL_FAILED=1 - fi - fi - - popd - fi - fi - - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR}/logs" - mkdir -p "$LOGS_OUTPUT" - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Waiting 60s before killing router and prefill server..." - sleep 60 - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing router and prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - kill $proxy_pid - kill $prefill0_pid - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - echo "${host_name}:${host_ip} is Prefill Node (rank ${NODE_RANK})" - - prefill_worker_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_IP="${PREFILL_IPS[$prefill_worker_idx]}" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & - set +x - prefill_pid=$! - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - check_env_vars WAIT_REMOTE_ROUTER_TIMEOUT - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-$WAIT_REMOTE_ROUTER_TIMEOUT}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing prefill server (rank ${NODE_RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $prefill_pid 2>/dev/null; fi - -else - RANK=$((NODE_RANK - NODE_OFFSET)) - echo "${host_name}:${host_ip} is Decode Node (rank ${RANK})" - - _MAX_CONC=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - CUDAGRAPH_SIZES='[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' - - DECODE_MAX_NUM_SEQS="${_MAX_CONC}" - - DECODE_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${DECODE_PORT} \ - --trust-remote-code \ - ${DECODE_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${DECODE_MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_consumer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - eval "$DECODE_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log & - set +x - decode_pid=$! - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - check_env_vars WAIT_REMOTE_ROUTER_TIMEOUT - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-$WAIT_REMOTE_ROUTER_TIMEOUT}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing decode server (rank ${RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $decode_pid 2>/dev/null; fi -fi - -echo "Script completed successfully" -exit 0 \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh deleted file mode 100755 index 158bdefc62..0000000000 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ /dev/null @@ -1,485 +0,0 @@ -#!/bin/bash - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only - -check_env_vars \ - NODE0_ADDR NODE_RANK MODEL_NAME xP yD \ - IPADDRS PREFILL_TP_SIZE DECODE_TP_SIZE PREFILL_ENABLE_EP PREFILL_ENABLE_DP \ - DECODE_ENABLE_EP DECODE_ENABLE_DP BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_RANDOM_RANGE_RATIO \ - BENCH_REQUEST_RATE BENCH_NUM_PROMPTS_MULTIPLIER BENCH_MAX_CONCURRENCY DRY_RUN GPUS_PER_NODE \ - RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK BENCHMARK_LOGS_DIR MODEL_DIR \ - WS_PATH ROUTER_PORT SERVER_PORT PROXY_PING_PORT MODEL_PATH - -# vLLM Disaggregated Server Launcher with Model-Specific Configurations -# -# Node role assignment (by NODE_RANK): -# 0 -> Proxy/Router + first Prefill node (kv_producer) -# 1..xP-1 -> Additional Prefill nodes (kv_producer) -# xP..xP+yD-1 -> Decode nodes (kv_consumer) -# -# Total nodes = xP + yD (router co-located with first prefill, like SGLang). - -# setup_deps.sh is idempotent; required on the base vLLM image. -source "$(dirname "${BASH_SOURCE[0]}")/setup_deps.sh" - -# Prefer MODEL_PATH from job.slurm (handles HF cache snapshot resolution) - -source $WS_PATH/env.sh - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -# RDMA IP for Nixl KV transfer (prefer 192.168.x.x subnet if available) -rdma_ip=$(hostname -I | tr ' ' '\n' | grep '^192\.168\.' | head -1) -rdma_ip="${rdma_ip:-$host_ip}" -host_name=$(hostname) - -echo "[INFO] Management IP (barriers/proxy): $host_ip" -echo "[INFO] RDMA IP (Nixl KV transfer): $rdma_ip" - -setup_rdma_env() { - # Pensando ionic (RoCEv2) point-to-point /31 route fix. - # Each benic interface has a /31 to the TOR switch. Without explicit routes, - # traffic to other nodes' RDMA IPs falls through to the management network. - if [[ "$rdma_ip" =~ ^192\.168\.([0-9]+)\.([0-9]+)$ ]]; then - local rdma_subnet="${BASH_REMATCH[1]}" - local rdma_host="${BASH_REMATCH[2]}" - local rdma_gw="192.168.${rdma_subnet}.$(( rdma_host | 1 ))" - local rdma_iface - rdma_iface=$(ip -o addr show | awk -v ip="$rdma_ip" '$4 ~ ip {print $2}' | head -1) - if [[ -n "$rdma_iface" ]]; then - ip route replace "192.168.${rdma_subnet}.0/24" via "$rdma_gw" dev "$rdma_iface" 2>/dev/null && \ - echo "[RDMA-ROUTE] Added 192.168.${rdma_subnet}.0/24 via $rdma_gw dev $rdma_iface" || \ - echo "[RDMA-ROUTE] Route add failed for 192.168.${rdma_subnet}.0/24" - fi - fi - - # Nixl UCX backend: ucx_error_handling_mode=none. Under high concurrency (C512+) - # UCX's default UCP_ERR_HANDLING_MODE_PEER runs transport-level error recovery on - # ibv_post_send failures, which stops RIXL RDMA READ retries from recovering; the - # prefill KV cache then fills to 100% and the pipeline deadlocks. Needed on every - # NIC type, not just ionic. - local nixl_api - nixl_api=$(python3 -c "import rixl._api; print(rixl._api.__file__)" 2>/dev/null) - if [[ -n "$nixl_api" ]]; then - if ! grep -q 'ucx_error_handling_mode' "$nixl_api"; then - sed -i '/self\.create_backend(bknd, init)/i\ init["ucx_error_handling_mode"] = "none"' "$nixl_api" - echo "[PATCH] Added ucx_error_handling_mode=none to $nixl_api (IBDEVICES=${IBDEVICES:-unset})" - else - echo "[PATCH] ucx_error_handling_mode already set in $nixl_api" - fi - fi -} - -setup_rdma_env - -if [[ -z "$UCX_NET_DEVICES" ]]; then - echo "Error: UCX_NET_DEVICES is empty after env.sh detection" >&2 - exit 1 -fi - -MODELS_YAML="${WS_PATH}/models_vllm.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "$MODEL_NAME" ]]; then - echo "ERROR: MODEL_NAME is not set"; exit 1 -fi - -eval "$(python3 -c " -import yaml, sys - -with open('${MODELS_YAML}') as f: - models = yaml.safe_load(f) - -model_name = '${MODEL_NAME}' -if model_name not in models: - print(f'echo \"ERROR: Model {model_name} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_name] - -def bash_escape(s): - \"\"\"Escape a value for safe embedding in a bash double-quoted assignment.\"\"\" - return s.replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"').replace('\$', '\\\\\$').replace('\`', '\\\\\`') - -pf = bash_escape(m.get('prefill_flags', '--tensor-parallel-size 8')) -df = bash_escape(m.get('decode_flags', '--tensor-parallel-size 8')) -ev = bash_escape(m.get('env', '')) -dev = bash_escape(m.get('decode_env', '')) -pev = bash_escape(m.get('prefill_env', '')) -print(f'PREFILL_SERVER_CONFIG=\"{pf}\"') -print(f'DECODE_SERVER_CONFIG=\"{df}\"') -print(f'MODEL_ENVS=\"{ev}\"') -print(f'DECODE_MODEL_ENVS=\"{dev}\"') -print(f'PREFILL_MODEL_ENVS=\"{pev}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -if [[ -n "${PREFILL_TP_SIZE:-}" ]]; then - if echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${PREFILL_TP_SIZE}/g") - else - PREFILL_SERVER_CONFIG+=" --tensor-parallel-size ${PREFILL_TP_SIZE}" - fi -fi -if [[ -n "${DECODE_TP_SIZE:-}" ]]; then - if echo "$DECODE_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${DECODE_TP_SIZE}/g") - else - DECODE_SERVER_CONFIG+=" --tensor-parallel-size ${DECODE_TP_SIZE}" - fi -fi -if [[ "${PREFILL_ENABLE_EP}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - PREFILL_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${PREFILL_ENABLE_DP}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - PREFILL_SERVER_CONFIG+=" --enable-dp-attention" -fi -if [[ "${DECODE_ENABLE_EP}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - DECODE_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${DECODE_ENABLE_DP}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - DECODE_SERVER_CONFIG+=" --enable-dp-attention" -fi - -echo "PREFILL_SERVER_CONFIG (after TP/EP/DP): $PREFILL_SERVER_CONFIG" -echo "DECODE_SERVER_CONFIG (after TP/EP/DP): $DECODE_SERVER_CONFIG" - -echo "Waiting at the container creation barrier on $host_name" -python3 $WS_PATH/sync.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout 600 - -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_ARGS="" -DECODE_ARGS="" - -for ((i=0; i "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for all prefill and decode servers to be up . . ." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: skipping barrier (wait-for-all-ports)" - else - python3 $WS_PATH/sync.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports $SERVER_PORT \ - --wait-for-all-ports \ - --timeout 1800 - fi - - echo "Congratulations!!! All prefill and decode servers are up . . ." - - HEALTH_BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-health \ - --health-endpoint /health \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - eval "$HEALTH_BARRIER_CMD" - echo "MoRI-IO proxy is ready for benchmarking" - fi - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - echo "Benchmarking on ${host_name}:${host_ip}" - cd $WS_PATH - - export ROUTER_PORT=$ROUTER_PORT - BENCH_CMD="bash $WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - if [[ "${RUN_EVAL}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --port $ROUTER_PORT (framework=${EVAL_FRAMEWORK}, conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - run_eval --port "$ROUTER_PORT" - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export EP_SIZE=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && EP_SIZE="${PREFILL_TP_SIZE}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - [[ "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" - export DECODE_NUM_WORKERS="${yD}" - export DP_ATTENTION="${PREFILL_ENABLE_DP}" - export PREFILL_DP_ATTENTION="${PREFILL_ENABLE_DP}" - export DECODE_DP_ATTENTION="${DECODE_ENABLE_DP}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - append_lm_eval_summary - - fi - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - if stage_eval_artifacts \ - "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then - echo "Eval artifacts staged in $EVAL_COPY_DIR" - else - echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2 - EVAL_FAILED=1 - fi - fi - - popd - fi - fi - - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Killing the prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - [[ -n "${prefill_pid:-}" ]] && kill $prefill_pid 2>/dev/null || true - sleep 2 - pkill -f "vllm serve" 2>/dev/null || true - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then - echo "${host_name}:${host_ip} is Additional Prefill Node (Model: ${MODEL_NAME})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${PREFILL_MODEL_ENVS}; do - export "$env_pair" - echo "[PREFILL_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - PREFILL_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${PREFILL_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - PREFILL_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log" - set -x - eval "$PREFILL_CMD" > "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the prefill server" - [[ "$DRY_RUN" -eq 0 ]] && kill $prefill_pid 2>/dev/null || true - -else - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${DECODE_MODEL_ENVS}; do - export "$env_pair" - echo "[DECODE_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - DECODE_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_consumer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${DECODE_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - DECODE_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log" - set -x - eval "$DECODE_CMD" > "$DECODE_LOG_FILE" 2>&1 & - set +x - decode_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the decode server" - [[ "$DRY_RUN" -eq 0 ]] && kill $decode_pid 2>/dev/null || true -fi - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh deleted file mode 100644 index a7e5df573d..0000000000 --- a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh deleted file mode 100644 index aed3695934..0000000000 --- a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" \ No newline at end of file diff --git a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 560daa8673..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 560daa8673..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml new file mode 100644 index 0000000000..5189e3f052 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml @@ -0,0 +1,727 @@ +schema: 2 +base: + name: mi355x-dsr1-fp4-disagg-fixed-seq + model: + path: /it-share/data/DeepSeek-R1-0528-MXFP4-v2 + container: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + precision: fp4 + identity: + model: + repo: amd/DeepSeek-R1-0528-MXFP4-v2 + container: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + frameworks: + sglang: 0.5.17.dev20260809+g7120f3ee13 + sglang-router: 0.3.2 + amd-mori: 0.5.17.dev20260809+g7120f3ee13 + slurm: + time_limit: 08:00:00 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + PYTHONUNBUFFERED: '1' + IBDEVICES: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + GLOO_SOCKET_IFNAME: eno0 + NCCL_SOCKET_IFNAME: eno0 + MORI_RDMA_TC: '104' + MORI_IO_TC: '104' + MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' + MORI_IO_QP_MAX_SEND_WR: '16384' + MORI_IO_QP_MAX_CQE: '32768' + MORI_IO_QP_MAX_SGE: '2' + MORI_SHMEM_MODE: ISOLATION + MORI_ENABLE_SDMA: 'false' + MORI_EP_LAUNCH_CONFIG_MODE: AUTO + SGLANG_USE_AITER: '1' + AITER_LOG_LEVEL: ERROR + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: '0' + SGLANG_MORI_DISPATCH_DTYPE: auto + SGLANG_MORI_QP_PER_TRANSFER: '4' + SGLANG_MORI_NUM_WORKERS: '4' + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_MORI_MOE_MAX_INPUT_TOKENS: '32768' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '1024' + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: '32' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600' + SGLANG_HEALTH_CHECK_TIMEOUT: '600' + args: + served-model-name: amd/DeepSeek-R1-0528-MXFP4-v2 + trust-remote-code: true + tp-size: 8 + disaggregation-transfer-backend: mori + kv-cache-dtype: fp8_e4m3 + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.8 + max-running-requests: 128 + chunked-prefill-size: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + enable-metrics: true + enable-cache-report: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + PYTHONUNBUFFERED: '1' + IBDEVICES: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + GLOO_SOCKET_IFNAME: eno0 + NCCL_SOCKET_IFNAME: eno0 + MORI_RDMA_TC: '104' + MORI_IO_TC: '104' + MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' + MORI_IO_QP_MAX_SEND_WR: '16384' + MORI_IO_QP_MAX_CQE: '32768' + MORI_IO_QP_MAX_SGE: '2' + MORI_SHMEM_MODE: ISOLATION + MORI_ENABLE_SDMA: 'false' + MORI_EP_LAUNCH_CONFIG_MODE: AUTO + SGLANG_USE_AITER: '1' + AITER_LOG_LEVEL: ERROR + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_ENABLE_OVERLAP_PLAN_STREAM: '0' + SGLANG_MORI_DISPATCH_DTYPE: auto + SGLANG_MORI_QP_PER_TRANSFER: '4' + SGLANG_MORI_NUM_WORKERS: '4' + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MORI_MOE_MAX_INPUT_TOKENS: '2703' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '1024' + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: '32' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600' + SGLANG_HEALTH_CHECK_TIMEOUT: '600' + args: + served-model-name: amd/DeepSeek-R1-0528-MXFP4-v2 + trust-remote-code: true + tp-size: 8 + disaggregation-transfer-backend: mori + kv-cache-dtype: fp8_e4m3 + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.85 + max-running-requests: 128 + chunked-prefill-size: 262144 + disable-radix-cache: false + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + enable-metrics: true + enable-cache-report: true + prefill-round-robin-balance: true + sbatch_directives: + cpus-per-task: '128' + mem: '0' + srun_options: + mem: '0' + container-writable: '' + container-remap-root: '' + health_check: + max_attempts: 720 + interval_seconds: 5 + benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP USE_CHAT_TEMPLATE BENCHMARK_VARIANT + result_dir=/logs/sa-bench_isl_8192_osl_1024 + mkdir -p "${result_dir}" + ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + gen=$((DECODE_NUM_WORKERS * DECODE_TP)) + chat_args=() + if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then + chat_args+=(--use-chat-template) + fi + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model amd/DeepSeek-R1-0528-MXFP4-v2 \ + --tokenizer /model \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + "${chat_args[@]}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_dir}" \ + --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + BENCHMARK_VARIANT: stp-1p1d-tp8 + USE_CHAT_TEMPLATE: '0' +override_stp_1p1d_tp8: + name: mi355x-dsr1-fp4-stp-1p1d-tp8-fixed-seq + benchmark: + env: + BENCHMARK_VARIANT: stp-1p1d-tp8 +override_stp_1p2d_tp8: + name: mi355x-dsr1-fp4-stp-1p2d-tp8-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + decode: + nodes: 2 + workers: 2 + benchmark: + env: + BENCHMARK_VARIANT: stp-1p2d-tp8 +override_stp_1p2d_tp4_tp8: + name: mi355x-dsr1-fp4-stp-1p2d-tp4-tp8-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + args: + tp-size: 4 + decode: + nodes: 2 + workers: 2 + gpus: 8 + benchmark: + env: + BENCHMARK_VARIANT: stp-1p2d-tp4-tp8 +override_stp_1p1d_dep8: + name: mi355x-dsr1-fp4-stp-1p1d-dep8-fixed-seq + roles: + prefill: + env: + MORI_ENABLE_SDMA: 'true' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '64' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '128' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, + 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, + 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512] + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + benchmark: + env: + BENCHMARK_VARIANT: stp-1p1d-dep8 +override_stp_2p1d_dep8: + name: mi355x-dsr1-fp4-stp-2p1d-dep8-fixed-seq + resources: {} + roles: + prefill: + nodes: 2 + workers: 2 + env: + MORI_ENABLE_SDMA: 'true' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + nodes: 1 + workers: 1 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '1024' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, + 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, + 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512] + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + benchmark: + env: + BENCHMARK_VARIANT: stp-2p1d-dep8 +override_mtp3_1p1d_tp8: + name: mi355x-dsr1-fp4-mtp3-1p1d-tp8-fixed-seq + roles: + prefill: + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + decode: + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048' + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + BENCHMARK_VARIANT: mtp3-1p1d-tp8 + USE_CHAT_TEMPLATE: '1' +override_mtp3_1p2d_tp8_wide: + name: mi355x-dsr1-fp4-mtp3-1p2d-tp8-wide-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + decode: + nodes: 2 + workers: 2 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048' + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + BENCHMARK_VARIANT: mtp3-1p2d-tp8-wide + USE_CHAT_TEMPLATE: '1' +override_mtp3_1p2d_tp8_narrow: + name: mi355x-dsr1-fp4-mtp3-1p2d-tp8-narrow-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + decode: + nodes: 2 + workers: 2 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '2048' + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + benchmark: + env: + BENCHMARK_VARIANT: mtp3-1p2d-tp8-narrow + USE_CHAT_TEMPLATE: '1' +override_mtp2_1p2d_tp8: + name: mi355x-dsr1-fp4-mtp2-1p2d-tp8-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + decode: + nodes: 2 + workers: 2 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1536' + args: + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + benchmark: + env: + BENCHMARK_VARIANT: mtp2-1p2d-tp8 + USE_CHAT_TEMPLATE: '1' +override_mtp3_1p1d_dep8: + name: mi355x-dsr1-fp4-mtp3-1p1d-dep8-fixed-seq + roles: + prefill: + env: + MORI_ENABLE_SDMA: 'true' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + max-running-requests: 640 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '320' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '160' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 3 + speculative-num-draft-tokens: 4 + max-running-requests: 640 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, + 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, + 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512] + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + benchmark: + env: + BENCHMARK_VARIANT: mtp3-1p1d-dep8 + USE_CHAT_TEMPLATE: '1' +override_mtp1_1p1d_dep8: + name: mi355x-dsr1-fp4-mtp1-1p1d-dep8-fixed-seq + roles: + prefill: + env: + MORI_ENABLE_SDMA: 'true' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '128' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '128' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 512 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, + 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, + 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512] + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + benchmark: + env: + BENCHMARK_VARIANT: mtp1-1p1d-dep8 + USE_CHAT_TEMPLATE: '1' +override_mtp1_2p1d_dep8: + name: mi355x-dsr1-fp4-mtp1-2p1d-dep8-fixed-seq + resources: {} + roles: + prefill: + nodes: 2 + workers: 2 + env: + MORI_ENABLE_SDMA: 'true' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3] + decode: + nodes: 1 + workers: 1 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1024' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '1024' + args: + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-draft-model-path: SGLang/DeepSeek-R1-NextN + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-attention-mode: decode + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 4096 + chunked-prefill-size: 65536 + context-length: 9217 + max-total-tokens: 131072 + enable-two-batch-overlap: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, + 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, + 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, + 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, + 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, + 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, + 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, + 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, + 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, + 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, + 432, 433, 434, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, + 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494, 495, 496, 497, 498, 499, 500, + 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512] + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + benchmark: + env: + BENCHMARK_VARIANT: mtp1-2p1d-dep8 + USE_CHAT_TEMPLATE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml new file mode 100644 index 0000000000..cd3d9d16f0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml @@ -0,0 +1,385 @@ +schema: 2 +base: + name: mi355x-dsr1-fp8-disagg-fixed-seq + model: + path: hf:deepseek-ai/DeepSeek-R1-0528 + container: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + precision: fp8 + identity: + model: + repo: deepseek-ai/DeepSeek-R1-0528 + container: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + frameworks: + sglang: 0.5.17.dev20260809+g7120f3ee13 + sglang-router: 0.3.2 + amd-mori: 0.5.17.dev20260809+g7120f3ee13 + slurm: + time_limit: 08:00:00 + resources: + gpu_type: mi355x + gpus_per_node: 8 + frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + engine: sglang + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + PYTHONUNBUFFERED: '1' + IBDEVICES: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + GLOO_SOCKET_IFNAME: eno0 + NCCL_SOCKET_IFNAME: eno0 + MORI_RDMA_TC: '104' + MORI_IO_TC: '104' + MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' + MORI_IO_QP_MAX_SEND_WR: '16384' + MORI_IO_QP_MAX_CQE: '32768' + MORI_IO_QP_MAX_SGE: '2' + MORI_SHMEM_MODE: ISOLATION + SGLANG_USE_AITER: '1' + AITER_LOG_LEVEL: ERROR + SGLANG_MORI_DISPATCH_DTYPE: auto + SGLANG_MORI_QP_PER_TRANSFER: '4' + SGLANG_MORI_NUM_WORKERS: '4' + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '8192' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '1024' + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: '32' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600' + SGLANG_HEALTH_CHECK_TIMEOUT: '600' + args: + served-model-name: deepseek-ai/DeepSeek-R1-0528 + trust-remote-code: true + tp-size: 8 + disaggregation-transfer-backend: mori + kv-cache-dtype: fp8_e4m3 + attention-backend: aiter + load-balance-method: auto + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.8 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + enable-metrics: true + enable-cache-report: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + PYTHONUNBUFFERED: '1' + IBDEVICES: rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 + GLOO_SOCKET_IFNAME: eno0 + NCCL_SOCKET_IFNAME: eno0 + MORI_RDMA_TC: '104' + MORI_IO_TC: '104' + MORI_IO_SQ_BACKOFF_TIMEOUT_US: '50000' + MORI_IO_QP_MAX_SEND_WR: '16384' + MORI_IO_QP_MAX_CQE: '32768' + MORI_IO_QP_MAX_SGE: '2' + MORI_SHMEM_MODE: ISOLATION + SGLANG_USE_AITER: '1' + AITER_LOG_LEVEL: ERROR + SGLANG_MORI_DISPATCH_DTYPE: auto + SGLANG_MORI_QP_PER_TRANSFER: '4' + SGLANG_MORI_NUM_WORKERS: '4' + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '1024' + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: '32' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '3600' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '3600' + SGLANG_HEALTH_CHECK_TIMEOUT: '600' + args: + served-model-name: deepseek-ai/DeepSeek-R1-0528 + trust-remote-code: true + tp-size: 8 + disaggregation-transfer-backend: mori + kv-cache-dtype: fp8_e4m3 + attention-backend: aiter + load-balance-method: auto + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.85 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: false + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128] + enable-metrics: true + enable-cache-report: true + prefill-round-robin-balance: true + sbatch_directives: + cpus-per-task: '128' + mem: '0' + srun_options: + mem: '0' + container-writable: '' + container-remap-root: '' + health_check: + max_attempts: 720 + interval_seconds: 5 + benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP USE_CHAT_TEMPLATE BENCHMARK_VARIANT + result_dir=/logs/sa-bench_isl_8192_osl_1024 + mkdir -p "${result_dir}" + ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + gen=$((DECODE_NUM_WORKERS * DECODE_TP)) + chat_args=() + if [[ "${USE_CHAT_TEMPLATE}" == "1" ]]; then + chat_args+=(--use-chat-template) + fi + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-R1-0528 \ + --tokenizer deepseek-ai/DeepSeek-R1-0528 \ + --trust-remote-code \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + "${chat_args[@]}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_dir}" \ + --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + BENCHMARK_VARIANT: base + USE_CHAT_TEMPLATE: '0' +override_stp_2p1d_dep8: + name: mi355x-dsr1-fp8-stp-2p1d-dep8-fixed-seq + resources: {} + roles: + prefill: + nodes: 2 + workers: 2 + args: + moe-dense-tp-size: 1 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 2048 + chunked-prefill-size: 65536 + cuda-graph-bs: [1, 2, 3] + decode: + nodes: 1 + workers: 1 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '512' + args: + moe-dense-tp-size: 1 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + max-running-requests: 2048 + chunked-prefill-size: 65536 + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160] + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + benchmark: + env: + BENCHMARK_VARIANT: stp-2p1d-dep8 +override_stp_1p2d_tp8: + name: mi355x-dsr1-fp8-stp-1p2d-tp8-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + decode: + nodes: 2 + workers: 2 + benchmark: + env: + BENCHMARK_VARIANT: stp-1p2d-tp8 +override_stp_1p1d_tp4_tp8: + name: mi355x-dsr1-fp8-stp-1p1d-tp4-tp8-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + args: + tp-size: 4 + decode: + nodes: 1 + workers: 1 + gpus: 8 + benchmark: + env: + BENCHMARK_VARIANT: stp-1p1d-tp4-tp8 +override_mtp_2p1d_dep8: + name: mi355x-dsr1-fp8-mtp1-2p1d-dep8-fixed-seq + resources: {} + roles: + prefill: + nodes: 2 + workers: 2 + args: + moe-dense-tp-size: 1 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 2048 + chunked-prefill-size: 65536 + cuda-graph-bs: [1, 2, 3] + decode: + nodes: 1 + workers: 1 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' + SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD: '512' + args: + moe-dense-tp-size: 1 + ep-size: 8 + dp-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + ep-dispatch-algorithm: fake + moe-a2a-backend: mori + deepep-mode: normal + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-num-steps: 1 + speculative-num-draft-tokens: 2 + max-running-requests: 2048 + chunked-prefill-size: 65536 + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, + 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, + 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 158, 159, 160] + mem-fraction-static: 0.85 + disable-radix-cache: false + prefill-round-robin-balance: true + benchmark: + env: + BENCHMARK_VARIANT: mtp1-2p1d-dep8 + USE_CHAT_TEMPLATE: '1' +override_mtp_1p2d_tp8: + name: mi355x-dsr1-fp8-mtp2-1p2d-tp8-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + args: + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + decode: + nodes: 2 + workers: 2 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1536' + args: + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + benchmark: + env: + BENCHMARK_VARIANT: mtp2-1p2d-tp8 + USE_CHAT_TEMPLATE: '1' +override_mtp_1p1d_tp4_tp8: + name: mi355x-dsr1-fp8-mtp2-1p1d-tp4-tp8-fixed-seq + resources: {} + roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + args: + tp-size: 4 + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '1536' + args: + speculative-algorithm: NEXTN + speculative-eagle-topk: 1 + speculative-num-steps: 2 + speculative-num-draft-tokens: 3 + benchmark: + env: + BENCHMARK_VARIANT: mtp2-1p1d-tp4-tp8 + USE_CHAT_TEMPLATE: '1' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml new file mode 100644 index 0000000000..02175cd104 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml @@ -0,0 +1,154 @@ +# Production-scale MI355X port of the existing Qwen3.5 FP8 1P1D recipe. +# Each role owns one full 8-GPU node; the native SGLang Router provides the +# request plane and AMD MoRI moves KV directly between the P/D workers. + +schema: 2 +name: "mi355x-sglang-qwen3.5-fp8-disagg-1p1d-tp4p-tp8d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3.5-397B-A17B-FP8" + container: "lmsysorg/sglang:v0.5.16-rocm720-mi35x" + precision: "fp8" + +identity: + model: + repo: "Qwen/Qwen3.5-397B-A17B-FP8" + container: + image: "lmsysorg/sglang:v0.5.16-rocm720-mi35x" + frameworks: + sglang: "0.5.16" + sglang-router: "0.3.2" + +slurm: + time_limit: "08:00:00" + +# This topology owns every GPU on each MI355X node. Give SGLang the complete +# 128-core cpuset as well: its ROCm image enables per-GPU CPU affinity, which +# maps the eight TP ranks across the full dual-socket CPU topology. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 +frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &common_environment + HF_HOME: "/hf_hub_cache" + # Hugging Face stores hub snapshots under $HF_HOME/hub. Keep the explicit + # cache variables on that same path so srt-slurm's prefetch and every + # backend process resolve the identical, current snapshot. + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + args: &common_config + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + tensor-parallel-size: 4 + disaggregation-transfer-backend: mori + disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + kv-cache-dtype: "fp8_e4m3" + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8] + enable-metrics: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: *common_environment + args: + <<: *common_config + mem-fraction-static: 0.85 + prefill-round-robin-balance: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP + result_dir=/logs/sa-bench_isl_8192_osl_1024 + mkdir -p "${result_dir}" + ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + gen=$((DECODE_NUM_WORKERS * DECODE_TP)) + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if ((num_prompts < 16)); then + num_prompts=16 + fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3.5-397B-A17B-FP8 \ + --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_dir}" \ + --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml new file mode 100644 index 0000000000..8693f14fc1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml @@ -0,0 +1,148 @@ +# Production MI355X port of the legacy Qwen3.5 MXFP4 1P1D sweep. Each role +# owns one full 8-GPU node; SGLang Router carries requests and AMD MoRI moves KV. + +schema: 2 +name: "mi355x-sglang-qwen3.5-mxfp4-disagg-1p1d-tp8-fixed-seq" + +model: + path: "hf:amd/Qwen3.5-397B-A17B-MXFP4" + container: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + precision: "fp4" + +identity: + model: + repo: "amd/Qwen3.5-397B-A17B-MXFP4" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + +slurm: + time_limit: "08:00:00" + +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 +frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: &common_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + args: &common_config + served-model-name: "amd/Qwen3.5-397B-A17B-MXFP4" + tensor-parallel-size: 8 + disaggregation-transfer-backend: mori + disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + kv-cache-dtype: "fp8_e4m3" + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8] + enable-metrics: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: *common_environment + args: + <<: *common_config + mem-fraction-static: 0.85 + prefill-round-robin-balance: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST PREFILL_NUM_WORKERS PREFILL_TP DECODE_NUM_WORKERS DECODE_TP + result_dir=/logs/sa-bench_isl_8192_osl_1024 + mkdir -p "${result_dir}" + ctx=$((PREFILL_NUM_WORKERS * PREFILL_TP)) + gen=$((DECODE_NUM_WORKERS * DECODE_TP)) + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if ((num_prompts < 16)); then + num_prompts=16 + fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model amd/Qwen3.5-397B-A17B-MXFP4 \ + --tokenizer amd/Qwen3.5-397B-A17B-MXFP4 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_dir}" \ + --result-filename "results_concurrency_${concurrency}_gpus_$((ctx + gen))_ctx_${ctx}_gen_${gen}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a786bed1e5..51f3be4493 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -312,9 +312,10 @@ qwen3.5-fp8-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # 1P+1D TP4P+TP8D/EP1 baseline (no speculative decoding). - # TP4 prefill saves 4 GPUs vs TP8P while delivering identical decode - # interactivity and 24-31% better throughput/GPU (12 vs 16 GPUs). + # The srt-slurm recipe owns the complete c8-c128 sweep in one Slurm + # allocation so model initialization is paid once instead of per point. + # 1P+1D TP4P+TP8D/EP1 remains the current baseline: TP4 prefill saves + # four GPUs without changing the TP8 decode service. # dp-attn intentionally false: with --enable-dp-attention + # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size, # but is_deepep_class_backend() excludes MoRI, so @@ -322,14 +323,14 @@ qwen3.5-fp8-mi355x-sglang-disagg: # (num_experts - num_shared_slots) % moe_ep_size assertion in # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] + conc-list: [8, 16, 32, 64, 128] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml" decode: num-worker: 1 tp: 8 @@ -424,7 +425,7 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40] } qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 runner: cluster:mi355x-amds @@ -440,22 +441,20 @@ qwen3.5-fp4-mi355x-sglang-disagg: osl: 1024 search-space: - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] + # The recipe owns the complete production search in one allocation. + conc-list: [8, 16, 32, 64, 128, 256, 512] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp8-mxfp4-fixed-seq.yaml" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x @@ -531,7 +530,8 @@ dsr1-fp8-mi355x-atom-mtp: srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsr1/atom/mi355x-fp8-mtp/8k1k.yaml dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 + # Pure TP variants retain SGLang's dense-TP default; dense TP1 is DP-attention-only. + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: cluster:mi355x-amds @@ -546,67 +546,55 @@ dsr1-fp8-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # non-MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) + # Each matrix row selects one resolved topology from the shared + # srt-slurm recipe. The recipe owns the complete concurrency sweep in a + # single allocation so model initialization is paid once per topology. - spec-decoding: "none" - conc-list: [ 1024, 2048 ] + conc-list: [1024, 2048] prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml:override_stp_2p1d_dep8" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] + conc-list: [4, 8, 16, 32, 64, 128, 256] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml:override_stp_1p2d_tp8" decode: num-worker: 2 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] + conc-list: [1, 2, 4, 8, 16, 32, 64] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml:override_stp_1p1d_tp4_tp8" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 + # Pure TP variants retain SGLang's dense-TP default; dense TP1 is DP-attention-only. + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: cluster:mi355x-amds @@ -621,64 +609,48 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - isl: 8192 osl: 1024 search-space: - # MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] + conc-list: [1024, 2048] prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml:override_mtp_2p1d_dep8" decode: num-worker: 1 tp: 8 ep: 8 dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4, 2 ] + conc-list: [2, 4, 8, 16, 32, 64, 128, 256] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml:override_mtp_1p2d_tp8" decode: num-worker: 2 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] + conc-list: [1, 2, 4, 8, 16, 32, 64] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp8-disagg-fixed-seq.yaml:override_mtp_1p1d_tp4_tp8" decode: num-worker: 1 tp: 8 ep: 1 dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" kimik3-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-rocm100-af1c01499b289be555c475669ba50a88e96d846e @@ -755,7 +727,7 @@ minimaxm3-fp4-mi355x-atom-agentic-mtp: - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.5rc3+rocm7.2.4" }, conc-list: [40, 48], spec-decoding: mtp } dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: amd/DeepSeek-R1-0528-MXFP4-v2 model-prefix: dsr1 runner: cluster:mi355x-amds @@ -770,123 +742,59 @@ dsr1-fp4-mi355x-sglang-disagg: - isl: 8192 osl: 1024 search-space: - # non-MTP configurations - # 1P1D pure TP8 - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] + conc-list: [1, 2, 4, 8] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p1d_tp8" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] + conc-list: [2, 4, 8, 16, 32, 64, 128, 256] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p2d_tp8" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] + conc-list: [64, 128, 256] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP8 + 1*DEP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p2d_tp4_tp8" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] + conc-list: [128, 256, 512] prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 2*DEP8 + 1*DEP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_1p1d_dep8" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] + conc-list: [1024, 2048, 4096] prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_stp_2p1d_dep8" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: amd/DeepSeek-R1-0528-MXFP4-v2 model-prefix: dsr1 runner: cluster:mi355x-amds @@ -901,158 +809,46 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - isl: 8192 osl: 1024 search-space: - # MTP configurations - # 1P1D pure TP8 - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] + conc-list: [1, 2, 4, 8] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_tp8" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] + conc-list: [2, 4, 8, 16, 32, 64] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 32, 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 640, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p2d_tp8_wide" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" - conc-list: [ 64 ] + conc-list: [64, 128, 256, 512, 640] prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 2*DEP8 + 1*DEP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_dep8" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] + conc-list: [1024, 2048, 4096] prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_2p1d_dep8" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} dsr1-fp8-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm700-mi30x @@ -1132,7 +928,7 @@ dsv4-fp4-mi355x-atom-agentic-mtp: - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256] } dsr1-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 model: amd/DeepSeek-R1-0528-MXFP4-v2 model-prefix: dsr1 runner: cluster:mi355x-amds @@ -1147,120 +943,56 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: - isl: 8192 osl: 1024 search-space: - # MTP configurations - # 1P1D pure TP8 - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] + conc-list: [1, 2, 4, 8] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p1d_tp8" + decode: {num-worker: 1, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] + conc-list: [2, 4, 8, 16, 32] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp3_1p2d_tp8_narrow" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] + conc-list: [64, 128, 256] prefill: num-worker: 1 tp: 8 ep: 1 dp-attn: false additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP8 + 1*DEP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp2_1p2d_tp8" + decode: {num-worker: 2, tp: 8, ep: 1, dp-attn: false} - spec-decoding: "mtp" - conc-list: [ 128, 512 ] + conc-list: [64, 128, 256, 512] prefill: num-worker: 1 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 1*DEP8 + 1*DEP8 + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_1p1d_dep8" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} - spec-decoding: "mtp" - conc-list: [ 64, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] + conc-list: [1024, 2048, 4096] prefill: num-worker: 2 tp: 8 ep: 8 dp-attn: true additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" + - "CONFIG_FILE=recipes/sglang/dsr1/mi355x/fp4-disagg-fixed-seq.yaml:override_mtp1_2p1d_dep8" + decode: {num-worker: 1, tp: 8, ep: 8, dp-attn: true} minimaxm3-fp8-mi300x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.29.0 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 2395cb9288..c97c7beb55 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -52,9 +52,15 @@ if [[ "$EXECUTION_PATH" == multinode && -n "${CONFIG_FILE:-}" ]]; then make setup ARCH=x86_64 export INFMAX_WORKSPACE="$GITHUB_WORKSPACE" + SRT_EVAL_OVERRIDES=() + if [[ "$RUN_EVAL" == true || "$EVAL_ONLY" == true ]]; then + # Evals need real expert dispatch; throughput variants may use fake dispatch. + SRT_EVAL_OVERRIDES=(--unset roles.prefill.args.ep-dispatch-algorithm + --unset roles.decode.args.ep-dispatch-algorithm) + fi SRT_JOB_ID="" trap '[[ -n "$SRT_JOB_ID" ]] && slurm_job_is_active "$SRT_JOB_ID" && scancel "$SRT_JOB_ID"' EXIT - apply_srt_recipe "$CONFIG_FILE" "$FRAMEWORK" "${SRTCTL_EVAL_ARGS[@]}" \ + apply_srt_recipe "$CONFIG_FILE" "$FRAMEWORK" "${SRTCTL_EVAL_ARGS[@]}" "${SRT_EVAL_OVERRIDES[@]}" \ -f "$CONFIG_FILE" --json --yes > "$GITHUB_WORKSPACE/srt-submission.json" || { cat "$GITHUB_WORKSPACE/srt-submission.json" >&2 exit 1 @@ -159,26 +165,20 @@ if [[ "$IS_MULTINODE" == "true" ]]; then trap cleanup_and_save_logs EXIT fi - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" - if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]] || [[ "$FRAMEWORK" == "tilert" ]]; then - # Agentic recipes under multi_node/agentic/ export the HiCache tunables; - # fixed-seq-len recipes live at the multi_node/ root. - if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then - BENCHMARK_SUBDIR="multi_node/agentic" - else - BENCHMARK_SUBDIR="multi_node" - fi - else - BENCHMARK_SUBDIR="single_node/fixed_seq_len" + # Only AgentX recipes still use this path; fixed-sequence runs use srt-slurm. + if [[ "$IS_AGENTIC" != 1 ]]; then + echo "ERROR: MI355X multi-node fixed-sequence jobs require a CONFIG_FILE srt-slurm recipe" >&2 + exit 1 fi - JOB_ID=$(bash "benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME}") + SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" + JOB_ID=$(bash "benchmarks/multi_node/agentic/${SCRIPT_NAME}") # An empty JOB_ID means the recipe or submit.sh failed before sbatch. The # wait loop below would then poll for slurm_job-.out forever, because its # liveness guard degenerates to `grep -q ""` and matches any job this user # has queued. Fail here instead of burning the job's whole time limit. if [[ -z "${JOB_ID//[[:space:]]/}" ]]; then - echo "ERROR: benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME} returned no Slurm job id;" \ + echo "ERROR: benchmarks/multi_node/agentic/${SCRIPT_NAME} returned no Slurm job id;" \ "the recipe or submit.sh failed before sbatch (see its stderr above)" >&2 exit 1 fi @@ -212,43 +212,6 @@ if [[ "$IS_MULTINODE" == "true" ]]; then set -x - - - - if [[ "${EVAL_ONLY}" != "true" && "${IS_AGENTIC}" != "1" ]]; then - cat > collect_latest_results.py <<'PY' -import os, sys -job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] -logs_root = f"{job_dir}/logs/" -candidates = [] -if os.path.isdir(logs_root): - for name in os.listdir(logs_root): - subdir = f"{logs_root}{name}/{framework}_isl_{isl}_osl_{osl}" - if os.path.isdir(subdir): - candidates.append(subdir) -for path in sorted(candidates, key=os.path.getmtime, reverse=True)[:nexp]: - print(path) -PY - - LOGS_DIR=$(python3 collect_latest_results.py "$BENCHMARK_LOGS_DIR" "$ISL" "$OSL" 1 "$FRAMEWORK") - if [ -z "$LOGS_DIR" ]; then - echo "No logs directory found for ISL=${ISL}, OSL=${OSL}" - exit 1 - fi - - echo "Found logs directory: $LOGS_DIR" - ls -la "$LOGS_DIR" - - for result_file in $(find $LOGS_DIR -type f); do - file_name=$(basename $result_file) - if [ -f $result_file ]; then - WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${file_name}" - echo "Found result file ${result_file}. Copying it to ${WORKSPACE_RESULT_FILE}" - cp $result_file $WORKSPACE_RESULT_FILE - fi - done - fi - if [[ "${RUN_EVAL}" == "true" ]]; then EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR/logs" -type d -name eval_results 2>/dev/null | head -1) if [ -n "$EVAL_DIR" ] && [ -d "$EVAL_DIR" ]; then From 97a721a3228c4785cd6c51bb2b00044b1596b103 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 17:43:30 -0500 Subject: [PATCH 5/6] refactor(amd): remove legacy multi-node fixed-sequence orchestration Delete the retired MI355X disaggregated launch scripts, the amd_utils vLLM and ATOM engine paths, the fixed-sequence bench client, and the per-model fixed-sequence server settings. AgentX keeps its SGLang and TileRT paths. --- benchmarks/multi_node/amd_utils/env.sh | 77 +--- benchmarks/multi_node/amd_utils/job.slurm | 339 ++++-------------- benchmarks/multi_node/amd_utils/models.yaml | 282 --------------- benchmarks/multi_node/amd_utils/server.sh | 9 +- .../multi_node/amd_utils/server_sglang.sh | 10 +- .../multi_node/amd_utils/server_tilert.sh | 3 +- benchmarks/multi_node/amd_utils/setup_deps.sh | 53 +-- benchmarks/multi_node/amd_utils/submit.sh | 4 - .../multi_node/amd_utils/trace_replay.sh | 7 +- benchmarks/multi_node/llm-d/server.sh | 2 +- benchmarks/multi_node/runtime_settings.sh | 14 +- 11 files changed, 81 insertions(+), 719 deletions(-) diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh index 551a55ac2b..ba48879cb4 100755 --- a/benchmarks/multi_node/amd_utils/env.sh +++ b/benchmarks/multi_node/amd_utils/env.sh @@ -3,8 +3,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only check_env_vars ENGINE # MoRI-IO queue-pair tuning, the UCX RoCE GID index, SGLang router logging and the -# SGLang decode cuda-graph NCCL workaround. Only the SGLang and vLLM MoRI KV paths -# below read these. ENGINE=tilert moves KV over mooncake and starts no SGLang +# SGLang decode cuda-graph NCCL workaround. Only the SGLang MoRI KV path +# below reads these. ENGINE=tilert moves KV over mooncake and starts no SGLang # router, so it is neither given nor reads them: validating them there would force # the recipe to invent MoRI tuning for a transport it never uses. if [[ "$ENGINE" != "tilert" ]]; then @@ -15,7 +15,7 @@ if [[ "$ENGINE" != "tilert" ]]; then fi # Dual-engine environment setup for multi-node disaggregated serving. # -# ENGINE=sglang-disagg or vllm-disagg selects the engine-specific block. +# ENGINE=sglang-disagg or tilert selects the engine-specific block. # # REQUIRED ENVIRONMENT VARIABLES: # IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,... or mlx5_0,mlx5_1,...) @@ -119,76 +119,7 @@ else fi fi -if [[ "$ENGINE" == "vllm-disagg" ]]; then - export VLLM_USE_V1=1 - export VLLM_SERVER_DEV_MODE=0 - export VLLM_DISABLE_REQUEST_ID_RANDOMIZATION=1 - - set -x - - # UCX_NET_DEVICES: Use the first tw-eth interface for UCX TCP transport - if [[ -z "$UCX_NET_DEVICES" ]]; then - UCX_NET_DEV=$(ip -o link show 2>/dev/null | awk -F': ' '/tw-eth/{print $2}' | head -1) - if [[ -n "$UCX_NET_DEV" ]]; then - export UCX_NET_DEVICES="$UCX_NET_DEV" - else - FIRST_IB=$(echo "$IBDEVICES" | cut -d',' -f1) - if [[ -n "$FIRST_IB" ]]; then - export UCX_NET_DEVICES="${FIRST_IB}:1" - fi - fi - echo "[INFO] Auto-set UCX_NET_DEVICES=$UCX_NET_DEVICES" - else - echo "[INFO] Using UCX_NET_DEVICES=$UCX_NET_DEVICES (set by environment)" - fi - - # RoCEv2: use IPv4-mapped GID (index 1) for inter-node RDMA routing - export UCX_IB_GID_INDEX - - if [[ -n "$UCX_IB_TRAFFIC_CLASS" ]]; then - echo "[INFO] Using UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS (set by environment)" - elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null | awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - export UCX_IB_TRAFFIC_CLASS=$(( 4 * ND_DSCP )) - export UCX_IB_SL=$ND_PRIO - echo "[INFO] Detected QoS from nicctl: UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS, UCX_IB_SL=$UCX_IB_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - fi - fi - else - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - else - echo "[INFO] No nicctl and unable to detect from hostname. Skipping QoS configuration." - fi - fi - - set +x - echo "[INFO] IBDEVICES=$IBDEVICES UCX_NET_DEVICES=$UCX_NET_DEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}" - -elif [[ "$ENGINE" == "tilert" ]]; then +if [[ "$ENGINE" == "tilert" ]]; then echo "[INFO] tilert: IBDEVICES=$IBDEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME NCCL_IB_HCA=$NCCL_IB_HCA" else diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 0c779476ea..5a978ce435 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -9,7 +9,7 @@ # --output and --error are set by submit.sh via BENCHMARK_LOGS_DIR source "$(pwd)/../../benchmark_lib.sh" --validation-only -check_env_vars INFERENCEX_RUNTIME_ENV_VARS VLLM_ROUTER_IMAGE SKIP_RDMA_CHECK SKIP_GPU_SANITY +check_env_vars INFERENCEX_RUNTIME_ENV_VARS SKIP_RDMA_CHECK SKIP_GPU_SANITY check_env_vars \ ENGINE MODEL_NAME MODEL_DIR xP yD \ BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER BENCH_MAX_CONCURRENCY \ @@ -34,11 +34,7 @@ echo "" # Use $(pwd) not BASH_SOURCE — sbatch copies the script to /var/spool/slurmd/ # at runtime, but the CWD remains the submit-time directory (amd_utils/). -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODELS_YAML="$(pwd)/models_vllm.yaml" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - MODELS_YAML="$(pwd)/models_atom.yaml" -elif [[ "$ENGINE" == "tilert" ]]; then +if [[ "$ENGINE" == "tilert" ]]; then MODELS_YAML="$(pwd)/models_tilert.yaml" else MODELS_YAML="$(pwd)/models.yaml" @@ -99,123 +95,77 @@ export DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) export MODEL_DIR -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # vLLM: Extract hf_dir from models.yaml, search multiple paths, resolve HF cache snapshots - DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - DISK_DIR_NAME="${DISK_DIR_NAME:-$MODEL_NAME}" - echo "Looking for model: $MODEL_NAME (disk dir: $DISK_DIR_NAME)" - - resolve_hf_cache_path() { - local base_path=$1 - if [[ -d "${base_path}/snapshots" ]]; then - local snapshot=$(ls -1 "${base_path}/snapshots" 2>/dev/null | head -1) - if [[ -n "$snapshot" ]]; then - echo "${base_path}/snapshots/${snapshot}" - return 0 - fi - fi - echo "$base_path" - return 1 - } - - MODEL_PATH="" - SEARCH_PATHS=( - "${MODEL_DIR}/${DISK_DIR_NAME}" - "${MODEL_DIR}/${MODEL_NAME}" - "/nfsdata/hf_hub_cache-0/${DISK_DIR_NAME}" - "/nfsdata/hf_hub_cache-0/${MODEL_NAME}" - ) - - for search_path in "${SEARCH_PATHS[@]}"; do - if [[ -d "$search_path" ]]; then - RESOLVED=$(resolve_hf_cache_path "$search_path") - MODEL_PATH="$RESOLVED" - echo "Found MODEL_PATH: $MODEL_PATH" - break - fi - done - - if [[ -z "$MODEL_PATH" ]]; then - echo "FATAL: Model '$MODEL_NAME' not found. Searched:" - for p in "${SEARCH_PATHS[@]}"; do echo " - $p"; done - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -else - # SGLang: Validate model path across all allocated nodes - echo "Looking for model: $MODEL_NAME" - echo "Checking model availability across all allocated nodes..." - - ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") - TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) - echo "Total allocated nodes: $TOTAL_NODES" - echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" - - check_model_path() { - local path=$1 - local check_name=$2 - echo "Checking $check_name: $path" - srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " - if [ -d '$path' ]; then - echo \"\$(hostname): Found $path\" - exit 0 - else - echo \"\$(hostname): Missing $path\" - exit 1 - fi - " - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo "$check_name available on ALL nodes" - return 0 +# SGLang: Validate model path across all allocated nodes +echo "Looking for model: $MODEL_NAME" +echo "Checking model availability across all allocated nodes..." + +ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") +TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) +echo "Total allocated nodes: $TOTAL_NODES" +echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" + +check_model_path() { + local path=$1 + local check_name=$2 + echo "Checking $check_name: $path" + srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " + if [ -d '$path' ]; then + echo \"\$(hostname): Found $path\" + exit 0 else - echo "$check_name NOT available on all nodes" - return 1 - fi - } - - # Extract hf_dir from models.yaml (same as vllm-disagg path above) - SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" - - # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); - # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. - if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then - # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) - # Use it directly if it exists on all nodes, otherwise try subdirectory combos. - if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then - echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" - elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then - MODEL_PATH="$MODEL_PATH/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_PATH" - echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" - echo " - $MODEL_PATH/$MODEL_NAME" + echo \"\$(hostname): Missing $path\" exit 1 fi - elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" + " + local exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "$check_name available on ALL nodes" + return 0 + else + echo "$check_name NOT available on all nodes" + return 1 + fi +} + +# Extract hf_dir from models.yaml +SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} + found && /^[^ ]/{exit} + found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") +SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" + +# Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); +# fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. +if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then + # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) + # Use it directly if it exists on all nodes, otherwise try subdirectory combos. + if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then + echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" + elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then - MODEL_PATH="$MODEL_DIR/$MODEL_NAME" + elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then + MODEL_PATH="$MODEL_PATH/$MODEL_NAME" echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" else echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" - echo " - $MODEL_DIR/$MODEL_NAME" + echo " - $MODEL_PATH" + echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" + echo " - $MODEL_PATH/$MODEL_NAME" exit 1 fi - echo "Final MODEL_PATH: $MODEL_PATH" +elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then + MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" +elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then + MODEL_PATH="$MODEL_DIR/$MODEL_NAME" + echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" +else + echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" + echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" + echo " - $MODEL_DIR/$MODEL_NAME" + exit 1 fi +echo "Final MODEL_PATH: $MODEL_PATH" # ============================================================================= # Node Selection @@ -327,10 +277,6 @@ export EVAL_LIMIT="${EVAL_LIMIT:-}" SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" -# vLLM external router container. -# NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older -# dated tags are garbage-collected (manifest unknown) -ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" # Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" @@ -502,40 +448,7 @@ DOCKER_ENV_COMMON=( ) # Engine-specific env vars -if [[ "$ENGINE" == "vllm-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e VLLM_WS_PATH=${WS_PATH} - -e UCX_TLS=tcp,self,shm,rocm_ipc,rocm_copy,cma - -e UCX_SOCKADDR_TLS_PRIORITY=tcp - -e UCX_MEMTYPE_CACHE=y - -e UCX_RNDV_SCHEME=get_zcopy - -e UCX_RNDV_THRESH=4k - -e UCX_ROCM_IPC_MIN_ZCOPY=0 - -e UCX_LOG_LEVEL=warn - -e HSA_ENABLE_SDMA=1 - -e PROXY_STREAM_IDLE_TIMEOUT=\${PROXY_STREAM_IDLE_TIMEOUT} - -e PYTHONPYCACHEPREFIX=/tmp/pycache - ) -elif [[ "$ENGINE" == "atom-disagg" ]]; then - check_env_vars \ - PREFILL_PORT DECODE_PORT HANDSHAKE_PORT MEM_FRAC_STATIC KV_CACHE_DTYPE \ - BLOCK_SIZE MAX_NUM_SEQS - DOCKER_ENV_ENGINE=( - -e ATOM_WS_PATH=${WS_PATH} - -e PREFILL_PORT=${PREFILL_PORT} - -e DECODE_PORT=${DECODE_PORT} - -e ROUTER_PORT=${ROUTER_PORT} - -e HANDSHAKE_PORT=${HANDSHAKE_PORT} - -e MEM_FRAC_STATIC=${MEM_FRAC_STATIC} - -e KV_CACHE_DTYPE=${KV_CACHE_DTYPE} - -e BLOCK_SIZE=${BLOCK_SIZE} - -e MAX_NUM_SEQS=${MAX_NUM_SEQS} - -e MAX_MODEL_LEN=${MAX_MODEL_LEN:-} - -e MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-} - -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-} - -e IBDEVICES=${IBDEVICES:-} - ) -elif [[ "$ENGINE" == "tilert" ]]; then +if [[ "$ENGINE" == "tilert" ]]; then DOCKER_ENV_ENGINE=( -e MODEL_PATH=$DOCKER_MODEL_PATH -e PREFILL_IMAGE=${PREFILL_IMAGE} @@ -644,121 +557,13 @@ echo \"Rank \$SLURM_PROCID on \$(hostname)\" eval \"\$DOCKER_CMD_DETECT\" echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" -# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack) -RDMA_MOUNTS=() -if [[ "$ENGINE" == "atom-disagg" ]]; then - -# When the container base OS differs from the host (e.g. Ubuntu 24.04 image -# on a 22.04 host), the container's bundled libibverbs/libionic may be -# ABI-incompatible with the host kernel drivers. Detect the NIC type and -# bind-mount the host's out-of-tree RDMA userspace libraries into the -# container so the RDMA stack always matches the running kernel. -_detect_nic_type() { - if [[ -n \"\${MORI_NIC_TYPE:-}\" ]]; then echo \"\$MORI_NIC_TYPE\"; return; fi - local bnxt=0 mlx5=0 ionic=0 - if [[ -d /sys/class/infiniband ]]; then - for dev in /sys/class/infiniband/*; do - local name; name=\$(basename \"\$dev\") - case \"\$name\" in - bnxt_re*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; - *) - local drv; drv=\$(basename \"\$(readlink -f \"\$dev/device/driver\" 2>/dev/null)\" 2>/dev/null || true) - case \"\$drv\" in bnxt*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; esac ;; - esac - done - fi - if (( bnxt >= mlx5 && bnxt >= ionic && bnxt > 0 )); then echo bnxt - elif (( ionic >= mlx5 && ionic > 0 )); then echo ionic - else echo mlx5; fi -} - -_find_host_ibverbs() { - for c in /usr/lib64/libibverbs.so.1 /lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1.14.39.0 /usr/lib/x86_64-linux-gnu/libibverbs.so.1; do - local r; r=\$(readlink -f \"\$c\" 2>/dev/null || true) - [[ \"\$r\" == *libibverbs.so.1.14.57.0 ]] && continue - if [[ -f \"\$r\" ]]; then echo \"\$r\"; return; fi - done -} - -_NIC_TYPE=\$(_detect_nic_type) -echo \"[rdma] NIC type: \${_NIC_TYPE} on \$(hostname)\" - -if [[ \"\$_NIC_TYPE\" == \"ionic\" || \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - _host_ibv=\$(_find_host_ibverbs) - if [[ -n \"\$_host_ibv\" ]]; then - RDMA_MOUNTS+=(-v \"\$_host_ibv:/lib/x86_64-linux-gnu/libibverbs.so.1\") - fi -fi - -if [[ \"\$_NIC_TYPE\" == \"ionic\" ]]; then - for _dir in /usr/local/lib /usr/lib/x86_64-linux-gnu; do - for _lib in \"\$_dir\"/libionic*.so; do - [[ -f \"\$_lib\" ]] || continue - _real=\$(readlink -f \"\$_lib\") - [[ -f \"\$_real\" ]] && RDMA_MOUNTS+=(-v \"\$_real:\$_real\") - RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - done - if [[ -d /usr/lib/x86_64-linux-gnu/libibverbs ]]; then - for _lib in /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:\$_lib\") - done - fi - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -elif [[ \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - for _lib in /usr/local/lib/libbnxt_re-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/libibverbs/\$(basename \"\$_lib\")\") - done - for _lib in /usr/local/lib/libbnxt_re.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -fi - -if [[ \${#RDMA_MOUNTS[@]} -gt 0 ]]; then - echo \"[rdma] bind-mounts: \${RDMA_MOUNTS[*]}\" -else - echo \"[rdma] no out-of-tree RDMA mounts needed\" -fi -fi # end: if ENGINE == atom-disagg - -# Start vLLM external router container on node 0 -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true - \$DOCKER_CMD run -d \ - --name \"$ROUTER_CONT_NAME\" \ - --network host \ - --ulimit nofile=1048576:1048576 \ - -v /tmp:/run_logs \ - \"$VLLM_ROUTER_IMAGE\" \ - bash -lc \"mkdir -p /run_logs/slurm_job-${SLURM_JOB_ID} && exec vllm-router \ - --vllm-pd-disaggregation \ - --kv-connector moriio \ - --vllm-discovery-address 0.0.0.0:${PROXY_PING_PORT} \ - --port ${ROUTER_PORT} \ - --host 0.0.0.0 \ - --policy consistent_hash \ - --prefill-policy consistent_hash \ - --decode-policy consistent_hash \ - --log-level info 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" -fi - -# Skip exec on vllm-disagg rank 0 so we can stop the router after the main -# container exits. Without this, decode nodes block forever waiting for the -# router port to close (the router is a separate container). -MAYBE_EXEC=exec -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - MAYBE_EXEC= - set +e -fi - RANK_IMAGE= if [[ \"$ENGINE\" == \"tilert\" && \"\$SLURM_PROCID\" -lt \"$xP\" ]]; then RANK_IMAGE=\"$PREFILL_IMAGE\" echo \"[tilert] rank \$SLURM_PROCID is a prefill rank; using PREFILL_IMAGE=\$RANK_IMAGE\" fi -\$MAYBE_EXEC \$DOCKER_CMD run \ +exec \$DOCKER_CMD run \ --init \ --stop-timeout 10 \ --device /dev/dri \ @@ -793,7 +598,6 @@ fi -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ ${EXTRA_DOCKER_MOUNTS:-} \ ${CLIENT_DOCKER_MOUNTS} \ - \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ ${DOCKER_ENV_COMMON[*]} \ ${DOCKER_ENV_ENGINE[*]} \ ${CLIENT_DOCKER_ENV} \ @@ -804,24 +608,11 @@ fi mkdir -p /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"' '"$RUN_FILE_FULL"' 2>&1 | tee /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'/server_\$(hostname).log ' - -# Only reached when exec was skipped (vllm-disagg rank 0) -DOCKER_EXIT_CODE=\$? -echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-router...\" -\$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true -exit \$DOCKER_EXIT_CODE " SERVER_SRUN_RC=$? if [[ "${KEEP_CONTAINERS}" != "1" ]]; then srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' - - # Clean up vLLM external router container on node 0 - if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then - srun --nodes=1 --ntasks=1 --nodelist="$MASTER_NODE" bash -c ' - eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$ROUTER_CONT_NAME"' 2>/dev/null || true - ' - fi fi # /run_logs is backed by each compute node's local /tmp, so the node-0 copy diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml index e29dc5a01c..0b86cbeb09 100644 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ b/benchmarks/multi_node/amd_utils/models.yaml @@ -39,288 +39,6 @@ # chunked_prefill_size: int # cuda_graph_bs_range: str -DeepSeek-V3: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V3-0324: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-Preview: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: 16384 - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-v2: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode " - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 " - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - context_length: 9217 - max_total_tokens: 131072 - enable_two_batch_overlap: true - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-512" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - DeepSeek-V4-Pro-AgentX: &DeepSeek-V4-Pro-AgentX base_flags: "--enable-deepseek-v4-fp4-indexer --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --enforce-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --tokenizer-worker-num 8 --stream-interval 20 --log-level info --log-level-http error" # --enable-dp-lm-head is required by SGLang for DSpark under DP attention; it diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh index 08cb65d2a9..a3830b57aa 100755 --- a/benchmarks/multi_node/amd_utils/server.sh +++ b/benchmarks/multi_node/amd_utils/server.sh @@ -4,8 +4,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only # Multi-Engine Disaggregated Server Dispatcher # Dispatches to the engine-specific server launcher based on ENGINE env var. # ENGINE=sglang-disagg (default) -> server_sglang.sh (SGLang + MoRI) -# ENGINE=vllm-disagg -> server_vllm.sh (vLLM + Nixl/MoRI-IO) -# ENGINE=atom-disagg -> server_atom.sh (ATOM + mooncake) # ENGINE=tilert -> server_tilert.sh (vLLM prefill + TileRT decode) check_env_vars ENGINE WS_PATH @@ -18,12 +16,7 @@ export WS_PATH ENGINE echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" -if [[ "$ENGINE" == "vllm-disagg" ]]; then - source "$WS_PATH/server_vllm.sh" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - export ATOM_WS_PATH="$WS_PATH" - source "$WS_PATH/server_atom.sh" -elif [[ "$ENGINE" == "tilert" ]]; then +if [[ "$ENGINE" == "tilert" ]]; then source "$WS_PATH/server_tilert.sh" else source "$WS_PATH/server_sglang.sh" diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh index 39ab32be20..1d36d914d6 100755 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ b/benchmarks/multi_node/amd_utils/server_sglang.sh @@ -1104,14 +1104,8 @@ if [ "$NODE_RANK" -eq 0 ]; then $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" else - # bench.sh signature: - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path - # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - echo "Benchmark runner: bench.sh (fixed-seq-len)" + echo "ERROR: fixed-sequence runs use srt-slurm recipes, not amd_utils" >&2 + exit 1 fi IS_AGENTIC_RUN=0 diff --git a/benchmarks/multi_node/amd_utils/server_tilert.sh b/benchmarks/multi_node/amd_utils/server_tilert.sh index a22cd5c433..2efec9d8ce 100644 --- a/benchmarks/multi_node/amd_utils/server_tilert.sh +++ b/benchmarks/multi_node/amd_utils/server_tilert.sh @@ -45,8 +45,7 @@ export INFMAX_CONTAINER_WORKSPACE=/workspace source /workspace/benchmarks/benchmark_lib.sh # Model-specific engine environment (not caller configuration): the prefill -# vLLM env block lives with the model, exactly as models_atom.yaml carries the -# ATOM `env` string. Everything else is passed in by the recipe. +# vLLM env block lives with the model. Everything else is passed in by the recipe. MODELS_YAML="${WS_PATH}/models_tilert.yaml" eval "$("$PY" - "$MODELS_YAML" "$MODEL_NAME" <<'PYEOF' import shlex, sys, yaml diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh index 8adb438681..1a5c5cfacf 100644 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ b/benchmarks/multi_node/amd_utils/setup_deps.sh @@ -1,50 +1,12 @@ #!/bin/bash source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars ROCM_PATH UCX_HOME RIXL_HOME -# Install missing disagg dependencies at container start; sourced by server_vllm.sh -# and server_sglang.sh so PATH / LD_LIBRARY_PATH exports persist. Each installer is -# idempotent and gated on $ENGINE (vllm-disagg / sglang-disagg). +# Install missing disagg dependencies at container start. Each installer is +# idempotent and gated on $ENGINE. _SETUP_START=$(date +%s) _SETUP_INSTALLED=() -# ibv_devinfo (ibverbs-utils) and ip (iproute2) for in-container NIC/RDMA checks. -install_recipe_deps() { - if command -v ibv_devinfo >/dev/null 2>&1 && command -v ip >/dev/null 2>&1; then - echo "[SETUP] Container RDMA/net tools already present" - return 0 - fi - - echo "[SETUP] Installing ibv_devinfo + iproute2 in container..." - apt-get update -q -y && apt-get install -q -y \ - ibverbs-utils iproute2 \ - && rm -rf /var/lib/apt/lists/* - - if ! command -v ibv_devinfo >/dev/null 2>&1 || ! command -v ip >/dev/null 2>&1; then - echo "[SETUP] ERROR: Failed to install ibv_devinfo/iproute2"; exit 1 - fi - _SETUP_INSTALLED+=("ibverbs-utils+iproute2") -} - -# ROCm vLLM lacks the quark dependency needed for MXFP4 models: -# https://github.com/vllm-project/vllm/issues/35633 -install_amd_quark() { - if python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] amd-quark already present" - return 0 - fi - - echo "[SETUP] Installing amd-quark for MXFP4 quantization support..." - pip install --quiet amd-quark - - if ! python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] WARN: amd-quark install failed (non-fatal for non-MXFP4 models)" - return 0 - fi - _SETUP_INSTALLED+=("amd-quark") -} - # Pinned by the recipe (TILERT_VERSION); the rest are fixed properties of the # TileRT 0.1.x runtime rather than caller configuration. TILERT_PACKAGE=tilert @@ -155,16 +117,7 @@ install_tilert_prefill() { "$PY" -c "import tilert.pd_vllm.prefill_connector" 2>&1 | tail -3; } } -if [[ "$ENGINE" == "vllm-disagg" ]]; then - install_recipe_deps - install_amd_quark - - export ROCM_PATH - export UCX_HOME - export RIXL_HOME - export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" - export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" -elif [[ "$ENGINE" == "tilert" ]]; then +if [[ "$ENGINE" == "tilert" ]]; then check_env_vars TILERT_VERSION TILERT_PIP_SPEC="$TILERT_PACKAGE==$TILERT_VERSION" _tilert_resolve_python diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 428f8e0db4..c4a5ff8c8f 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -88,10 +88,6 @@ export MODEL_DIR=$MODEL_PATH export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE export PROFILER_ARGS=$profiler_args -if [[ "$ENGINE" == "vllm-disagg" ]]; then - check_env_vars PROXY_STREAM_IDLE_TIMEOUT - export PROXY_STREAM_IDLE_TIMEOUT -fi # xP = prefill workers, yD = decode workers (may span multiple nodes) export xP=$PREFILL_WORKERS export yD=$DECODE_WORKERS diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh index 42fa16ca3f..c1d5f76859 100644 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ b/benchmarks/multi_node/amd_utils/trace_replay.sh @@ -13,12 +13,7 @@ fi model_path=$1 model_name=$2 concurrency_list=${3} -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODEL="${MODEL_NAME}" -else - MODEL="${MODEL_PATH}" -fi +MODEL="${MODEL_PATH}" log_path=${4} IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index c5d4948d7a..2f5d271699 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -517,7 +517,7 @@ PY # Benchmark sweep. BENCH_MAX_CONCURRENCY is 'x'-delimited from submit.sh (e.g. "1024x512"). IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" # GPU counts are embedded in the result filename as _gpus_/_ctx_/_gen_ so the CI - # "Process result" step can parse them (same convention as amd_utils/bench.sh). + # "Process result" step can parse them. # ctx = prefill GPUs, gen = decode GPUs. _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) diff --git a/benchmarks/multi_node/runtime_settings.sh b/benchmarks/multi_node/runtime_settings.sh index 725efcdba6..f16ad9fdc2 100644 --- a/benchmarks/multi_node/runtime_settings.sh +++ b/benchmarks/multi_node/runtime_settings.sh @@ -9,8 +9,8 @@ export BENCH_NUM_PROMPTS_MULTIPLIER=10 DRY_RUN=0 KEEP_CONTAINERS=0 export AIPERF_DRAIN_TIMEOUT_SECONDS=1800 AIPERF_DRAIN_POLL_SECONDS=10 case "$FRAMEWORK" in - sglang-disagg|vllm-disagg|atom-disagg) - export VLLM_ROUTER_IMAGE=vllm/vllm-router:nightly-20260716-1fbcde7 SKIP_RDMA_CHECK=0 SKIP_GPU_SANITY=0 + sglang-disagg) + export SKIP_RDMA_CHECK=0 SKIP_GPU_SANITY=0 export ROUTER_TYPE=vllm-router ROUTER_PORT=30000 PROXY_PING_PORT=36367 export DECODE_MTP_SIZE=0 export HEADNODE_PORT=20000 SERVER_PORT=2584 PROXY_STREAM_IDLE_TIMEOUT=300 @@ -41,11 +41,6 @@ case "$FRAMEWORK" in export MORI_IO_SQ_BACKOFF_TIMEOUT_US=500000 MORI_IO_QP_MAX_SEND_WR=32768 fi fi - if [[ "$FRAMEWORK" == atom-disagg ]]; then - export PREFILL_PORT=8010 DECODE_PORT=8020 HANDSHAKE_PORT=6301 - export MEM_FRAC_STATIC=0.85 KV_CACHE_DTYPE=fp8 BLOCK_SIZE=16 MAX_NUM_SEQS=256 - export WAIT_SERVER_TIMEOUT=2500 WAIT_LOCAL_ROUTER_TIMEOUT=300 WAIT_REMOTE_ROUTER_TIMEOUT=2800 - fi ;; tilert) # RUNNER_TYPE selects the AMD block below, so a missing value must fail @@ -62,14 +57,11 @@ case "$FRAMEWORK" in fi # The MI355X TileRT recipe runs through the shared amd_utils chain # (submit.sh -> job.slurm -> server.sh -> setup_deps.sh), which validates - # the same orchestration inputs the AMD SGLang/vLLM/ATOM arms receive. + # the same orchestration inputs the AMD SGLang arm receives. # Without them submit.sh exits before sbatch and the launcher never gets # a job id. The B200 TileRT lane goes through srt-slurm and reads none of # these, so they are scoped to the AMD pool. if [[ "$RUNNER_TYPE" == *mi355x-amds* ]]; then - # Validated by job.slurm; only the vllm-disagg router branch reads - # VLLM_ROUTER_IMAGE, which ENGINE=tilert never enters. - export VLLM_ROUTER_IMAGE=vllm/vllm-router:nightly-20260716-1fbcde7 export SKIP_RDMA_CHECK=0 SKIP_GPU_SANITY=0 # The B200 profile above points BENCHMARK_LOGS_DIR at the workspace # itself; launch_mi355x-amds.sh's EXIT trap does `rm -rf From 41e8bfc092eea9f1f67efab9addf4013bc370f81 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 24 Sep 2026 15:41:34 -0500 Subject: [PATCH 6/6] fix(amd): serve Qwen3.5 FP8 disagg from the full MI355X checkpoint The cached Hub snapshot's config is text-only (qwen3_5_moe_text, no vision_config), so SGLang's Qwen-VL processor crashed at startup. The legacy path loaded /it-share/data/Qwen3.5-397B-A17B-FP8; register that as a model alias and use it in the recipe. --- .../sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml | 2 +- runners/srt-slurm/mi355x-amds.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml index 02175cd104..9f05cdddfa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml @@ -6,7 +6,7 @@ schema: 2 name: "mi355x-sglang-qwen3.5-fp8-disagg-1p1d-tp4p-tp8d-fixed-seq" model: - path: "hf:Qwen/Qwen3.5-397B-A17B-FP8" + path: "Qwen3.5-397B-A17B-FP8" container: "lmsysorg/sglang:v0.5.16-rocm720-mi35x" precision: "fp8" diff --git a/runners/srt-slurm/mi355x-amds.yaml b/runners/srt-slurm/mi355x-amds.yaml index b49b37548e..2746e8e08c 100644 --- a/runners/srt-slurm/mi355x-amds.yaml +++ b/runners/srt-slurm/mi355x-amds.yaml @@ -22,6 +22,8 @@ default_sbatch_directives: model_paths: DeepSeek-V4-Pro-0813: /it-share/data/DeepSeek-V4-Pro-0813 + # The Hub snapshot's config drops vision_config, which SGLang's Qwen-VL processor requires. + Qwen3.5-397B-A17B-FP8: /it-share/data/Qwen3.5-397B-A17B-FP8 default_mounts: /dev/kfd: /dev/kfd