From fd4d3a3156a9d998bca89ffef731e93acb363617 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 18 Sep 2026 17:37:12 -0500 Subject: [PATCH 1/6] feat(amd): refresh shared srt-slurm v2 integration on current InferenceX --- benchmarks/benchmark_lib.sh | 26 +- .../agentic/dsv4_fp4_mi355x_sglang-disagg.sh | 193 --- benchmarks/multi_node/agentic_srt.sh | 60 +- benchmarks/multi_node/amd_utils/bench.sh | 105 -- .../amd_utils/deprecated/glm5_setup.sh | 30 - .../amd_utils/deprecated/models.yaml | 69 - .../amd_utils/deprecated/models_atom.yaml | 57 - .../amd_utils/deprecated/models_vllm.yaml | 47 - .../amd_utils/deprecated/node_excludes.yaml | 14 - benchmarks/multi_node/amd_utils/env.sh | 299 ---- benchmarks/multi_node/amd_utils/env_atom.sh | 40 - .../amd_utils/helpers/rdma_check.sh | 146 -- benchmarks/multi_node/amd_utils/job.slurm | 797 --------- benchmarks/multi_node/amd_utils/models.yaml | 373 ---- .../multi_node/amd_utils/models_atom.yaml | 30 - .../multi_node/amd_utils/models_vllm.yaml | 25 - .../multi_node/amd_utils/node_excludes.yaml | 11 - .../multi_node/amd_utils/preflight_node.sh | 31 - benchmarks/multi_node/amd_utils/server.sh | 27 - .../multi_node/amd_utils/server_atom.sh | 532 ------ .../multi_node/amd_utils/server_sglang.sh | 1494 ----------------- .../multi_node/amd_utils/server_vllm.sh | 485 ------ benchmarks/multi_node/amd_utils/setup_deps.sh | 64 - benchmarks/multi_node/amd_utils/sglang_cli.py | 28 - .../multi_node/amd_utils/stage_node_logs.sh | 23 - benchmarks/multi_node/amd_utils/submit.sh | 306 ---- benchmarks/multi_node/amd_utils/sync.py | 220 --- .../multi_node/amd_utils/trace_replay.sh | 136 -- .../deprecated/dsv4_fp4_mi355x_atom-disagg.sh | 83 - .../dsv4_fp4_mi355x_sglang-disagg.sh | 83 - .../glm5_fp8_mi355x_sglang-disagg.sh | 84 - .../kimik2.5_fp4_mi355x_atom-disagg.sh | 78 - .../kimik2.5_fp4_mi355x_vllm-disagg.sh | 80 - .../minimaxm2.5_fp8_mi355x_vllm-disagg.sh | 78 - .../minimaxm3_fp4_mi355x_atom-disagg.sh | 78 - .../minimaxm3_fp4_mi355x_vllm-disagg.sh | 78 - .../minimaxm3_fp8_mi355x_atom-disagg.sh | 78 - .../minimaxm3_fp8_mi355x_vllm-disagg.sh | 83 - .../dsr1_fp4_mi355x_sglang-disagg.sh | 78 - .../dsr1_fp8_mi355x_sglang-disagg.sh | 78 - benchmarks/multi_node/llm-d/README.md | 3 +- benchmarks/multi_node/llm-d/server.sh | 2 +- benchmarks/multi_node/llm-d/submit.sh | 2 +- .../qwen3.5_fp4_mi355x_sglang-disagg.sh | 79 - .../qwen3.5_fp8_mi355x_sglang-disagg.sh | 79 - .../qwen3-0.6b/mi355x/agg-fixed-seq.yaml | 97 ++ .../mi355x/disagg-1p1d-fixed-seq.yaml | 111 ++ configs/amd-master.yaml | 821 +-------- configs/deprecated/amd-master.yaml | 1464 ---------------- docs/configuration-procedures.md | 12 +- infx/srt_slurm/synthetic_acceptance.py | 3 + infx/workflows/srt_slurm.py | 354 ++++ perf-changelog.yaml | 17 + runners/check_mi355x_rdma.sh | 54 + runners/launch_mi355x-amds.sh | 280 +-- runners/prepare_mi355x_srt_host.sh | 54 + runners/runtime_settings.sh | 6 + runners/slurm_utils.sh | 10 +- runners/srt-slurm/mi355x-amds.yaml | 46 + utils/test_amd_sglang_cli.py | 44 - utils/test_srt_slurm.py | 161 ++ utils/test_synthetic_acceptance.py | 6 +- 62 files changed, 1113 insertions(+), 9219 deletions(-) delete mode 100755 benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh delete mode 100755 benchmarks/multi_node/amd_utils/bench.sh delete mode 100644 benchmarks/multi_node/amd_utils/deprecated/glm5_setup.sh delete mode 100644 benchmarks/multi_node/amd_utils/deprecated/models.yaml delete mode 100644 benchmarks/multi_node/amd_utils/deprecated/models_atom.yaml delete mode 100644 benchmarks/multi_node/amd_utils/deprecated/models_vllm.yaml delete mode 100644 benchmarks/multi_node/amd_utils/deprecated/node_excludes.yaml delete mode 100755 benchmarks/multi_node/amd_utils/env.sh delete mode 100644 benchmarks/multi_node/amd_utils/env_atom.sh delete mode 100755 benchmarks/multi_node/amd_utils/helpers/rdma_check.sh delete mode 100755 benchmarks/multi_node/amd_utils/job.slurm delete mode 100644 benchmarks/multi_node/amd_utils/models.yaml delete mode 100644 benchmarks/multi_node/amd_utils/models_atom.yaml delete mode 100644 benchmarks/multi_node/amd_utils/models_vllm.yaml delete mode 100644 benchmarks/multi_node/amd_utils/node_excludes.yaml delete mode 100644 benchmarks/multi_node/amd_utils/preflight_node.sh delete mode 100755 benchmarks/multi_node/amd_utils/server.sh delete mode 100755 benchmarks/multi_node/amd_utils/server_atom.sh delete mode 100755 benchmarks/multi_node/amd_utils/server_sglang.sh delete mode 100755 benchmarks/multi_node/amd_utils/server_vllm.sh delete mode 100644 benchmarks/multi_node/amd_utils/setup_deps.sh delete mode 100644 benchmarks/multi_node/amd_utils/sglang_cli.py delete mode 100755 benchmarks/multi_node/amd_utils/stage_node_logs.sh delete mode 100755 benchmarks/multi_node/amd_utils/submit.sh delete mode 100755 benchmarks/multi_node/amd_utils/sync.py delete mode 100644 benchmarks/multi_node/amd_utils/trace_replay.sh delete mode 100644 benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_atom-disagg.sh delete mode 100755 benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_sglang-disagg.sh delete mode 100755 benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh delete mode 100644 benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh delete mode 100755 benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh delete mode 100644 benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh delete mode 100644 benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh delete mode 100755 benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh delete mode 100644 benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh delete mode 100644 benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh delete mode 100644 benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh delete mode 100644 benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh delete mode 100755 benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh delete mode 100755 benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml create mode 100644 infx/workflows/srt_slurm.py create mode 100644 runners/check_mi355x_rdma.sh create mode 100644 runners/prepare_mi355x_srt_host.sh create mode 100644 runners/srt-slurm/mi355x-amds.yaml delete mode 100644 utils/test_amd_sglang_cli.py create mode 100644 utils/test_srt_slurm.py diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 60bdfecead..80392b6f4f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -106,28 +106,6 @@ exit_after_background_process_cleanup() { exit "$final_status" } -# Finish preflight on every allocated node before any server container starts its -# peer-readiness deadline. A failed node prevents the entire serving step. -run_amd_multinode_after_preflight() { - local nodelist="$1" node_count="$2" preflight_script="$3" - local container_filter="$4" skip_gpu_sanity="$5" - shift 5 - local preflight_rc - if srun --nodelist="$nodelist" \ - --nodes="$node_count" --ntasks="$node_count" --ntasks-per-node=1 \ - --kill-on-bad-exit=1 --unbuffered \ - bash "$preflight_script" "$container_filter" "$skip_gpu_sanity"; then - echo "[preflight] all nodes ready; launching server containers" - else - preflight_rc=$? - echo "[preflight][ERROR] node preflight failed; no server containers launched" >&2 - return "$preflight_rc" - fi - srun --nodelist="$nodelist" \ - --nodes="$node_count" --ntasks="$node_count" --ntasks-per-node=1 \ - --kill-on-bad-exit=1 --signal=TERM@30 --unbuffered "$@" -} - # Launchers may load only input validation, without benchmark initialization. if [[ "${1-}" == "--validation-only" ]]; then return 0 @@ -2097,7 +2075,9 @@ run_lm_eval() { export INFERENCEX_LM_EVAL_RUNTIME_READY=true fi - local openai_server_base="http://0.0.0.0:${port}" + # A routed eval client may run on a different node from the API server. + local openai_server_host="${EVAL_SERVER_HOST:-0.0.0.0}" + local openai_server_base="http://${openai_server_host}:${port}" local openai_chat_base="${openai_server_base}/v1/chat/completions" export OPENAI_API_KEY=${OPENAI_API_KEY} MODEL_NAME=${MODEL_NAME:-$MODEL} # Prefer MODEL_NAME, else MODEL diff --git a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 0a78f97afb..0000000000 --- a/benchmarks/multi_node/agentic/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env bash - -# Agentic trace-replay recipe for a disaggregated SGLang server on MI355X -# (DeepSeek-V4-Pro FP4, 1P1D TP8). Driven by environment variables; submits a SLURM -# job via submit.sh. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../../benchmark_lib.sh" --validation-only - -check_env_vars \ - TIME_LIMIT MODEL_PREFIX PRECISION RESULT_FILENAME DURATION \ - MAX_MODEL_LEN DISABLE_CUSTOM_ALL_REDUCE KV_OFFLOADING MORI_IO_SQ_BACKOFF_TIMEOUT_US \ - MORI_IO_QP_MAX_SEND_WR PREFILL_ROUTER_POLICY ENABLE_METRICS DECODE_MTP_SIZE - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - DURATION \ - KV_OFFLOADING \ - IS_AGENTIC \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -export MODEL_PREFIX -export PRECISION -export RESULT_FILENAME - -export DURATION -export MAX_MODEL_LEN - -# --disable-custom-all-reduce avoids a known aiter fault on MI355X. -export DISABLE_CUSTOM_ALL_REDUCE - -# ── KV cache offloading ── -# KV_OFFLOADING=none | dram (passed from YAML). -# KV_OFFLOAD_BACKEND selects the backend when offloading is on: -# hicache GPU + CPU-DRAM host pool (HICACHE_TIER L2), optionally + a -# Mooncake L3 store (HICACHE_TIER L3). The tunables below. -# umbp-linker UMBP as a DIRECT external store for the unified radix tree, -# with NO host cache tier in between. A different sglang code -# path, not a variation of HiCache -- sglang rejects the two -# together -- so it reads NONE of the HICACHE_*/MC_* tunables -# and takes UMBP_* instead (block further down). Implemented in -# amd_utils/server_sglang.sh; prefill-side only, like HiCache -# on this path, and dp-attn: true only. -export KV_OFFLOADING -if [[ "$KV_OFFLOADING" != "none" ]]; then - check_env_vars KV_OFFLOAD_BACKEND -fi -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" ]]; then - check_env_vars \ - HICACHE_TIER HICACHE_HOST_POOL_COUNT HICACHE_PAGE_SIZE HICACHE_RATIO HICACHE_MEM_LAYOUT \ - HICACHE_IO_BACKEND HICACHE_WRITE_POLICY HICACHE_PREFETCH_POLICY MC_MASTER_PORT MC_METADATA_PORT \ - MC_METRICS_PORT MC_MASTER_THREADS MC_EVICTION_HIGH_WATERMARK MC_PROTOCOL \ - MC_GLOBAL_SEG - export HICACHE_TIER - export HICACHE_HOST_POOL_COUNT - # DSV4 uses page-size 256 (set in models.yaml); HiCache must match. - export HICACHE_PAGE_SIZE - export HICACHE_RATIO - # server_sglang.sh prefers --hicache-size over --hicache-ratio when TOTAL_CPU_DRAM_GB - # is set; opt out via FORCE_HICACHE_RATIO rather than unsetting TOTAL_CPU_DRAM_GB, - # which benchmark_lib.sh also requires client-side when KV_OFFLOADING=dram. - export FORCE_HICACHE_RATIO=1 - - if [[ "${HICACHE_TIER^^}" == "L3" ]]; then - export HICACHE_MEM_LAYOUT - export HICACHE_IO_BACKEND - export HICACHE_WRITE_POLICY - if [[ -z "${HICACHE_STORAGE_BACKEND:-}" ]]; then - export HICACHE_STORAGE_BACKEND=mooncake - fi - else - export HICACHE_MEM_LAYOUT - export HICACHE_IO_BACKEND - export HICACHE_WRITE_POLICY - export HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - fi - export HICACHE_PREFETCH_POLICY - # Shared nodes: use non-default Mooncake ports to avoid collisions. - export MC_MASTER_PORT - export MC_METADATA_PORT - export MC_METRICS_PORT - export MC_MASTER_THREADS - export MC_EVICTION_HIGH_WATERMARK - export MC_PROTOCOL - export MC_GLOBAL_SEG - export MC_DEVICE="${MC_DEVICE:-}" - export MC_MASTER_ADDR="${MC_MASTER_ADDR:-}" - export MC_METADATA_SERVER="${MC_METADATA_SERVER:-}" -fi - -# ── UMBP direct-linker tunables ── -# Only read when KV_OFFLOAD_BACKEND is a umbp-linker* arm. Defaults live in -# server_sglang.sh; these exports exist so the values are visible in the -# recipe (and in the commands dump) rather than buried, and so job.slurm has -# something to forward. -# UMBP_DRAM_BYTES NODE total for the tier, on the prefill node only. -# 1.5 TB matches the single-node linker arms, so a PD -# number can be read against them directly. Guarded in -# server_sglang.sh against half of the host's MemTotal. -# UMBP_MAX_TOTAL_TOKENS optional device KV pool cap. UNSET on purpose: the -# linker is compared against the HiCache control at an -# IDENTICAL profiled pool, not at a capped one. -# UMBP_SA_WAIT_SECONDS ceiling for each of the three server-readiness waits -# (socket -> data plane -> host memory registered for -# GPU access). A 1.5 TB tier can take many minutes to -# register on a node holding a lot of page cache. -if [[ "$KV_OFFLOADING" != "none" && "${KV_OFFLOAD_BACKEND:-}" == umbp-linker* ]]; then - export UMBP_DRAM_BYTES="${UMBP_DRAM_BYTES:-1500000000000}" - export UMBP_DRAM_USE_HUGEPAGES="${UMBP_DRAM_USE_HUGEPAGES:-0}" - export UMBP_SA_WAIT_SECONDS="${UMBP_SA_WAIT_SECONDS:-1800}" - export UMBP_SA_WAIT_REGISTERED="${UMBP_SA_WAIT_REGISTERED:-1}" - export MORI_UMBP_LOG_LEVEL="${MORI_UMBP_LOG_LEVEL:-info}" -fi - -# ── MoRIIO RDMA Send Queue tuning ── -export MORI_IO_SQ_BACKOFF_TIMEOUT_US -export MORI_IO_QP_MAX_SEND_WR - -export PREFILL_ROUTER_POLICY -export ENABLE_METRICS - -export DECODE_MTP_SIZE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch the job. CONC_LIST is space-delimited in YAML; submit.sh wants 'x'. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 338fcb966b..56520e5d77 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -7,7 +7,7 @@ set -x # against the already-ready frontend on the head node. source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" --validation-only -check_env_vars INFMAX_CONTAINER_WORKSPACE RESULT_DIR EVAL_ONLY AIPERF_DRAIN_TIMEOUT_SECONDS AIPERF_DRAIN_POLL_SECONDS +check_env_vars INFMAX_CONTAINER_WORKSPACE RESULT_DIR EVAL_ONLY FRAMEWORK AIPERF_DRAIN_TIMEOUT_SECONDS AIPERF_DRAIN_POLL_SECONDS source "$INFMAX_CONTAINER_WORKSPACE/benchmarks/benchmark_lib.sh" if [[ -n "${SRT_FRONTEND_HOST:-}" ]]; then @@ -62,6 +62,61 @@ if [[ "${EVAL_ONLY}" == "true" ]]; then _wait_for_openai_chat_route --port "$PORT" fi +# Preserve the legacy DP-attention replay contract. The SGLang router uses this +# header to keep every request in one AgentX correlation tree on a stable DP +# route, which is important for both session continuity and prefix-cache reuse. +if [[ "$FRAMEWORK" == sglang || "$FRAMEWORK" == sglang-disagg ]] && [[ "${PREFILL_DP_ATTN:-false}" == "true" ]]; then + export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true +fi + +# Reset every advertised SGLang worker before each concurrency point, matching +# the retired amd_utils trace replay. /flush_cache covers GPU radix + host +# HiCache; the storage-backend endpoint is best-effort because L3 is optional. +clear_agentic_worker_caches() { + check_env_vars FLUSH_DRAIN_TIMEOUT + local timeout_seconds="$FLUSH_DRAIN_TIMEOUT" + local metrics_csv="${AIPERF_SERVER_METRICS_URLS:-}" + if [[ -z "$metrics_csv" ]]; then + echo "[clear_caches] WARN: AIPERF_SERVER_METRICS_URLS unset; skipping cache flush" >&2 + return 0 + fi + + local -a metrics_urls + IFS=',' read -r -a metrics_urls <<< "$metrics_csv" + local metrics_url base_url start response code flushed + for metrics_url in "${metrics_urls[@]}"; do + [[ -n "$metrics_url" ]] || continue + base_url="${metrics_url%/metrics}" + start=$(date +%s) + flushed=0 + response="" + while :; do + response=$(curl -sf -m 10 -X POST "${base_url}/flush_cache" 2>/dev/null || true) + if grep -qi "Cache flushed" <<< "$response"; then + flushed=1 + break + fi + if (( $(date +%s) - start >= timeout_seconds )); then + break + fi + sleep 3 + done + if (( flushed )); then + echo "[clear_caches] ${base_url}: L1+L2 flushed" + else + echo "[clear_caches] WARN ${base_url}: L1+L2 flush not confirmed after ${timeout_seconds}s" >&2 + fi + + code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' \ + -X POST "${base_url}/hicache/storage-backend/clear" 2>/dev/null || true) + if [[ "$code" == "200" ]]; then + echo "[clear_caches] ${base_url}: L3 store cleared" + else + echo "[clear_caches] ${base_url}: L3 clear http=${code:-000} (optional backend unavailable)" + fi + done +} + wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS}" local poll_seconds="${AIPERF_DRAIN_POLL_SECONDS}" @@ -144,6 +199,9 @@ for index in "${!CONCURRENCIES[@]}"; do mkdir -p "$RESULT_DIR" echo "Running agentic concurrency $concurrency of: ${CONCURRENCIES[*]}" + if [[ "${CLEAR_CACHE_BETWEEN_CONC:-}" == "1" ]]; then + clear_agentic_worker_caches + fi build_replay_cmd "$RESULT_DIR" run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/multi_node/amd_utils/bench.sh b/benchmarks/multi_node/amd_utils/bench.sh deleted file mode 100755 index f889d26fb2..0000000000 --- a/benchmarks/multi_node/amd_utils/bench.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -# Disaggregated fixed-seq-len benchmark runner; writes JSON results via -# benchmark_serving.py for the CI pipeline. -# -# Usage: bash bench.sh \ -# \ -# - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars ENGINE MODEL_PATH MODEL_NAME ROUTER_PORT -if [[ $# -ne 13 ]]; then - echo "Error: bench.sh requires 13 positional arguments" >&2 - exit 1 -fi - -n_prefill=$1 -n_decode=$2 -prefill_gpus=$3 -decode_gpus=$4 -model_path=$5 -model_name=$6 -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - BENCH_MODEL="${MODEL_NAME}" -else - BENCH_MODEL="${MODEL_PATH}" -fi -log_path=$7 - -chosen_isl=${8} -chosen_osl=${9} -concurrency_list=${10} -chosen_req_rate=${11} -random_range_ratio=${12} -num_prompts_multiplier=${13} - -IFS='x' read -r -a chosen_concurrencies <<< "$concurrency_list" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -echo "Config ${chosen_isl}; ${chosen_osl}; ${chosen_concurrencies[0]}; ${chosen_req_rate}" - -profile_folder="${log_path}/${ENGINE}_isl_${chosen_isl}_osl_${chosen_osl}" -mkdir -p "$profile_folder" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" - -for max_concurrency in "${chosen_concurrencies[@]}"; do - - export_file="${profile_folder}/concurrency_${max_concurrency}_req_rate_${chosen_req_rate}_gpus_$((prefill_gpus+decode_gpus))_ctx_${prefill_gpus}_gen_${decode_gpus}" - - num_prompts=$(( max_concurrency * num_prompts_multiplier )) - if [[ "$num_prompts" -lt 16 ]]; then - num_prompts=16 - fi - - echo "profile_folder: $profile_folder" - echo "max_concurrency: $max_concurrency" - echo "chosen_req_rate: $chosen_req_rate" - echo "MODEL_PATH: $MODEL_PATH" - echo "ROUTER_PORT: $ROUTER_PORT" - echo "chosen_isl: $chosen_isl" - echo "chosen_osl: $chosen_osl" - echo "num_prompts: $num_prompts" - echo "export_file: $export_file" - - extra_flags="" - if [[ "$ENGINE" == "vllm-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - elif [[ "$ENGINE" == "atom-disagg" ]]; then - extra_flags="--trust-remote-code --tokenizer $MODEL_PATH" - if [ "$IS_MTP" = "true" ]; then - # just override extra_flags as dsv3 use different tokenizer path - extra_flags="--use-chat-template" - fi - else - if [ "$IS_MTP" = "true" ]; then - extra_flags="--use-chat-template" - fi - fi - - run_benchmark_serving \ - --bench-serving-dir "$REPO_ROOT" \ - --model "$BENCH_MODEL" \ - --port "$ROUTER_PORT" \ - --backend openai \ - --input-len "$chosen_isl" \ - --output-len "$chosen_osl" \ - --random-range-ratio "$random_range_ratio" \ - --num-prompts "$num_prompts" \ - --max-concurrency "$max_concurrency" \ - --result-filename "$export_file" \ - --result-dir /workspace/ \ - $extra_flags - - echo "-----------------------------------------" - - if [[ "$ENGINE" == "vllm-disagg" ]]; then - echo "[BENCH] Cooldown: waiting 10s for idle KV block reaper..." - sleep 10 - fi -done diff --git a/benchmarks/multi_node/amd_utils/deprecated/glm5_setup.sh b/benchmarks/multi_node/amd_utils/deprecated/glm5_setup.sh deleted file mode 100644 index 94b05edff0..0000000000 --- a/benchmarks/multi_node/amd_utils/deprecated/glm5_setup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Historical GLM-5 setup, retained for explicit replay only. -# Source after the shared setup helper when restoring the archived GLM-5 registry. -# GLM-5 needs a transformers build with the glm_moe_dsa model type, which the mori -# images do not ship. Gated on any GLM model name. -install_transformers_glm5() { - if [[ "$MODEL_NAME" != *GLM* ]]; then - return 0 - fi - - if python3 -c "from transformers import AutoConfig; AutoConfig.from_pretrained('zai-org/GLM-5-FP8', trust_remote_code=True)" 2>/dev/null; then - echo "[SETUP] transformers already supports GLM-5 model type" - return 0 - fi - - echo "[SETUP] Installing transformers with GLM-5 (glm_moe_dsa) support..." - pip install --quiet -U --no-cache-dir \ - "git+https://github.com/huggingface/transformers.git@6ed9ee36f608fd145168377345bfc4a5de12e1e2" - _SETUP_INSTALLED+=("transformers-glm5") -} - -install_transformers_glm5 - -# Historical GLM-5 environment overrides. -if [[ "$MODEL_NAME" == "GLM-5-FP8" ]]; then - export SGLANG_ROCM_FUSED_DECODE_MLA=0 - export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - export SAFETENSORS_FAST_GPU=1 -fi diff --git a/benchmarks/multi_node/amd_utils/deprecated/models.yaml b/benchmarks/multi_node/amd_utils/deprecated/models.yaml deleted file mode 100644 index d6dfd0d39c..0000000000 --- a/benchmarks/multi_node/amd_utils/deprecated/models.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# Historical server registry entries; excluded from active server lookup. - -GLM-5-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --disaggregation-transfer-backend mori --tool-call-parser glm47 --reasoning-parser glm45 --model-loader-extra-config '{\\\"enable_multithread_load\\\": true, \\\"num_threads\\\": 8}'" - mtp_flags: "" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - - -DeepSeek-V4-Pro-DI: - base_flags: "--decode-log-interval 100 --log-level info --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --disable-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori" - # DeepSeek-V4 built-in MTP: load via the EAGLE spec path with eagle-topk 1 (a single - # MTP chain). NOT NEXTN -- the V3/R1 NEXTN loader crashes the dsv4 decode server at - # init on the V4 architecture. Mirrors the sglang dsv4 recipes - # (srt-slurm-recipes/dsv4/sglang/*: EAGLE + eagle-topk 1, in-checkpoint draft, - # no draft-model-path). Consumed by build_server_config only when DECODE_MTP_SIZE > 0 - # (spec-decoding: mtp); a no-op for the spec-none base scenario (DECODE_MTP_SIZE=0). - # build_server_config appends --speculative-num-steps/--speculative-num-draft-tokens - # from DECODE_MTP_SIZE. - mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - disable_cuda_graph: true - dp: - max_running_requests: 1024 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 2097152 - no_dp: - max_running_requests: 128 - chunked_prefill_size: 131072 - context_length: 9217 - max_total_tokens: 2097152 - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 1024 - cuda_graph_bs_range: "1-128" - no_dp: - max_running_requests: 128 - cuda_graph_bs_range: "1-128" diff --git a/benchmarks/multi_node/amd_utils/deprecated/models_atom.yaml b/benchmarks/multi_node/amd_utils/deprecated/models_atom.yaml deleted file mode 100644 index adaaa090eb..0000000000 --- a/benchmarks/multi_node/amd_utils/deprecated/models_atom.yaml +++ /dev/null @@ -1,57 +0,0 @@ -# Historical server registry entries; excluded from active server lookup. - -Kimi-K2.5-MXFP4: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 AITER_MXFP4_INTERMEDIATE=1" - kv_cache_flags: "--kv_cache_dtype fp8" - mem_frac_static: "0.9" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" - -DeepSeek-V4-Pro: - env: "ATOM_MOE_GU_ITLV=1 AITER_BF16_FP8_MOE_BOUND=0" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention --enable-tbo" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo" - decode_tp_dp_flags: "--enable-dp-attention --enable-tbo" - tp_dp_env: "GPU_MAX_HW_QUEUES=5 ATOM_CPU_AFFINITY=1" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method mtp --num-speculative-tokens" - -MiniMax-M3-MXFP4: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill" - decode_tp_dp_flags: "--enable-dp-attention" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens" - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - block_size: "128" - mem_frac_static: "0.8" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" - -MiniMax-M3-MXFP8: - env: "AITER_QUICK_REDUCE_QUANTIZATION=INT4 ATOM_FORCE_ATTN_TRITON=1" - kv_cache_flags: "--kv_cache_dtype fp8" - tp_dp_flags: "--enable-dp-attention" - prefill_tp_dp_flags: "--enable-dp-attention --enable-tbo prefill" - decode_tp_dp_flags: "--enable-dp-attention" - ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - prefill_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - decode_ep_dp_flags: "--enable-expert-parallel --enable-dp-attention" - mtp_flags: "--method eagle3 --draft-model Inferact/MiniMax-M3-EAGLE3 --num-speculative-tokens" - online_quant_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - online_quant_dpa_config: '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*.gate.*","*.block_sparse_moe.experts*"]}' - block_size: "128" - mem_frac_static: "0.8" - max_model_len: "32768" - max_num_seqs: "256" - max_num_batched_tokens: "32768" diff --git a/benchmarks/multi_node/amd_utils/deprecated/models_vllm.yaml b/benchmarks/multi_node/amd_utils/deprecated/models_vllm.yaml deleted file mode 100644 index 4ee11e7e77..0000000000 --- a/benchmarks/multi_node/amd_utils/deprecated/models_vllm.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Historical server registry entries; excluded from active server lookup. - -Kimi-K2.5-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" - decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" - hf_dir: "models--amd--Kimi-K2.5-MXFP4" - - -MiniMax-M2.5: - # AITER fused-MoE kernel fmoe_bf16_blockscaleFp8_g1u1_vs_silu_32x384 for gfx950 writes OOB when run with MiniMax's shapes at M=8K(=num batched tokens), crashing vllm during AITER warmup. - # Set token budget to 4k to avoid using that shape, instead of disabling AITER_MOE. - prefill_flags: "--max-num-batched-tokens 4K --tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --no-enable-prefix-caching --gpu-memory-utilization 0.95 --block-size 32" - decode_flags: "--max-num-batched-tokens 4K --tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --no-enable-prefix-caching --gpu-memory-utilization 0.95 --block-size 32" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ENGINE_READY_TIMEOUT_S=3600 VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1" - hf_dir: "models--MiniMaxAI--MiniMax-M2.5" - - -gpt-oss-120b: - prefill_flags: "--tensor-parallel-size 8" - decode_flags: "--tensor-parallel-size 8" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_TRITON_BF16_GEMM=0 VLLM_USE_AITER_UNIFIED_ATTENTION=1 VLLM_ROCM_USE_AITER_MHA=0 ROCM_TRITON_MOE_PRESHUFFLE_SCALES=0" - -MiniMax-M3-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - decode_flags: "--tensor-parallel-size 8 --max-num-batched-tokens 32768 --max-num-seqs 512 --block-size 128 --language-model-only --attention-backend TRITON_ATTN --moe-backend aiter --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" - prefill_env: "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048" - hf_dir: "models--amd--MiniMax-M3-MXFP4" - -MiniMax-M3-MXFP8: - # MiniMax-M3 MXFP8 disagg, no EP. The --tensor-parallel-size 8 below is just a - # placeholder: server_vllm.sh sed-rewrites it to PREFILL_TP_SIZE/DECODE_TP_SIZE - # from the master-config prefill/decode tp (the sweep uses TP4 workers only, - # varying the prefill:decode worker ratio rather than TP). - # --block-size 128 is mandatory (MSA sparse/index cache); text-only benchmark - # so --language-model-only frees the vision encoder. gfx950 uses FP8 KV cache. - # Serve flags + env kept in sync with the single-node recipe - # benchmarks/single_node/fixed_seq_len/deprecated/minimaxm3_fp8_mi355x.sh: - # --moe-backend aiter (AITER fused MoE), --linear-backend emulation, and the - # larger --max-num-batched-tokens 32768 prefill budget; plus the AITER - # router-append shared-experts fusion (self-disables under EP; the sweep is - # EP1 so it stays active) and INT6 quick all-reduce quantization. - prefill_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - decode_flags: "--tensor-parallel-size 8 --block-size 128 --language-model-only --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --moe-backend aiter --linear-backend emulation --max-num-batched-tokens 32768 --no-enable-prefix-caching --gpu-memory-utilization 0.90 --tool-call-parser minimax_m3 --reasoning-parser minimax_m3 --enable-auto-tool-choice" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT6 VLLM_USE_BREAKABLE_CUDAGRAPH=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" - hf_dir: "models--MiniMaxAI--MiniMax-M3-MXFP8" diff --git a/benchmarks/multi_node/amd_utils/deprecated/node_excludes.yaml b/benchmarks/multi_node/amd_utils/deprecated/node_excludes.yaml deleted file mode 100644 index 6ba840328e..0000000000 --- a/benchmarks/multi_node/amd_utils/deprecated/node_excludes.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Per (framework, model) SLURM node exclusions, consumed by submit.sh. -# -# A rule applies when FRAMEWORK matches "framework" exactly and MODEL_NAME is -# in "models"; its exclude_nodes are then passed to sbatch --exclude. Any -# (framework, model) combination with no matching rule gets an empty exclude -# list (i.e. no nodes excluded). -# -# SLURM_EXCLUDE_NODES env var, if set, always overrides this file. - -rules: - - framework: vllm-disagg - models: - - Kimi-K2.5-MXFP4 - exclude_nodes: mia1-p01-g09,mia1-p01-g10,mia1-p01-g11,mia1-p01-g12 diff --git a/benchmarks/multi_node/amd_utils/env.sh b/benchmarks/multi_node/amd_utils/env.sh deleted file mode 100755 index dbe93fa88d..0000000000 --- a/benchmarks/multi_node/amd_utils/env.sh +++ /dev/null @@ -1,299 +0,0 @@ -#!/bin/bash - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars \ - MORI_IO_SQ_BACKOFF_TIMEOUT_US MORI_IO_QP_MAX_SEND_WR MORI_IO_QP_MAX_CQE MORI_IO_QP_MAX_SGE MORI_IO_TC_DISABLE \ - UCX_IB_GID_INDEX MORI_APP_LOG_LEVEL SGLANG_ROUTER_STDOUT_LOGS TORCH_NCCL_BLOCKING_WAIT NCCL_BLOCKING_WAIT \ - SGLANG_OPT_USE_AITER_INDEXER -# Dual-engine environment setup for multi-node disaggregated serving. -# -# ENGINE=sglang-disagg or vllm-disagg selects the engine-specific block. -# -# REQUIRED ENVIRONMENT VARIABLES: -# IBDEVICES - RDMA/InfiniBand device names (e.g., ionic_0,ionic_1,... or mlx5_0,mlx5_1,...) -# Set by runner or auto-detected from hostname. -set -x - -check_env_vars ENGINE -export PYTHONDONTWRITEBYTECODE=1 - -# job.slurm writes the recipe's HiCache/Mooncake tunables to hicache_mc_.env and -# mounts it at /config/hicache_mc.env. Source it (auto-export) so values like -# HICACHE_PAGE_SIZE=256 reach the container before server_sglang.sh validates them. -if [[ -f /config/hicache_mc.env ]]; then - set -a - source /config/hicache_mc.env - set +a - echo "[env.sh] sourced HiCache config from /config/hicache_mc.env (HICACHE_PAGE_SIZE=${HICACHE_PAGE_SIZE:-unset})" -fi - -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - echo "ERROR: Unable to detect RDMA devices. Set IBDEVICES explicitly." >&2 - exit 1 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - -if [[ -z "$GLOO_SOCKET_IFNAME" ]]; then - export GLOO_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi -if [[ -z "$NCCL_SOCKET_IFNAME" ]]; then - export NCCL_SOCKET_IFNAME=$(ip route 2>/dev/null | grep '^default' | awk '{print $5}' | head -n 1) -fi - -set +x - -export NCCL_IB_HCA=${NCCL_IB_HCA:-$IBDEVICES} - -# MoRI settings shared by the vLLM MoRIIOConnector and the SGLang/MoRI KV-transfer path. - -export MORI_IO_SQ_BACKOFF_TIMEOUT_US -export MORI_IO_QP_MAX_SEND_WR -export MORI_IO_QP_MAX_CQE -export MORI_IO_QP_MAX_SGE -export MORI_IO_TC_DISABLE - -if [[ -n "$MORI_RDMA_TC" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC (set by runner or environment)" -elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null| awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - TC=$(( 4 * ND_DSCP )) - export MORI_RDMA_SL=$ND_PRIO - export MORI_IO_SL=$ND_PRIO - export MORI_RDMA_TC=$TC - export MORI_IO_TC=$TC - echo "[INFO] Detected QoS config from nicctl: MORI_RDMA_TC=$MORI_RDMA_TC, MORI_RDMA_SL=$MORI_RDMA_SL, MORI_IO_TC=$MORI_IO_TC, MORI_IO_SL=$MORI_IO_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] Unable to detect MORI_RDMA_TC from hostname. Skipping RDMA QoS configuration." - fi - fi -else - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export MORI_RDMA_TC=96 - export MORI_IO_TC=96 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export MORI_RDMA_TC=104 - export MORI_IO_TC=104 - echo "[INFO] Auto-detected MORI_RDMA_TC=$MORI_RDMA_TC from hostname $NODENAME" - else - echo "[INFO] nicctl not found and unable to detect from hostname. Skipping RDMA QoS configuration." - echo " This is normal for clusters without QoS or outside Docker containers." - fi -fi - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - export VLLM_USE_V1=1 - export VLLM_SERVER_DEV_MODE=0 - export VLLM_DISABLE_REQUEST_ID_RANDOMIZATION=1 - - set -x - - # UCX_NET_DEVICES: Use the first tw-eth interface for UCX TCP transport - if [[ -z "$UCX_NET_DEVICES" ]]; then - UCX_NET_DEV=$(ip -o link show 2>/dev/null | awk -F': ' '/tw-eth/{print $2}' | head -1) - if [[ -n "$UCX_NET_DEV" ]]; then - export UCX_NET_DEVICES="$UCX_NET_DEV" - else - FIRST_IB=$(echo "$IBDEVICES" | cut -d',' -f1) - if [[ -n "$FIRST_IB" ]]; then - export UCX_NET_DEVICES="${FIRST_IB}:1" - fi - fi - echo "[INFO] Auto-set UCX_NET_DEVICES=$UCX_NET_DEVICES" - else - echo "[INFO] Using UCX_NET_DEVICES=$UCX_NET_DEVICES (set by environment)" - fi - - # RoCEv2: use IPv4-mapped GID (index 1) for inter-node RDMA routing - export UCX_IB_GID_INDEX - - if [[ -n "$UCX_IB_TRAFFIC_CLASS" ]]; then - echo "[INFO] Using UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS (set by environment)" - elif command -v nicctl &> /dev/null; then - ND_PRIO=$(nicctl show qos 2>/dev/null | awk '/PFC no-drop priorities/ {print $NF; exit}') - ND_DSCP=$(nicctl show qos 2>/dev/null | awk -v p="$ND_PRIO" ' -$1 == "DSCP" && $2 == ":" && $NF == p { - print $3; exit -}') - # nicctl may emit trailing commas (e.g. "24,"); keep the leading integer so the - # arithmetic can't choke and unparseable output falls back to hostname detection. - ND_PRIO="${ND_PRIO%%,*}"; ND_PRIO="${ND_PRIO//[!0-9]/}" - ND_DSCP="${ND_DSCP%%,*}"; ND_DSCP="${ND_DSCP//[!0-9]/}" - if [[ "$ND_DSCP" =~ ^[0-9]+$ ]] && [[ "$ND_PRIO" =~ ^[0-9]+$ ]]; then - export UCX_IB_TRAFFIC_CLASS=$(( 4 * ND_DSCP )) - export UCX_IB_SL=$ND_PRIO - echo "[INFO] Detected QoS from nicctl: UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS, UCX_IB_SL=$UCX_IB_SL" - else - echo "[WARN] nicctl available but QoS data unavailable; trying hostname detection." - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - fi - fi - else - NODENAME=$(hostname -s) - if [[ $NODENAME == GPU* ]] || [[ $NODENAME == smci355-ccs-aus* ]]; then - export UCX_IB_TRAFFIC_CLASS=96 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - elif [[ $NODENAME == mia1* ]]; then - export UCX_IB_TRAFFIC_CLASS=104 - echo "[INFO] Auto-detected UCX_IB_TRAFFIC_CLASS=$UCX_IB_TRAFFIC_CLASS from hostname $NODENAME" - else - echo "[INFO] No nicctl and unable to detect from hostname. Skipping QoS configuration." - fi - fi - - set +x - echo "[INFO] IBDEVICES=$IBDEVICES UCX_NET_DEVICES=$UCX_NET_DEVICES NCCL_SOCKET_IFNAME=$NCCL_SOCKET_IFNAME UCX_IB_GID_INDEX=$UCX_IB_GID_INDEX UCX_IB_TRAFFIC_CLASS=${UCX_IB_TRAFFIC_CLASS:-unset}" - -else - - export SGLANG_USE_AITER=1 - export AITER_LOG_LEVEL=ERROR - - export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL="" - export MORI_COMBINE_DTYPE_DECODE="" - export SGLANG_MORI_QP_PER_TRANSFER=4 - export SGLANG_MORI_NUM_WORKERS=4 - - export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=3600 - export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=3600 - - export SGLANG_HEALTH_CHECK_TIMEOUT=600 - - # Disable allocating memory in one pass - export MORI_SHMEM_MODE=ISOLATION - - export SGLANG_ENABLE_SPEC_V2=1 - export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=0 - - export SGLANG_LOG_MS=true - export SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS=32 - - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=512 - - export MORI_MOE_MAX_INPUT_TOKENS_PREFILL=32768 - export MORI_MOE_MAX_INPUT_TOKENS_DECODE=2703 - - # set MTP size=1 when EP16 - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - - export MORI_EP_LAUNCH_CONFIG_MODE=AUTO - - # Default to WARNING to cut per-op MoRI log spam on long multinode/eval - # runs; override with MORI_APP_LOG_LEVEL=INFO when debugging. - export MORI_APP_LOG_LEVEL - - # SGLANG_ROUTER_STDOUT_LOGS=1 mirrors router logs to stdout via tee; 0 keeps the - # noisy per-request access logs in the file only. - export SGLANG_ROUTER_STDOUT_LOGS - - # Workaround for the 0305 upstream image. - export PYTHONPATH=/sgl-workspace/aiter:${PYTHONPATH} - - # ROCm <= 7.2.0's HIP runtime does not honor cudaStreamCaptureModeThreadLocal, so - # torch's ProcessGroupNCCL watchdog thread polling hipEventQuery() invalidates the - # main thread's decode cuda-graph capture (hipErrorCapturedEvent -> "Rank 0 - # scheduler died during initialization"). Refs: sgl-project/sglang#29235, #24011; - # ROCm/hip#3876; pytorch/pytorch#176251. Fixed in ROCm 7.2.2+; until the base image - # is bumped, blocking NCCL waits avoid querying events during capture. - export TORCH_NCCL_BLOCKING_WAIT - export NCCL_BLOCKING_WAIT - - # DeepSeek-V4-Pro overrides; last in the block so they win over the defaults above. - # The SGLANG_OPT_*/AITER_* knobs steer DSv4 off the default aiter CK fused-MoE path, - # which raises "Unsupported kernel config for moe heuristic dispatch" at decode on - # this fp4 model. CLI flags live in models.yaml; NIC/socket vars stay runner-derived. - if [[ "$MODEL_NAME" == DeepSeek-V4-Pro* ]]; then - export SGLANG_AITER_MLA_PERSIST=0 - ## resolve the OOR issue - export HSA_NO_SCRATCH_RECLAIM=0 - export MORI_IO_QP_MAX_SEND_WR=32767 - # Unified radix tree: per-component (full-attn / SWA) cache management for - # hybrid-attention models; set unconditionally, not gated on hicache. - export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 - # Free out-of-window SWA KV slots during chunked prefill. Otherwise in-flight - # requests pin SWA KV for their whole context, LRU flushes the trailing window of - # cached sessions, and the prefix-cache hit rate collapses on multi-turn traces. - export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 - - export SGLANG_MORI_DISPATCH_DTYPE=auto - export MORI_COMBINE_DTYPE_PREFILL=auto - export MORI_COMBINE_DTYPE_DECODE=auto - - export MORI_MAX_DISPATCH_TOKENS_PREFILL=8192 - export MORI_MAX_DISPATCH_TOKENS_DECODE=64 - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE - - export SGLANG_MORI_RECV_BOUND=1 - - # PER_RANK dispatch tokens pinned independently (16384 prefill / 128 - # decode); server_sglang.sh prefers these over the MORI_MAX_DISPATCH_* - # coupling when set. - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL=16384 - export MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE=128 - - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=4096 - - export SGLANG_DEFAULT_THINKING=1 - export SGLANG_DSV4_REASONING_EFFORT=high - export SGLANG_USE_ROCM700A=0 - export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton - export SGLANG_OPT_FP8_WO_A_FUSED_INVROPE=1 - export AITER_BF16_FP8_MOE_BOUND=0 - export TORCH_BLAS_PREFER_HIPBLASLT=1 - # aiter batched GEMM for the absorbed MLA projections, carried by the v0.5.18 - # image and off by default in environ.py. - export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 - # DP-attention-only SGLang internal knobs (shared-expert TP1 placement, - # gatherv/reduce-scatter collectives) plus the wider HW-queue count DP - # ranks need to overlap MoRI dispatch with compute. - if [[ "$PREFILL_ENABLE_DP" == "true" || "$DECODE_ENABLE_DP" == "true" ]]; then - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES="${GPU_MAX_HW_QUEUES_DP:-5}" - else - export GPU_MAX_HW_QUEUES=2 - fi - fi - -fi diff --git a/benchmarks/multi_node/amd_utils/env_atom.sh b/benchmarks/multi_node/amd_utils/env_atom.sh deleted file mode 100644 index 71cbdf06ff..0000000000 --- a/benchmarks/multi_node/amd_utils/env_atom.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# ATOM/mooncake environment, sourced by server_atom.sh in place of env.sh. -# IBDEVICES: RDMA device names (e.g. ionic_0,ionic_1,...), set by the runner or -# auto-detected. - -set -x - -export PYTHONUNBUFFERED=1 -export PYTHONDONTWRITEBYTECODE=1 - - -if [[ -z "$IBDEVICES" ]]; then - DETECTED=$(ibv_devinfo 2>/dev/null | grep "hca_id:" | awk '{print $2}' | paste -sd',') - if [[ -n "$DETECTED" ]]; then - export IBDEVICES="$DETECTED" - echo "[INFO] Auto-detected IBDEVICES=$IBDEVICES via ibv_devinfo on $(hostname -s)" - else - # ATOM passes no IB device to the server (mooncake picks its own RDMA device via - # proxy_ip/handshake_port), so a missing IBDEVICES is non-fatal here. - echo "[WARN] Unable to detect RDMA devices via ibv_devinfo; IBDEVICES unset (non-fatal for ATOM/mooncake)" >&2 - fi -else - echo "[INFO] Using IBDEVICES=$IBDEVICES (set by runner or environment)" -fi -export IBDEVICES - - -export LD_LIBRARY_PATH=/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib:${LD_LIBRARY_PATH:-} - -export SAFETENSORS_FAST_GPU=1 - -export VLLM_LOG_LEVEL=WARNING -export ATOM_LOG_LEVEL=WARNING -export AITER_LOG_LEVEL=WARNING -export LOG_LEVEL=WARNING -export LOGLEVEL=WARNING - -set +x - -echo "[INFO] ATOM env: IBDEVICES=$IBDEVICES LD_LIBRARY_PATH includes mooncake" \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh b/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh deleted file mode 100755 index 706f8fbcfc..0000000000 --- a/benchmarks/multi_node/amd_utils/helpers/rdma_check.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash -# Pre-flight RDMA QoS/DCQCN validation, run by job.slurm via srun on every allocated -# node before any container/GPU time is spent. A misconfigured NIC (PFC not covering -# the RoCE priority, DCQCN disabled) does not make MoRI's cross-node RDMA error out; -# it quietly degrades into unexplained tail latency hours later. -# -# check_qos()/check_dcqcn() are trimmed from ROCm/mori tools/env_check.sh -# (https://github.com/ROCm/mori/blob/main/tools/env_check.sh); its full-mesh -# ib_write_bw/ib_write_lat tests are deliberately not ported since this must take -# seconds. Scoped to AMD Pollara (ionic) NICs via nicctl; port the bnxt_*/mlx5_* -# checks from upstream if those NICs join the fleet. -# -# Exit 0 = OK or gracefully skipped (no ionic NICs); 1 = hard misconfiguration. -set -o pipefail - -AINIC_MIN_VER="1.117.5-a-45" # minimum recommended AINIC firmware for IBGDA - -GREEN='\033[0;32m' -RED='\033[0;31m' -YELLOW='\033[0;33m' -NC='\033[0m' - -log_ok() { echo -e "[$(hostname -s)] ${GREEN}[OK]${NC} $*"; } -log_fail() { echo -e "[$(hostname -s)] ${RED}[FAIL]${NC} $*"; } -log_warn() { echo -e "[$(hostname -s)] ${YELLOW}[WARN]${NC} $*"; } -die() { log_fail "$@"; exit 1; } - -# version_ge -> true if >= (dotted/hyphenated, via sort -V) -version_ge() { - local cand="$1" min="$2" - [[ "$cand" == "$min" ]] && return 0 - [[ "$(printf '%s\n%s\n' "$cand" "$min" | sort -V | head -1)" == "$min" ]] -} - -# check_ainic_version_recommendation -# warns if firmware is on the IBGDA-incapable 1.117.1 branch, or below the -# recommended minimum for cross-node MORI (EP over RDMA / IBGDA). -check_ainic_version_recommendation() { - local ver="$1" - [[ -n "$ver" ]] || { log_warn "cannot verify AINIC firmware version against recommendation (empty)"; return; } - if [[ "$ver" =~ ^1\.117\.1([.-]|$) ]]; then - log_warn "AINIC firmware $ver is on the 1.117.1 branch, which does NOT support IBGDA -- upgrade to >= $AINIC_MIN_VER" - elif version_ge "$ver" "$AINIC_MIN_VER"; then - log_ok "AINIC firmware $ver meets the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" - else - log_warn "AINIC firmware $ver is below the recommended minimum (>= $AINIC_MIN_VER) for cross-node IBGDA" - fi -} - -# check_versions() -- informational only (never hard-fails the job). -check_versions() { - local fw_output sw_output - fw_output=$(sudo nicctl show version firmware 2>/dev/null) - sw_output=$(sudo nicctl show version host-software 2>/dev/null) - - local fw_versions fw_count - fw_versions=$(echo "$fw_output" | grep -i "firmware" | awk '{print $NF}' | sort -u) - fw_count=$(echo "$fw_versions" | grep -c . || true) - if [[ "$fw_count" -ne 1 ]]; then - log_warn "firmware versions not consistent across NICs:" - echo "$fw_versions" - local v - while read -r v; do [[ -n "$v" ]] && check_ainic_version_recommendation "$v"; done <<< "$fw_versions" - else - log_ok "firmware : $fw_versions" - check_ainic_version_recommendation "$fw_versions" - fi - - local nicctl_ver - nicctl_ver=$(echo "$sw_output" | grep "nicctl" | awk '{print $NF}') - [[ -n "$nicctl_ver" ]] && log_ok "nicctl : $nicctl_ver" || log_warn "cannot determine nicctl version" -} - -# check_qos() -- HARD gate: classification type must be DSCP, and PFC no-drop -# must be enabled and cover every no-drop priority. Dies (exit 1) otherwise. -check_qos() { - local qos_output - qos_output=$(sudo nicctl show qos 2>/dev/null) - [[ -n "$qos_output" ]] || die "sudo nicctl show qos returned nothing" - - local class_type - class_type=$(echo "$qos_output" | grep "Classification type" | head -1 | awk '{print $NF}') - [[ "$class_type" == "DSCP" ]] || die "classification type is '$class_type', expected 'DSCP'" - log_ok "classification type : DSCP" - - local nd_prio_raw - nd_prio_raw=$(echo "$qos_output" | grep "PFC no-drop priorities" | head -1 | awk '{print $NF}') - [[ -n "$nd_prio_raw" ]] || die "cannot find PFC no-drop priority" - local nd_prios=() - IFS=',' read -ra nd_prios <<< "$nd_prio_raw" - log_ok "no-drop priorities : ${nd_prios[*]}" - - local pfc_bitmap - pfc_bitmap=$(echo "$qos_output" | grep "PFC priority bitmap" | head -1 | awk '{print $NF}') - [[ -n "$pfc_bitmap" && "$pfc_bitmap" != "0x0" ]] || die "PFC is not enabled (bitmap=$pfc_bitmap)" - local p - for p in "${nd_prios[@]}"; do - (( pfc_bitmap & (1 << p) )) || die "PFC bitmap $pfc_bitmap does not cover priority $p" - done - log_ok "PFC enabled for priorities ${nd_prios[*]} (bitmap=$pfc_bitmap)" -} - -# check_dcqcn() -- HARD gate: DCQCN must be enabled on every ROCE device, and -# the CNP DSCP must be consistent across NICs. Exits 1 otherwise. -check_dcqcn() { - local dcqcn_output - dcqcn_output=$(sudo nicctl show dcqcn 2>/dev/null) - [[ -n "$dcqcn_output" ]] || die "sudo nicctl show dcqcn returned nothing" - - local total - total=$(echo "$dcqcn_output" | grep -c "ROCE device") - [[ "$total" -gt 0 ]] || die "no ROCE devices found in dcqcn output" - - local disabled - disabled=$(echo "$dcqcn_output" | grep "Status" | grep -v "Enabled" || true) - if [[ -n "$disabled" ]]; then - log_fail "some ROCE devices have DCQCN disabled:" - echo "$disabled" - exit 1 - fi - log_ok "DCQCN enabled on all $total ROCE devices" - - local cnp_values cnp_count - cnp_values=$(echo "$dcqcn_output" | grep "DSCP value used for CNP" | awk '{print $NF}' | sort -u) - cnp_count=$(echo "$cnp_values" | grep -c . || true) - [[ "$cnp_count" -eq 1 ]] || die "CNP DSCP not consistent across NICs: $cnp_values" - log_ok "CNP DSCP = $cnp_values (consistent across all NICs)" -} - -if ! command -v nicctl &>/dev/null; then - log_warn "nicctl not found on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check (not an ionic NIC host, or nicctl not on PATH)" - exit 0 -fi - -# nicctl exits 0 even with no NIC present, so check its output rather than its exit code. -_nicctl_probe=$(sudo nicctl show version firmware 2>&1 || true) -if echo "$_nicctl_probe" | grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC'; then - log_warn "nicctl present but no ionic NIC detected/accessible on $(hostname -s) -- skipping RDMA QoS/DCQCN pre-flight check" - exit 0 -fi - -check_versions -check_qos -check_dcqcn - -log_ok "RDMA QoS/DCQCN pre-flight check passed on $(hostname -s)" diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm deleted file mode 100755 index d1f5ea11a7..0000000000 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ /dev/null @@ -1,797 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=disagg-bench -#SBATCH -N 3 # Overridden by submit.sh -N flag -#SBATCH -n 3 # Overridden by submit.sh -n flag -#SBATCH --ntasks-per-node=1 -#SBATCH --spread-job -#SBATCH --gres=gpu:8 -#SBATCH --time=24:00:00 -# --output and --error are set by submit.sh via BENCHMARK_LOGS_DIR - -source "$(pwd)/../../benchmark_lib.sh" --validation-only -check_env_vars INFERENCEX_RUNTIME_ENV_VARS VLLM_ROUTER_IMAGE SKIP_RDMA_CHECK SKIP_GPU_SANITY -check_env_vars \ - ENGINE MODEL_NAME MODEL_DIR xP yD \ - BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER BENCH_MAX_CONCURRENCY \ - BENCH_REQUEST_RATE GPUS_PER_NODE PREFILL_ENABLE_EP PREFILL_ENABLE_DP DECODE_ENABLE_EP \ - DECODE_ENABLE_DP PREFILL_TP_SIZE DECODE_TP_SIZE DECODE_MTP_SIZE ROUTER_TYPE \ - ROUTER_PORT PROXY_PING_PORT HEADNODE_PORT SERVER_PORT DRY_RUN \ - BENCHMARK_LOGS_DIR KEEP_CONTAINERS RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK \ - IS_MULTINODE SWEBENCH_USE_MODAL IS_AGENTIC KV_OFFLOADING PREFILL_EP \ - PREFILL_DP_ATTN DECODE_EP DECODE_DP_ATTN DURATION ENABLE_METRICS \ - PREFILL_ROUTER_POLICY DECODE_ROUTER_POLICY DISAGG - -echo "=== Job Start Time ===" -echo "UTC Time: $(TZ=UTC date '+%Y-%m-%d %H:%M:%S %Z')" -echo "PST Time: $(TZ=America/Los_Angeles date '+%Y-%m-%d %H:%M:%S %Z')" -echo "ENGINE: $ENGINE" -echo "=======================" -echo "" - -# ============================================================================= -# Model Validation -# ============================================================================= - -# Use $(pwd) not BASH_SOURCE — sbatch copies the script to /var/spool/slurmd/ -# at runtime, but the CWD remains the submit-time directory (amd_utils/). -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODELS_YAML="$(pwd)/models_vllm.yaml" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - MODELS_YAML="$(pwd)/models_atom.yaml" -else - MODELS_YAML="$(pwd)/models.yaml" -fi - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "Error: models YAML not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "${DOCKER_IMAGE_NAME:-}" ]]; then - echo "Error: DOCKER_IMAGE_NAME is not set." - exit 1 -fi - -# Resolve the models.yaml entry the same way server_sglang.sh does: agentic runs -# (IS_AGENTIC) use the '-AgentX' recipe, non-agentic disaggregated runs use -# '-DI'. Fall back to the bare model name if the variant key is absent. -# MODEL_NAME itself is left unchanged so env.sh/server_sglang.sh still see the base name. -if [[ "${IS_AGENTIC}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - MODEL_YAML_KEY="${MODEL_NAME}-AgentX" -else - MODEL_YAML_KEY="${MODEL_NAME}-DI" -fi -if ! grep -q "^${MODEL_YAML_KEY}:" "$MODELS_YAML"; then - if grep -q "^${MODEL_NAME}:" "$MODELS_YAML"; then - MODEL_YAML_KEY="$MODEL_NAME" - else - echo "Error: Model '$MODEL_YAML_KEY' (nor bare '$MODEL_NAME') not found in $MODELS_YAML" - echo "Available models:" - grep -E '^[A-Za-z]' "$MODELS_YAML" | sed 's/:.*$//' | sed 's/^/ - /' - exit 1 - fi -fi -echo "Model found: $MODEL_YAML_KEY (MODEL_NAME=$MODEL_NAME, IS_AGENTIC=${IS_AGENTIC})" - -RUN_FILE="server.sh" -echo "Runfile set: $RUN_FILE" - -# DI_REPO_DIR points to the repo root. -# $(pwd) is amd_utils/ (the sbatch submit dir); go up 3 levels to reach the repo root. -export DI_REPO_DIR=$(cd "$(pwd)/../../.." && pwd) - -# ============================================================================= -# Model Path Resolution -# ============================================================================= - -export MODEL_DIR - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - # vLLM: Extract hf_dir from models.yaml, search multiple paths, resolve HF cache snapshots - DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - DISK_DIR_NAME="${DISK_DIR_NAME:-$MODEL_NAME}" - echo "Looking for model: $MODEL_NAME (disk dir: $DISK_DIR_NAME)" - - resolve_hf_cache_path() { - local base_path=$1 - if [[ -d "${base_path}/snapshots" ]]; then - local snapshot=$(ls -1 "${base_path}/snapshots" 2>/dev/null | head -1) - if [[ -n "$snapshot" ]]; then - echo "${base_path}/snapshots/${snapshot}" - return 0 - fi - fi - echo "$base_path" - return 1 - } - - MODEL_PATH="" - SEARCH_PATHS=( - "${MODEL_DIR}/${DISK_DIR_NAME}" - "${MODEL_DIR}/${MODEL_NAME}" - "/nfsdata/hf_hub_cache-0/${DISK_DIR_NAME}" - "/nfsdata/hf_hub_cache-0/${MODEL_NAME}" - ) - - for search_path in "${SEARCH_PATHS[@]}"; do - if [[ -d "$search_path" ]]; then - RESOLVED=$(resolve_hf_cache_path "$search_path") - MODEL_PATH="$RESOLVED" - echo "Found MODEL_PATH: $MODEL_PATH" - break - fi - done - - if [[ -z "$MODEL_PATH" ]]; then - echo "FATAL: Model '$MODEL_NAME' not found. Searched:" - for p in "${SEARCH_PATHS[@]}"; do echo " - $p"; done - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -else - # SGLang: Validate model path across all allocated nodes - echo "Looking for model: $MODEL_NAME" - echo "Checking model availability across all allocated nodes..." - - ALL_NODES=$(scontrol show hostnames "$SLURM_JOB_NODELIST") - TOTAL_NODES=$(echo "$ALL_NODES" | wc -l) - echo "Total allocated nodes: $TOTAL_NODES" - echo "Nodes: $(echo "$ALL_NODES" | tr '\n' ' ')" - - check_model_path() { - local path=$1 - local check_name=$2 - echo "Checking $check_name: $path" - srun --nodes=$SLURM_NNODES --ntasks=$SLURM_NNODES /bin/bash -c " - if [ -d '$path' ]; then - echo \"\$(hostname): Found $path\" - exit 0 - else - echo \"\$(hostname): Missing $path\" - exit 1 - fi - " - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo "$check_name available on ALL nodes" - return 0 - else - echo "$check_name NOT available on all nodes" - return 1 - fi - } - - # Extract hf_dir from models.yaml (same as vllm-disagg path above) - SGL_DISK_DIR_NAME=$(awk '/^'"$MODEL_NAME"':/{found=1; next} - found && /^[^ ]/{exit} - found && /hf_dir:/{gsub(/[" ]/, "", $2); print $2; exit}' "$MODELS_YAML") - SGL_DISK_DIR_NAME="${SGL_DISK_DIR_NAME:-$MODEL_NAME}" - - # Prefer the caller-supplied MODEL_PATH (recipe scripts set this explicitly); - # fall back to MODEL_DIR/hf_dir then MODEL_DIR/MODEL_NAME. - if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL_DIR" ]]; then - # Caller already resolved the path (e.g. MODEL_PATH=/it-share/hf_cache/models--...) - # Use it directly if it exists on all nodes, otherwise try subdirectory combos. - if check_model_path "$MODEL_PATH" "MODEL_PATH (caller-supplied)"; then - echo "Selected MODEL_PATH: $MODEL_PATH (caller-supplied, available on all nodes)" - elif check_model_path "$MODEL_PATH/$SGL_DISK_DIR_NAME" "$MODEL_PATH/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_PATH/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_PATH/$MODEL_NAME" "$MODEL_PATH/$MODEL_NAME"; then - MODEL_PATH="$MODEL_PATH/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_PATH" - echo " - $MODEL_PATH/$SGL_DISK_DIR_NAME" - echo " - $MODEL_PATH/$MODEL_NAME" - exit 1 - fi - elif check_model_path "$MODEL_DIR/$SGL_DISK_DIR_NAME" "$MODEL_DIR/$SGL_DISK_DIR_NAME"; then - MODEL_PATH="$MODEL_DIR/$SGL_DISK_DIR_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - elif check_model_path "$MODEL_DIR/$MODEL_NAME" "$MODEL_DIR"; then - MODEL_PATH="$MODEL_DIR/$MODEL_NAME" - echo "Selected MODEL_PATH: $MODEL_PATH (available on all nodes)" - else - echo "FATAL ERROR: Model '$MODEL_NAME' not found on ALL allocated nodes in:" - echo " - $MODEL_DIR/$SGL_DISK_DIR_NAME" - echo " - $MODEL_DIR/$MODEL_NAME" - exit 1 - fi - echo "Final MODEL_PATH: $MODEL_PATH" -fi - -# ============================================================================= -# Node Selection -# ============================================================================= - -NUM_NODES=$((xP + yD)) -echo "NUM_NODES: $NUM_NODES (xP=$xP + yD=$yD)" - -FULL_NODELIST=$(scontrol show hostnames "$SLURM_JOB_NODELIST") -SELECTED_NODES=$(echo "$FULL_NODELIST" | head -n $NUM_NODES) -SELECTED_NODELIST_STR=$(echo "$SELECTED_NODES" | tr '\n' ',' | sed 's/,$//') - -# Docker privilege detection — evaluated per-node since group membership varies. -# Exported as a snippet so every srun participant resolves it locally. -export DOCKER_CMD_DETECT='if docker ps &>/dev/null 2>&1; then DOCKER_CMD=docker; else DOCKER_CMD="sudo docker"; fi' - -# Update SLURM environment variables -export SLURM_NNODES=$NUM_NODES -export SLURM_NTASKS=$NUM_NODES -export SLURM_JOB_NUM_NODES=$NUM_NODES -export SLURM_NPROCS=$NUM_NODES -export SLURM_JOB_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_NODELIST="$SELECTED_NODELIST_STR" -export SLURM_TASKS_PER_NODE="1(x$NUM_NODES)" -export SLURM_NTASKS_PER_NODE=1 - -echo "" -echo "Selected nodes: $SELECTED_NODELIST_STR" - -# ============================================================================= -# IP Resolution -# ============================================================================= - -USER_NAME=$(whoami) -MASTER_NODE=$(echo "$SELECTED_NODES" | head -n 1) -NODE0_ADDR=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$MASTER_NODE" bash -c 'ip route get 1.1.1.1') -NODE0_ADDR=$(echo "$NODE0_ADDR" | awk '/src/ {print $7}') - -IPS=() -for NODE in $SELECTED_NODES; do - IP=$(srun --nodes=1 --ntasks=1 --time=00:20:00 --nodelist="$NODE" bash -c 'ip route get 1.1.1.1') - IP=$(echo "$IP" | awk '/src/ {print $7}') - IPS+=("$IP") -done - -echo "Node IPs: ${IPS[*]}" - -DOCKER_MOUNT_PATH="/workspace" -WS_PATH="${DOCKER_MOUNT_PATH}/benchmarks/multi_node/amd_utils" - -NNODES=$NUM_NODES - -echo "MASTER_NODE: ${MASTER_NODE}" -echo "NODE0_ADDR: ${NODE0_ADDR}" -echo "NNODES: ${NNODES}" -echo "REPO DIR: ${DI_REPO_DIR}" -echo "USER: ${USER_NAME}" - -# Reduce log spam -export TQDM_MININTERVAL=20 - -# Translate the host-resolved MODEL_PATH to the Docker mount namespace -DOCKER_MODEL_PATH="${MODEL_PATH/#$MODEL_DIR//models}" - -export DI_REPO_DIR=$DI_REPO_DIR -export WS_PATH=$WS_PATH -export NNODES=$NNODES -export NODE0_ADDR=$NODE0_ADDR -export MODEL_PATH=$MODEL_PATH -export MODEL_DIR=$MODEL_DIR -export xP=$xP -export yD=$yD -export MODEL_NAME=$MODEL_NAME -export USER_NAME=$USER_NAME -export IPADDRS="$(echo "${IPS[*]}" | sed 's/ /,/g')" -export GPUS_PER_NODE=$GPUS_PER_NODE -export BENCH_INPUT_LEN=$BENCH_INPUT_LEN -export BENCH_OUTPUT_LEN=$BENCH_OUTPUT_LEN -export BENCH_RANDOM_RANGE_RATIO=$BENCH_RANDOM_RANGE_RATIO -export BENCH_NUM_PROMPTS_MULTIPLIER=$BENCH_NUM_PROMPTS_MULTIPLIER -export BENCH_MAX_CONCURRENCY=$BENCH_MAX_CONCURRENCY -export BENCH_REQUEST_RATE=$BENCH_REQUEST_RATE -export DRY_RUN -export BENCHMARK_LOGS_DIR -export KEEP_CONTAINERS -export ENGINE=$ENGINE - -# Eval-related env vars (threaded from submit.sh) -export RUN_EVAL -export EVAL_ONLY -export EVAL_CONC="${EVAL_CONC:-}" -export EVAL_FRAMEWORK -export EVAL_SUITE="${EVAL_SUITE:-}" -export SWEBENCH_GEN_MODE="${SWEBENCH_GEN_MODE:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE -export SWEBENCH_USE_MODAL -export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" -export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" -export HF_TOKEN="${HF_TOKEN:-}" -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export EVAL_LIMIT="${EVAL_LIMIT:-}" - -SANITIZED_USER=$(echo "$USER_NAME" | tr -c 'a-zA-Z0-9_.-' '_') -export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${SLURM_JOB_ID}" - -# vLLM external router container. -# NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older -# dated tags are garbage-collected (manifest unknown) -ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" -# Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). -CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" -export RUN_FILE_FULL="$WS_PATH/${RUN_FILE}" - -SELECTED_NODELIST_SRUN=$(echo "$SELECTED_NODES" | paste -sd,) - -# ============================================================================= -# RDMA QoS / DCQCN Pre-flight Check -# ============================================================================= -# Gate the run on NIC QoS (PFC/DSCP) and DCQCN config on every node before -# any container/GPU time is spent. Runs on the bare host (nicctl is a host -# tool). See rdma_check.sh for details. -RDMA_CHECK_SCRIPT="$(pwd)/helpers/rdma_check.sh" -if [[ "${SKIP_RDMA_CHECK}" == "1" ]]; then - echo "[INFO] SKIP_RDMA_CHECK=1 set; skipping RDMA QoS/DCQCN pre-flight check" -elif [[ -f "$RDMA_CHECK_SCRIPT" ]]; then - echo "Checking RDMA QoS/DCQCN configuration on all $NUM_NODES allocated node(s)..." - srun --nodelist="$SELECTED_NODELIST_SRUN" --ntasks=$NUM_NODES bash "$RDMA_CHECK_SCRIPT" - RDMA_CHECK_RC=$? - if [[ $RDMA_CHECK_RC -ne 0 ]]; then - echo "FATAL: RDMA QoS/DCQCN pre-flight check failed on one or more nodes (see [FAIL] lines above)." - echo " Set SKIP_RDMA_CHECK=1 to bypass (not recommended -- MoRI cross-node transfers would run unprotected by PFC/DCQCN)." - exit 1 - fi - echo "RDMA QoS/DCQCN pre-flight check passed on all $NUM_NODES node(s)" -else - echo "[WARN] $RDMA_CHECK_SCRIPT not found; skipping RDMA QoS/DCQCN pre-flight check" -fi - -cleanup() { - echo "[${SLURM_JOB_ID}] termination received on $(hostname); cleaning up container + stale logs..." - # Backstop: on scancel/timeout/step-hang the foreground `exec docker run` - # client is killed before --rm can fire, so the container (and its GPU/ports/ - # :30000) leaks and the step never returns. Force-remove THIS job's container - # on every allocated node. Scoped to $DOCKER_CONT_NAME so it never touches - # other users' containers. (Ported from InferenceY 51ebfa88.) - srun --nodelist="$SELECTED_NODELIST_SRUN" \ - bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' 2>/dev/null || true' 2>/dev/null || true - rm -rf ${SLURM_SUBMIT_DIR}/logs 2>/dev/null || true - echo "[${SLURM_JOB_ID}] cleanup done." -} - -trap cleanup INT TERM HUP - -# Force NFS cache refresh on all nodes -echo "Refreshing NFS caches on all nodes..." -srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c ' - sync - ls -la '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils > /dev/null 2>&1 - stat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server.sh > /dev/null 2>&1 - cat '"$DI_REPO_DIR"'/benchmarks/multi_node/amd_utils/server.sh > /dev/null 2>&1 - echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null 2>&1 || true - echo "NFS cache refreshed on $(hostname)" -' - -# ============================================================================= -# Build engine-specific Docker environment variables -# ============================================================================= - -# Common env vars (always passed) -DOCKER_ENV_COMMON=( - -e SLURM_JOB_ID=\$SLURM_JOB_ID - -e SLURM_JOB_NODELIST=\$SLURM_JOB_NODELIST - -e NNODES=\$NNODES - -e NODE_RANK=\$SLURM_PROCID - -e NODE0_ADDR=\$NODE0_ADDR - -e MODEL_DIR=/models - -e MODEL_PATH=$DOCKER_MODEL_PATH - -e MODEL_NAME=\$MODEL_NAME - -e GPUS_PER_NODE=\$GPUS_PER_NODE - -e xP=\$xP - -e yD=\$yD - -e IPADDRS=\$IPADDRS - -e BENCH_INPUT_LEN=\$BENCH_INPUT_LEN - -e BENCH_OUTPUT_LEN=\$BENCH_OUTPUT_LEN - -e BENCH_RANDOM_RANGE_RATIO=\$BENCH_RANDOM_RANGE_RATIO - -e BENCH_NUM_PROMPTS_MULTIPLIER=\$BENCH_NUM_PROMPTS_MULTIPLIER - -e BENCH_MAX_CONCURRENCY=\$BENCH_MAX_CONCURRENCY - -e BENCH_REQUEST_RATE=\$BENCH_REQUEST_RATE - -e TQDM_MININTERVAL=\$TQDM_MININTERVAL - -e BENCHMARK_LOGS_DIR=/benchmark_logs - -e ENGINE=\$ENGINE - -e WS_PATH=${WS_PATH} - -e HEADNODE_PORT=\$HEADNODE_PORT - -e ROUTER_PORT=\$ROUTER_PORT - -e SERVER_PORT=\$SERVER_PORT - -e PROXY_PING_PORT=\$PROXY_PING_PORT - -e RUN_EVAL=\$RUN_EVAL - -e EVAL_ONLY=\$EVAL_ONLY - -e \"EVAL_CONC=\$EVAL_CONC\" - -e EVAL_FRAMEWORK=\$EVAL_FRAMEWORK - -e EVAL_LIMIT=\$EVAL_LIMIT - -e EVAL_SUITE=\$EVAL_SUITE - -e SWEBENCH_GEN_MODE=\$SWEBENCH_GEN_MODE - -e FRAMEWORK=\$FRAMEWORK - -e PRECISION=\$PRECISION - -e MODEL_PREFIX=\$MODEL_PREFIX - -e RUNNER_TYPE=\$RUNNER_TYPE - -e RESULT_FILENAME=\$RESULT_FILENAME - -e SPEC_DECODING=\$SPEC_DECODING - # DISAGG was never forwarded into the container at all (not even under a - # different name), so process_agentic_result.py's env_bool("DISAGG") always - # defaulted to false in the result JSON regardless of the actual topology. - -e DISAGG=\${DISAGG} - -e PREFILL_TP_SIZE=\$PREFILL_TP_SIZE - # PREFILL_TP/DECODE_TP/*_NUM_WORKERS (below, undecorated -- distinct from the - # *_SIZE vars server_sglang.sh uses for launch args) are what - # process_agentic_result.py's _gpu_shape() reads for multinode runs. Without - # these the container never sees them (only *_SIZE was passed), so agentic - # result JSONs silently recorded tp=0 / prefill_tp=0 / prefill_num_workers=0 - # for every multinode run. - -e PREFILL_TP=\$PREFILL_TP - -e PREFILL_NUM_WORKERS=\$PREFILL_NUM_WORKERS - -e PREFILL_ENABLE_EP=\$PREFILL_ENABLE_EP - -e PREFILL_ENABLE_DP=\$PREFILL_ENABLE_DP - # process_agentic_result.py reads these (distinct from the ENABLE_* launch - # booleans above) for agg_bmk.json's parallelism metadata; without them it - # defaulted to ep=1/dp_attention=false even for DEP8 arms. - -e PREFILL_EP=\${PREFILL_EP} - -e PREFILL_DP_ATTN=\${PREFILL_DP_ATTN} - -e PREFILL_CONTEXT_LENGTH=\${PREFILL_CONTEXT_LENGTH:-} - -e PREFILL_CHUNKED_PREFILL_SIZE=\${PREFILL_CHUNKED_PREFILL_SIZE:-} - -e DISABLE_CUSTOM_ALL_REDUCE=\${DISABLE_CUSTOM_ALL_REDUCE:-} - -e MAX_MODEL_LEN=\${MAX_MODEL_LEN:-} - -e DURATION=\${DURATION} - -e IS_AGENTIC=\${IS_AGENTIC} - -e FLUSH_DRAIN_TIMEOUT=\$FLUSH_DRAIN_TIMEOUT - -e CLEAR_CACHE_BETWEEN_CONC=\$CLEAR_CACHE_BETWEEN_CONC - -e KV_OFFLOADING=\${KV_OFFLOADING} - -e KV_OFFLOAD_BACKEND=\${KV_OFFLOAD_BACKEND:-} - -e KV_OFFLOAD_BACKEND_METADATA=\"\${KV_OFFLOAD_BACKEND_METADATA:-}\" - -e TOTAL_CPU_DRAM_GB=\${TOTAL_CPU_DRAM_GB:-} - # UMBP direct-linker arm (KV_OFFLOAD_BACKEND=umbp-linker*). server_sglang.sh - # starts the standalone server INSIDE this container on the prefill node and - # exports UMBP_STANDALONE_ADDRESS itself, so only the knobs the recipe sets - # need a route in. Empty for every other arm, and server_sglang.sh only - # reads them on the umbp-linker branch, so this is inert elsewhere. - -e UMBP_DRAM_BYTES=\${UMBP_DRAM_BYTES:-} - -e UMBP_DRAM_USE_HUGEPAGES=\${UMBP_DRAM_USE_HUGEPAGES:-} - -e UMBP_MAX_TOTAL_TOKENS=\${UMBP_MAX_TOTAL_TOKENS:-} - -e UMBP_SA_WAIT_SECONDS=\${UMBP_SA_WAIT_SECONDS:-} - -e UMBP_SA_WAIT_REGISTERED=\${UMBP_SA_WAIT_REGISTERED:-} - -e MORI_UMBP_LOG_LEVEL=\${MORI_UMBP_LOG_LEVEL:-} - -e CONTAINER_BARRIER_TIMEOUT=\${CONTAINER_BARRIER_TIMEOUT:-} - -e ENABLE_METRICS=\${ENABLE_METRICS} - -e PREFILL_ROUTER_POLICY=\${PREFILL_ROUTER_POLICY} - -e DECODE_ROUTER_POLICY=\${DECODE_ROUTER_POLICY} - -e MORI_IO_SQ_BACKOFF_TIMEOUT_US=\${MORI_IO_SQ_BACKOFF_TIMEOUT_US:-} - -e MORI_IO_QP_MAX_SEND_WR=\${MORI_IO_QP_MAX_SEND_WR:-} - -e DECODE_TP_SIZE=\$DECODE_TP_SIZE - -e DECODE_TP=\$DECODE_TP - -e DECODE_NUM_WORKERS=\$DECODE_NUM_WORKERS - -e DECODE_ENABLE_EP=\$DECODE_ENABLE_EP - -e DECODE_ENABLE_DP=\$DECODE_ENABLE_DP - # See PREFILL_EP/PREFILL_DP_ATTN comment above -- decode counterpart. - -e DECODE_EP=\${DECODE_EP} - -e DECODE_DP_ATTN=\${DECODE_DP_ATTN} - -e DECODE_MTP_SIZE=\$DECODE_MTP_SIZE - -e IS_MULTINODE=\$IS_MULTINODE - -e DRY_RUN=\${DRY_RUN} - # SWE-bench agentic eval runs inside this container and needs Modal/HF - # credentials to launch sandboxes and download datasets. - -e SWEBENCH_USE_MODAL=\${SWEBENCH_USE_MODAL} - -e MODAL_TOKEN_ID=\${MODAL_TOKEN_ID:-} - -e MODAL_TOKEN_SECRET=\${MODAL_TOKEN_SECRET:-} - -e HF_TOKEN=\${HF_TOKEN:-} - -e SCENARIO_TYPE=\${SCENARIO_TYPE:-} - -e \"EVAL_LIMIT=\${EVAL_LIMIT:-}\" -) - -# Engine-specific env vars -if [[ "$ENGINE" == "vllm-disagg" ]]; then - DOCKER_ENV_ENGINE=( - -e VLLM_WS_PATH=${WS_PATH} - -e UCX_TLS=tcp,self,shm,rocm_ipc,rocm_copy,cma - -e UCX_SOCKADDR_TLS_PRIORITY=tcp - -e UCX_MEMTYPE_CACHE=y - -e UCX_RNDV_SCHEME=get_zcopy - -e UCX_RNDV_THRESH=4k - -e UCX_ROCM_IPC_MIN_ZCOPY=0 - -e UCX_LOG_LEVEL=warn - -e HSA_ENABLE_SDMA=1 - -e PROXY_STREAM_IDLE_TIMEOUT=\${PROXY_STREAM_IDLE_TIMEOUT} - -e PYTHONPYCACHEPREFIX=/tmp/pycache - ) -elif [[ "$ENGINE" == "atom-disagg" ]]; then - check_env_vars \ - PREFILL_PORT DECODE_PORT HANDSHAKE_PORT MEM_FRAC_STATIC KV_CACHE_DTYPE \ - BLOCK_SIZE MAX_NUM_SEQS - DOCKER_ENV_ENGINE=( - -e ATOM_WS_PATH=${WS_PATH} - -e PREFILL_PORT=${PREFILL_PORT} - -e DECODE_PORT=${DECODE_PORT} - -e ROUTER_PORT=${ROUTER_PORT} - -e HANDSHAKE_PORT=${HANDSHAKE_PORT} - -e MEM_FRAC_STATIC=${MEM_FRAC_STATIC} - -e KV_CACHE_DTYPE=${KV_CACHE_DTYPE} - -e BLOCK_SIZE=${BLOCK_SIZE} - -e MAX_NUM_SEQS=${MAX_NUM_SEQS} - -e MAX_MODEL_LEN=${MAX_MODEL_LEN:-} - -e MAX_NUM_BATCHED_TOKENS=${MAX_NUM_BATCHED_TOKENS:-} - -e EXTRA_SERVER_ARGS=\${EXTRA_SERVER_ARGS:-} - -e IBDEVICES=${IBDEVICES:-} - ) -else - DOCKER_ENV_ENGINE=( - -e SGLANG_WS_PATH=${WS_PATH} - ) -fi - -# HiCache / Mooncake settings are delivered via a bind-mounted config file rather -# than a long list of docker -e flags. Write it once to the shared benchmark-logs -# dir (already a host path, visible on every node) and mount it read-only at -# /config/hicache_mc.env, where the server dispatcher sources it before validation. -# Shell quoting preserves explicit empty values and multi-word arguments. -HICACHE_MC_CONFIG="${BENCHMARK_LOGS_DIR}/hicache_mc_${SLURM_JOB_ID}.env" -{ - for setting in INFERENCEX_RUNTIME_ENV_VARS $INFERENCEX_RUNTIME_ENV_VARS \ - HICACHE_L2_MEM_LAYOUT HICACHE_L3_MEM_LAYOUT ROUTER_DEFAULT_POLICY_FLAGS WAIT_LOCAL_ROUTER_TIMEOUT WAIT_REMOTE_ROUTER_TIMEOUT \ - HICACHE_RATIO FORCE_HICACHE_RATIO HICACHE_HOST_POOL_COUNT HICACHE_PAGE_SIZE HICACHE_IO_BACKEND \ - HICACHE_MEM_LAYOUT HICACHE_WRITE_POLICY HICACHE_STORAGE_BACKEND HICACHE_PREFETCH_POLICY \ - MC_MASTER_PORT MC_METADATA_PORT MC_METRICS_PORT MC_MASTER_THREADS MC_EVICTION_HIGH_WATERMARK \ - MC_PROTOCOL MC_GLOBAL_SEG MC_DEVICE MC_MASTER_ADDR MC_METADATA_SERVER \ - BENCH_NUM_PROMPTS_MULTIPLIER DRY_RUN KEEP_CONTAINERS ROUTER_TYPE ROUTER_PORT \ - PROXY_PING_PORT HEADNODE_PORT SERVER_PORT PROXY_STREAM_IDLE_TIMEOUT ENABLE_METRICS \ - PREFILL_ROUTER_POLICY DECODE_ROUTER_POLICY FLUSH_DRAIN_TIMEOUT CLEAR_CACHE_BETWEEN_CONC ROCM_PATH \ - UCX_HOME RIXL_HOME MORI_IO_SQ_BACKOFF_TIMEOUT_US MORI_IO_QP_MAX_SEND_WR MORI_IO_QP_MAX_CQE \ - MORI_IO_QP_MAX_SGE MORI_IO_TC_DISABLE UCX_IB_GID_INDEX MORI_APP_LOG_LEVEL SGLANG_ROUTER_STDOUT_LOGS \ - TORCH_NCCL_BLOCKING_WAIT NCCL_BLOCKING_WAIT SGLANG_OPT_USE_AITER_INDEXER ROUTER_CACHE_THRESHOLD ROUTER_BALANCE_ABS_THRESHOLD \ - ROUTER_BALANCE_REL_THRESHOLD ROUTER_CANARY_TIMEOUT ROUTER_CANARY_REQ_TIMEOUT ROUTER_READINESS_CANARY ROUTER_CB_ARGS \ - ROUTER_RESILIENCE_FLAGS TIME_LIMIT MAX_MODEL_LEN DISABLE_CUSTOM_ALL_REDUCE \ - HICACHE_TIER PREFILL_PORT DECODE_PORT HANDSHAKE_PORT MEM_FRAC_STATIC \ - KV_CACHE_DTYPE BLOCK_SIZE MAX_NUM_SEQS WAIT_SERVER_TIMEOUT WAIT_ROUTER_TIMEOUT \ - WAIT_REMOTE_ROUTER_TIMEOUT ROUTER_POLICY_FLAGS NCCL_IB_HCA; do - if [[ -n "${!setting+x}" ]]; then - printf '%s=%q\n' "$setting" "${!setting}" - fi - done -} > "$HICACHE_MC_CONFIG" -echo "[config] wrote HiCache/Mooncake settings -> $HICACHE_MC_CONFIG" - -# Engine-specific container filter for pre-clean -CONT_FILTER="name=^container_${ENGINE}_" - -# ============================================================================= -# Optional: separate benchmark-client image (agentic runs) — node-0 sibling -# ============================================================================= -# When CLIENT_IMAGE is set, node 0 runs the aiperf trace replay in its own -# sibling container built from CLIENT_IMAGE (which ships a pre-baked aiperf + -# deps), instead of rebuilding the aiperf venv inside the server container every -# run. Give the server container access to the host docker socket + CLI and the -# host paths the sibling needs for its bind mounts. These fragments are expanded -# at submit time and injected into the server `docker run` below; empty (no-op) -# when CLIENT_IMAGE is unset, so the in-container aiperf path is unchanged. -CLIENT_DOCKER_MOUNTS="" -CLIENT_DOCKER_ENV="" -if [[ -n "${CLIENT_IMAGE:-}" ]]; then - HOST_DOCKER_BIN="$(command -v docker || echo /usr/bin/docker)" - CLIENT_DOCKER_MOUNTS="-v /var/run/docker.sock:/var/run/docker.sock -v ${HOST_DOCKER_BIN}:/usr/bin/docker" - CLIENT_DOCKER_ENV="-e CLIENT_IMAGE=${CLIENT_IMAGE} -e HOST_REPO_DIR=${DI_REPO_DIR} -e HOST_MODEL_DIR=${MODEL_DIR} -e HOST_BENCH_LOGS=${BENCHMARK_LOGS_DIR} -e CLIENT_CONT_NAME=${CLIENT_CONT_NAME}" - echo "[client] node-0 sibling benchmark-client image enabled: ${CLIENT_IMAGE}" - # Best-effort pre-pull on all nodes so node 0's sibling launch doesn't stall. - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD pull '"$CLIENT_IMAGE"' >/dev/null 2>&1 || true' 2>/dev/null || true -fi - -run_amd_multinode_after_preflight \ - "$SELECTED_NODELIST_SRUN" "$NUM_NODES" \ - "$DI_REPO_DIR/benchmarks/multi_node/amd_utils/preflight_node.sh" \ - "$CONT_FILTER" "$SKIP_GPU_SANITY" \ - bash -lc " -set -eo pipefail - -echo \"Rank \$SLURM_PROCID on \$(hostname)\" - -# Per-node docker privilege detection -eval \"\$DOCKER_CMD_DETECT\" -echo \"[docker-detect] rank \$SLURM_PROCID: DOCKER_CMD=\$DOCKER_CMD\" - -# Enable out-of-tree RDMA library mounts for atom-disagg (mooncake requires host RDMA stack) -RDMA_MOUNTS=() -if [[ "$ENGINE" == "atom-disagg" ]]; then - -# When the container base OS differs from the host (e.g. Ubuntu 24.04 image -# on a 22.04 host), the container's bundled libibverbs/libionic may be -# ABI-incompatible with the host kernel drivers. Detect the NIC type and -# bind-mount the host's out-of-tree RDMA userspace libraries into the -# container so the RDMA stack always matches the running kernel. -_detect_nic_type() { - if [[ -n \"\${MORI_NIC_TYPE:-}\" ]]; then echo \"\$MORI_NIC_TYPE\"; return; fi - local bnxt=0 mlx5=0 ionic=0 - if [[ -d /sys/class/infiniband ]]; then - for dev in /sys/class/infiniband/*; do - local name; name=\$(basename \"\$dev\") - case \"\$name\" in - bnxt_re*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; - *) - local drv; drv=\$(basename \"\$(readlink -f \"\$dev/device/driver\" 2>/dev/null)\" 2>/dev/null || true) - case \"\$drv\" in bnxt*) ((bnxt++)) ;; mlx5*) ((mlx5++)) ;; ionic*) ((ionic++)) ;; esac ;; - esac - done - fi - if (( bnxt >= mlx5 && bnxt >= ionic && bnxt > 0 )); then echo bnxt - elif (( ionic >= mlx5 && ionic > 0 )); then echo ionic - else echo mlx5; fi -} - -_find_host_ibverbs() { - for c in /usr/lib64/libibverbs.so.1 /lib/x86_64-linux-gnu/libibverbs.so.1 /usr/lib/x86_64-linux-gnu/libibverbs.so.1.14.39.0 /usr/lib/x86_64-linux-gnu/libibverbs.so.1; do - local r; r=\$(readlink -f \"\$c\" 2>/dev/null || true) - [[ \"\$r\" == *libibverbs.so.1.14.57.0 ]] && continue - if [[ -f \"\$r\" ]]; then echo \"\$r\"; return; fi - done -} - -_NIC_TYPE=\$(_detect_nic_type) -echo \"[rdma] NIC type: \${_NIC_TYPE} on \$(hostname)\" - -if [[ \"\$_NIC_TYPE\" == \"ionic\" || \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - _host_ibv=\$(_find_host_ibverbs) - if [[ -n \"\$_host_ibv\" ]]; then - RDMA_MOUNTS+=(-v \"\$_host_ibv:/lib/x86_64-linux-gnu/libibverbs.so.1\") - fi -fi - -if [[ \"\$_NIC_TYPE\" == \"ionic\" ]]; then - for _dir in /usr/local/lib /usr/lib/x86_64-linux-gnu; do - for _lib in \"\$_dir\"/libionic*.so; do - [[ -f \"\$_lib\" ]] || continue - _real=\$(readlink -f \"\$_lib\") - [[ -f \"\$_real\" ]] && RDMA_MOUNTS+=(-v \"\$_real:\$_real\") - RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - done - if [[ -d /usr/lib/x86_64-linux-gnu/libibverbs ]]; then - for _lib in /usr/lib/x86_64-linux-gnu/libibverbs/libionic-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:\$_lib\") - done - fi - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -elif [[ \"\$_NIC_TYPE\" == \"bnxt\" ]]; then - for _lib in /usr/local/lib/libbnxt_re-rdmav*.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/libibverbs/\$(basename \"\$_lib\")\") - done - for _lib in /usr/local/lib/libbnxt_re.so; do - [[ -f \"\$_lib\" ]] && RDMA_MOUNTS+=(-v \"\$_lib:/usr/lib/x86_64-linux-gnu/\$(basename \"\$_lib\")\") - done - [[ -d /etc/libibverbs.d ]] && RDMA_MOUNTS+=(-v /etc/libibverbs.d:/etc/libibverbs.d:ro) -fi - -if [[ \${#RDMA_MOUNTS[@]} -gt 0 ]]; then - echo \"[rdma] bind-mounts: \${RDMA_MOUNTS[*]}\" -else - echo \"[rdma] no out-of-tree RDMA mounts needed\" -fi -fi # end: if ENGINE == atom-disagg - -# Start vLLM external router container on node 0 -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - \$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true - \$DOCKER_CMD run -d \ - --name \"$ROUTER_CONT_NAME\" \ - --network host \ - --ulimit nofile=1048576:1048576 \ - -v /tmp:/run_logs \ - \"$VLLM_ROUTER_IMAGE\" \ - bash -lc \"mkdir -p /run_logs/slurm_job-${SLURM_JOB_ID} && exec vllm-router \ - --vllm-pd-disaggregation \ - --kv-connector moriio \ - --vllm-discovery-address 0.0.0.0:${PROXY_PING_PORT} \ - --port ${ROUTER_PORT} \ - --host 0.0.0.0 \ - --policy consistent_hash \ - --prefill-policy consistent_hash \ - --decode-policy consistent_hash \ - --log-level info 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/vllm_router_\$(hostname).log \" -fi - -# Skip exec on vllm-disagg rank 0 so we can stop the router after the main -# container exits. Without this, decode nodes block forever waiting for the -# router port to close (the router is a separate container). -MAYBE_EXEC=exec -if [[ \"$ENGINE\" == \"vllm-disagg\" && \"$ROUTER_TYPE\" == \"vllm-router\" && \"\$SLURM_PROCID\" == \"0\" ]]; then - MAYBE_EXEC= - set +e -fi - -\$MAYBE_EXEC \$DOCKER_CMD run \ - --init \ - --stop-timeout 10 \ - --device /dev/dri \ - --device /dev/kfd \ - --device /dev/infiniband \ - --device=/dev/infiniband/rdma_cm \ - --device=/dev/infiniband/uverbs0 \ - --device=/dev/infiniband/uverbs1 \ - --device=/dev/infiniband/uverbs2 \ - --device=/dev/infiniband/uverbs3 \ - --device=/dev/infiniband/uverbs4 \ - --device=/dev/infiniband/uverbs5 \ - --device=/dev/infiniband/uverbs6 \ - --device=/dev/infiniband/uverbs7 \ - --ulimit memlock=-1 \ - --ulimit stack=67108864 \ - --ulimit nofile=1048576:1048576 \ - --network host \ - --ipc host \ - --group-add video \ - --cap-add SYS_PTRACE \ - --security-opt seccomp=unconfined \ - --privileged \ - -v /sys:/sys \ - $(command -v nicctl >/dev/null 2>&1 && echo "-v $(which nicctl):/usr/sbin/nicctl") \ - -v ${MODEL_DIR}:/models \ - -v \$HOME/.ssh:/root/.ssh \ - --shm-size 128G \ - -v /tmp:/run_logs \ - -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ - -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ - -v ${HICACHE_MC_CONFIG}:/config/hicache_mc.env:ro \ - ${EXTRA_DOCKER_MOUNTS:-} \ - ${CLIENT_DOCKER_MOUNTS} \ - \${RDMA_MOUNTS[@]+"\${RDMA_MOUNTS[@]}"} \ - ${DOCKER_ENV_COMMON[*]} \ - ${DOCKER_ENV_ENGINE[*]} \ - ${CLIENT_DOCKER_ENV} \ - --name \"$DOCKER_CONT_NAME\" \ - --entrypoint \"\" \ - \"$DOCKER_IMAGE_NAME\" bash -lc ' - set -o pipefail - mkdir -p /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"' - '"$RUN_FILE_FULL"' 2>&1 | tee /run_logs/slurm_job-'\"\$SLURM_JOB_ID\"'/server_\$(hostname).log - ' - -# Only reached when exec was skipped (vllm-disagg rank 0) -DOCKER_EXIT_CODE=\$? -echo \"[rank 0] Main container exited (rc=\$DOCKER_EXIT_CODE). Stopping vllm-router...\" -\$DOCKER_CMD rm -f \"$ROUTER_CONT_NAME\" 2>/dev/null || true -exit \$DOCKER_EXIT_CODE -" -SERVER_SRUN_RC=$? - -if [[ "${KEEP_CONTAINERS}" != "1" ]]; then - srun --nodelist="$SELECTED_NODELIST_SRUN" bash -c 'eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$DOCKER_CONT_NAME"' '"$CLIENT_CONT_NAME"' 2>/dev/null || true' - - # Clean up vLLM external router container on node 0 - if [[ "$ENGINE" == "vllm-disagg" && "$ROUTER_TYPE" == "vllm-router" ]]; then - srun --nodes=1 --ntasks=1 --nodelist="$MASTER_NODE" bash -c ' - eval "$DOCKER_CMD_DETECT"; $DOCKER_CMD rm -f '"$ROUTER_CONT_NAME"' 2>/dev/null || true - ' - fi -fi - -# /run_logs is backed by each compute node's local /tmp, so the node-0 copy -# performed by the engine launcher cannot see prefill/decode logs written on -# other nodes. Collect after the server step and container cleanup so failed -# runs also include shutdown output. KEEP_CONTAINERS=1 retains a snapshot of -# any containers left running for debugging. -# Use sudo because the container-created source and the existing node-0 -# destination can be root-owned. Restore ownership after the fan-in so a -# subsequent runner job can clean the workspace normally. -SHARED_JOB_LOGS="${BENCHMARK_LOGS_DIR}/logs/slurm_job-${SLURM_JOB_ID}" -if ! srun --nodelist="$SELECTED_NODELIST_SRUN" \ - --nodes="$NUM_NODES" --ntasks="$NUM_NODES" --ntasks-per-node=1 \ - bash "$DI_REPO_DIR/benchmarks/multi_node/amd_utils/stage_node_logs.sh" \ - "/tmp/slurm_job-${SLURM_JOB_ID}" "$SHARED_JOB_LOGS"; then - echo "[logs][ERROR] failed to stage logs from one or more Slurm nodes" >&2 - if [[ "$SERVER_SRUN_RC" -eq 0 ]]; then - SERVER_SRUN_RC=1 - fi -fi - -if [[ -d "$SHARED_JOB_LOGS" ]]; then - sudo chown -R "$(id -u):$(id -g)" "$SHARED_JOB_LOGS" 2>/dev/null || true - chmod -R a+rwX "$SHARED_JOB_LOGS" 2>/dev/null || true -fi - -exit "$SERVER_SRUN_RC" diff --git a/benchmarks/multi_node/amd_utils/models.yaml b/benchmarks/multi_node/amd_utils/models.yaml deleted file mode 100644 index e29dc5a01c..0000000000 --- a/benchmarks/multi_node/amd_utils/models.yaml +++ /dev/null @@ -1,373 +0,0 @@ -# Model-specific SGLang server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# base_flags: str # Common flags for both prefill and decode -# mtp_flags: str # Appended to decode when DECODE_MTP_SIZE > 0 -# dp_flags: str # Appended when DP attention is enabled (prefill or decode) -# ep_flags: str # Appended when EP is enabled. EP-specific MoE knobs only -# # (a2a backend, deepep mode, ep-dispatch algorithm). With -# # ep=1 these are dropped so the MoE runs tensor-parallel (TP). -# prefill: -# mem_fraction_static: float -# disable_radix_cache: bool -# dp: # Config when data-parallel attention is enabled -# max_running_requests: int -# chunked_prefill_size: str # Can be integer or bash arithmetic expression -# cuda_graph_bs: str # Space-separated values -# no_dp: # Config when data-parallel attention is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str # "start-end" expanded via seq -# decode: -# mem_fraction_static: float -# dp: -# max_running_requests: int -# chunked_prefill_size: str -# cuda_graph_bs_range: str -# ep_only: # Config when EP is enabled but DP is disabled -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str -# no_dp: -# max_running_requests: int -# chunked_prefill_size: int -# cuda_graph_bs_range: str - -DeepSeek-V3: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V3-0324: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -Qwen3.5-397B-A17B-FP8: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori --moe-dense-tp-size 1" - mtp_flags: "" - dp_flags: "--enable-dp-attention --enable-dp-lm-head" - ep_flags: "--moe-a2a-backend mori" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-Preview: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: 16384 - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-algorithm NEXTN --speculative-eagle-topk 1" - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 24 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-160" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-R1-0528-MXFP4-v2: - base_flags: "--decode-log-interval 1000 --log-level warning --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend aiter --disaggregation-transfer-backend mori" - mtp_flags: "--speculative-draft-model-path SGLang/DeepSeek-R1-NextN --speculative-algorithm NEXTN --speculative-eagle-topk 1 --speculative-attention-mode decode " - dp_flags: "--enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --stream-interval 100 --tokenizer-worker-num 32 " - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - prefill: - mem_fraction_static: 0.8 - disable_radix_cache: true - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_PREFILL * PREFILL_TP_SIZE" - cuda_graph_bs: "1 2 3" - context_length: 9217 - max_total_tokens: 131072 - enable_two_batch_overlap: true - no_dp: - max_running_requests: 128 - chunked_prefill_size: 16384 - cuda_graph_bs_range: "1-128" - decode: - mem_fraction_static: 0.85 - dp: - max_running_requests: 4096 - chunked_prefill_size: "MORI_MAX_DISPATCH_TOKENS_DECODE * DECODE_TP_SIZE" - cuda_graph_bs_range: "1-512" - ep_only: - max_running_requests: 256 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-256" - no_dp: - max_running_requests: 128 - chunked_prefill_size: 262144 - cuda_graph_bs_range: "1-128" - -DeepSeek-V4-Pro-AgentX: &DeepSeek-V4-Pro-AgentX - base_flags: "--enable-deepseek-v4-fp4-indexer --watchdog-timeout 3600 --load-balance-method round_robin --kv-cache-dtype fp8_e4m3 --attention-backend dsv4 --page-size 256 --swa-full-tokens-ratio 0.1 --enforce-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --disaggregation-transfer-backend mori --tokenizer-worker-num 8 --stream-interval 20 --log-level info --log-level-http error" - # --enable-dp-lm-head is required by SGLang for DSpark under DP attention; it - # is harmless for the EAGLE/MTP arms, so it stays unconditional here rather - # than needing a second DP flag string. - dp_flags: "--enable-dp-attention --enable-dp-lm-head --swa-full-tokens-ratio 0.15 --enable-dp-attention-local-control-broadcast" - ep_flags: "--ep-dispatch-algorithm fake --moe-a2a-backend mori --deepep-mode normal" - mtp_flags: "--speculative-algorithm EAGLE --speculative-eagle-topk 1" - # DSpark draft head, selected when the sweep sets spec-decoding: draft_model. - # Kept alongside mtp_flags rather than replacing it, so recipes that stay on - # spec-decoding: mtp keep the EAGLE arm untouched. The -0813 checkpoint - # bundles the draft head (dspark_block_size / dspark_markov_rank / - # dspark_target_layer_ids in config.json), so --speculative-draft-model-path - # defaults to --model-path and no separate draft checkpoint is needed. - # server_sglang.sh appends the block size and the verify window from - # DECODE_MTP_SIZE (= gamma); unlike EAGLE, num-steps is pinned to 1. - dspark_flags: "--speculative-algorithm DSPARK --speculative-eagle-topk 1" - prefill: - disable_radix_cache: false - disable_cuda_graph: true - dp: - # Scales with the sweep's own concurrency (2x headroom) rather than a - # fixed cap; - mem_fraction_static: 0.92 - max_running_requests: "BENCH_MAX_CONC_VALUE*2" - chunked_prefill_size: "8192 * PREFILL_TP_SIZE" - context_length: 1048576 - enable_two_batch_overlap: false - no_dp: - mem_fraction_static: 0.86 - max_running_requests: "BENCH_MAX_CONC_VALUE*2" - chunked_prefill_size: "16384" - context_length: 1048576 - decode: - disagg_decode_enable_radix_cache: false - dp: - mem_fraction_static: 0.92 - max_running_requests: "BENCH_MAX_CONC_VALUE*2" - cuda_graph_bs_range: "1-BENCH_MAX_CONC_VALUE/4" - no_dp: - mem_fraction_static: 0.86 - max_running_requests: "BENCH_MAX_CONC_VALUE*2" - cuda_graph_bs_range: "1-BENCH_MAX_CONC_VALUE*2" - -# Pro-0813 serves the PD path with DSPARK (spec-decoding: draft_model), which -# measured clean across c4-c256; the EAGLE 3-1-4 arm remains available through -# spec-decoding: mtp. Synthetic acceptance is checkpoint-specific in -# server_sglang.sh and does not depend on which of the two runs: thinking-on, -# draft length 3 uses AL 3.01. -DeepSeek-V4-Pro-0813-AgentX: *DeepSeek-V4-Pro-AgentX diff --git a/benchmarks/multi_node/amd_utils/models_atom.yaml b/benchmarks/multi_node/amd_utils/models_atom.yaml deleted file mode 100644 index d3232c01da..0000000000 --- a/benchmarks/multi_node/amd_utils/models_atom.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Model-specific ATOM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the directory name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# env: str # Space-separated KEY=VALUE pairs exported unconditionally -# tp_dp_flags: str # Shared TP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_tp_dp_flags: str # TP+DPA flags for prefill only (overrides tp_dp_flags) -# decode_tp_dp_flags: str # TP+DPA flags for decode only (overrides tp_dp_flags) -# tp_dp_env: str # Space-separated KEY=VALUE pairs exported only in TP+DPA mode -# ep_dp_flags: str # Shared EP+DPA flags (fallback when prefill/decode-specific keys are absent) -# prefill_ep_dp_flags: str # EP+DPA flags for prefill only (overrides ep_dp_flags) -# decode_ep_dp_flags: str # EP+DPA flags for decode only (overrides ep_dp_flags) -# ep_dp_env: str # Space-separated KEY=VALUE pairs exported only in EP+DPA mode -# mtp_flags: str # Flags passed to SPEC_ARGS before $DECODE_MTP_SIZE (e.g. "--method mtp --num-speculative-tokens") -# kv_cache_flags: str # Full --kv_cache_dtype flag string (e.g. "--kv_cache_dtype fp8", or "" for none) -# online_quant_config: str # JSON string passed to --online_quant_config (used when DPA is disabled) -# online_quant_dpa_config: str # JSON string passed to --online_quant_config when DPA is enabled (falls back to online_quant_config) -# block_size: str # --block-size value (overrides server_atom.sh default of 16) -# mem_frac_static: str # --gpu-memory-utilization value (overrides default of 0.85) -# max_model_len: str # --max-model-len value (overrides default of unset) -# max_num_seqs: str # --max-num-seqs value (overrides default of 256) -# max_num_batched_tokens: str # --max-num-batched-tokens value (overrides default of unset) -# scheduler_delay_factor: str # --scheduler-delay-factor value (overrides default of unset) -# All registered model/scenario combinations are retired; see deprecated/models_atom.yaml. -{} diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml deleted file mode 100644 index 515e5caf83..0000000000 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Model-specific vLLM server configurations for disaggregated inference. -# -# Each top-level key is a MODEL_NAME value (must match the model identifier -# used in amd-master.yaml and the directory/HF-cache name under MODEL_DIR). -# -# To add a new model: add a new top-level entry following the same schema. -# No script changes are required. -# -# Schema: -# : -# prefill_flags: str # vLLM CLI flags for prefill workers -# decode_flags: str # vLLM CLI flags for decode workers -# env: str # Space-separated KEY=VALUE pairs exported before vllm serve -# hf_dir: str # (optional) On-disk directory name if it differs from the key -# # e.g. HF cache layout: models--org--checkpoint - -Llama-3.1-405B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" - -amd-Llama-3.3-70B-Instruct-FP8-KV: - prefill_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - decode_flags: "--tensor-parallel-size 8 --max-model-len 65536 --kv-cache-dtype fp8" - env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" diff --git a/benchmarks/multi_node/amd_utils/node_excludes.yaml b/benchmarks/multi_node/amd_utils/node_excludes.yaml deleted file mode 100644 index f18f7ad210..0000000000 --- a/benchmarks/multi_node/amd_utils/node_excludes.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Per (framework, model) SLURM node exclusions, consumed by submit.sh. -# -# A rule applies when FRAMEWORK matches "framework" exactly and MODEL_NAME is -# in "models"; its exclude_nodes are then passed to sbatch --exclude. Any -# (framework, model) combination with no matching rule gets an empty exclude -# list (i.e. no nodes excluded). -# -# SLURM_EXCLUDE_NODES env var, if set, always overrides this file. - -# Retired model rules are archived in deprecated/node_excludes.yaml. -rules: [] diff --git a/benchmarks/multi_node/amd_utils/preflight_node.sh b/benchmarks/multi_node/amd_utils/preflight_node.sh deleted file mode 100644 index 34759bab89..0000000000 --- a/benchmarks/multi_node/amd_utils/preflight_node.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../../benchmark_lib.sh" --validation-only -check_env_vars DOCKER_CMD_DETECT DI_REPO_DIR SLURM_JOB_ID -CONT_FILTER="$1" -SKIP_GPU_SANITY="$2" -check_env_vars CONT_FILTER SKIP_GPU_SANITY - -preflight_node() { - eval "$DOCKER_CMD_DETECT" - - # Preserve the existing pre-clean scope and ordering. Moving it into this - # separate Slurm step prevents one node starting while another still drains. - $DOCKER_CMD ps -aq --filter "$CONT_FILTER" | xargs -r $DOCKER_CMD rm -f || true - $DOCKER_CMD ps -aq | xargs -r $DOCKER_CMD stop -t 15 || true - $DOCKER_CMD ps -aq | xargs -r $DOCKER_CMD rm -f || true - sleep 2 - - if [[ "$SKIP_GPU_SANITY" == "1" ]]; then - echo "[INFO] SKIP_GPU_SANITY=1 set; skipping GPU pre-flight drain check" - else - # Avoid benchmark-only agentic initialization on the host, as before. - bash -c 'unset IS_AGENTIC SCENARIO_TYPE; source "$DI_REPO_DIR/benchmarks/benchmark_lib.sh" && wait_for_amd_gpu_clean' - fi -} - -NODE_LOG_DIR="/tmp/slurm_job-${SLURM_JOB_ID}" -mkdir -p "$NODE_LOG_DIR" -preflight_node 2>&1 | tee "$NODE_LOG_DIR/preflight_$(hostname).log" diff --git a/benchmarks/multi_node/amd_utils/server.sh b/benchmarks/multi_node/amd_utils/server.sh deleted file mode 100755 index 6b65281c35..0000000000 --- a/benchmarks/multi_node/amd_utils/server.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -# Multi-Engine Disaggregated Server Dispatcher -# Dispatches to the engine-specific server launcher based on ENGINE env var. -# ENGINE=sglang-disagg (default) -> server_sglang.sh (SGLang + MoRI) -# ENGINE=vllm-disagg -> server_vllm.sh (vLLM + Nixl/MoRI-IO) -# ENGINE=atom-disagg -> server_atom.sh (ATOM + mooncake) - -check_env_vars ENGINE WS_PATH -if [[ -f /config/hicache_mc.env ]]; then - set -a - source /config/hicache_mc.env - set +a -fi -export WS_PATH ENGINE - -echo "[DISPATCHER] ENGINE=$ENGINE WS_PATH=$WS_PATH" - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - source "$WS_PATH/server_vllm.sh" -elif [[ "$ENGINE" == "atom-disagg" ]]; then - export ATOM_WS_PATH="$WS_PATH" - source "$WS_PATH/server_atom.sh" -else - source "$WS_PATH/server_sglang.sh" -fi diff --git a/benchmarks/multi_node/amd_utils/server_atom.sh b/benchmarks/multi_node/amd_utils/server_atom.sh deleted file mode 100755 index e08b1cba5b..0000000000 --- a/benchmarks/multi_node/amd_utils/server_atom.sh +++ /dev/null @@ -1,532 +0,0 @@ -#!/bin/bash -# ATOM disaggregated launcher: mooncake RDMA KV transfer and atomesh routing. - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars \ - MODEL_NAME ROUTER_PORT PREFILL_PORT DECODE_PORT HANDSHAKE_PORT \ - MEM_FRAC_STATIC BLOCK_SIZE MAX_NUM_SEQS WAIT_SERVER_TIMEOUT - -check_env_vars \ - NODE0_ADDR NODE_RANK xP yD IPADDRS \ - PREFILL_TP_SIZE DECODE_TP_SIZE PREFILL_ENABLE_EP PREFILL_ENABLE_DP DECODE_ENABLE_EP \ - DECODE_ENABLE_DP DECODE_MTP_SIZE BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_RANDOM_RANGE_RATIO \ - BENCH_REQUEST_RATE BENCH_NUM_PROMPTS_MULTIPLIER BENCH_MAX_CONCURRENCY DRY_RUN GPUS_PER_NODE \ - RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK BENCHMARK_LOGS_DIR MODEL_DIR \ - ATOM_WS_PATH - -EXTRA_SERVER_ARGS="${EXTRA_SERVER_ARGS:-}" - -source $ATOM_WS_PATH/setup_deps.sh -source $ATOM_WS_PATH/env_atom.sh - -# lm-eval with high num_concurrent exhausts the default 1024 FD limit. -ulimit -n 65536 2>/dev/null || ulimit -n 8192 2>/dev/null || true -echo "ulimit -n (open files): $(ulimit -n)" - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -if [[ -z "$host_ip" ]]; then - host_ip=$(hostname -I 2>/dev/null | awk '{print $1}') -fi -host_name=$(hostname) - -set -x -_yaml_tmp=$(mktemp) -python3 << PYEOF > "$_yaml_tmp" -import yaml -with open('${ATOM_WS_PATH}/models_atom.yaml') as f: - m = yaml.safe_load(f).get('${MODEL_NAME}', {}) -def sh(v): return v.replace("'", "'\\''") -print(f"MODEL_ENVS='{sh(m.get('env', ''))}'") -_tp_dp = m.get('tp_dp_flags', '') -print(f"PREFILL_MODEL_TP_DP_FLAGS='{sh(m.get('prefill_tp_dp_flags', _tp_dp))}'") -print(f"DECODE_MODEL_TP_DP_FLAGS='{sh(m.get('decode_tp_dp_flags', _tp_dp))}'") -_ep_dp = m.get('ep_dp_flags', '') -print(f"PREFILL_MODEL_EP_DP_FLAGS='{sh(m.get('prefill_ep_dp_flags', _ep_dp))}'") -print(f"DECODE_MODEL_EP_DP_FLAGS='{sh(m.get('decode_ep_dp_flags', _ep_dp))}'") -print(f"MODEL_TP_DP_ENV='{sh(m.get('tp_dp_env', ''))}'") -print(f"MODEL_EP_DP_ENV='{sh(m.get('ep_dp_env', ''))}'") -print(f"MODEL_MTP_FLAGS='{sh(m.get('mtp_flags', ''))}'") -print(f"MODEL_KV_ARG='{sh(m.get('kv_cache_flags', ''))}'") -print(f"_ONLINE_QUANT_CONFIG='{sh(m.get('online_quant_config', ''))}'") -print(f"_ONLINE_QUANT_DPA_CONFIG='{sh(m.get('online_quant_dpa_config', m.get('online_quant_config', '')))}'") -print(f"_YAML_BLOCK_SIZE='{sh(m.get('block_size', ''))}'") -print(f"_YAML_MEM_FRAC_STATIC='{sh(m.get('mem_frac_static', ''))}'") -print(f"_YAML_MAX_MODEL_LEN='{sh(m.get('max_model_len', ''))}'") -print(f"_YAML_MAX_NUM_SEQS='{sh(m.get('max_num_seqs', ''))}'") -print(f"_YAML_MAX_NUM_BATCHED_TOKENS='{sh(m.get('max_num_batched_tokens', ''))}'") -print(f"_YAML_SCHEDULER_DELAY_FACTOR='{sh(m.get('scheduler_delay_factor', ''))}'") -PYEOF -# shellcheck source=/dev/null -source "$_yaml_tmp" -rm -f "$_yaml_tmp" -unset _yaml_tmp - -# Model YAML overrides the caller-provided server tuning. -BLOCK_SIZE="${_YAML_BLOCK_SIZE:-${BLOCK_SIZE}}" -MEM_FRAC_STATIC="${_YAML_MEM_FRAC_STATIC:-${MEM_FRAC_STATIC}}" -MAX_MODEL_LEN="${_YAML_MAX_MODEL_LEN:-${MAX_MODEL_LEN:-}}" -MAX_NUM_SEQS="${_YAML_MAX_NUM_SEQS:-${MAX_NUM_SEQS}}" -MAX_NUM_BATCHED_TOKENS="${_YAML_MAX_NUM_BATCHED_TOKENS:-${MAX_NUM_BATCHED_TOKENS:-}}" -SCHEDULER_DELAY_FACTOR="${_YAML_SCHEDULER_DELAY_FACTOR:-${SCHEDULER_DELAY_FACTOR:-}}" -unset _YAML_BLOCK_SIZE _YAML_MEM_FRAC_STATIC _YAML_MAX_MODEL_LEN _YAML_MAX_NUM_SEQS _YAML_MAX_NUM_BATCHED_TOKENS _YAML_SCHEDULER_DELAY_FACTOR - -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + GPUS_PER_NODE - 1) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -PREFILL_ARGS="" -PREFILL_IPS=() -for i in $(seq 0 $((xP - 1))); do - idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_IPS[$i]="${IP_ARRAY[$idx]}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$idx]}:${PREFILL_PORT}" -done - -DECODE_ARGS="" -DECODE_IPS=() -for i in $(seq 0 $((yD - 1))); do - idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_IPS[$i]="${IP_ARRAY[$idx]}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$idx]}:${DECODE_PORT}" -done - -PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE") #TP -ONLINE_QUANT_ARG="" -if [ "$PREFILL_ENABLE_DP" = "true" ]; then - if [ "$PREFILL_ENABLE_EP" = "true" ]; then #EP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - PREFILL_PARALLEL_ARGS=(-tp "$PREFILL_TP_SIZE" ${PREFILL_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi - if [[ -n "$_ONLINE_QUANT_DPA_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_DPA_CONFIG}'" - fi -else - if [[ -n "$_ONLINE_QUANT_CONFIG" ]]; then - ONLINE_QUANT_ARG="--online_quant_config '${_ONLINE_QUANT_CONFIG}'" - fi -fi - -DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE") #TP -if [ "$DECODE_ENABLE_DP" = "true" ]; then - if [ "$DECODE_ENABLE_EP" = "true" ]; then #EP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_EP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_EP_DP_ENV}; do export "$_dp_env_pair"; done - else #TP+DPA - DECODE_PARALLEL_ARGS=(-tp "$DECODE_TP_SIZE" ${DECODE_MODEL_TP_DP_FLAGS}) - for _dp_env_pair in ${MODEL_TP_DP_ENV}; do export "$_dp_env_pair"; done - fi -fi -unset _dp_env_pair -unset _ONLINE_QUANT_CONFIG _ONLINE_QUANT_DPA_CONFIG - -for _env_pair in ${MODEL_ENVS}; do - export "$_env_pair" -done -unset _env_pair - -SPEC_ARGS=() -if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE}" -gt 0 ]]; then - SPEC_ARGS=(${MODEL_MTP_FLAGS} "$DECODE_MTP_SIZE") -fi - -KV_CACHE_ARG="${MODEL_KV_ARG}" - -MODEL_LEN_ARGS="" -if [[ -n "$MAX_MODEL_LEN" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-model-len ${MAX_MODEL_LEN}" -fi -if [[ -n "$MAX_NUM_BATCHED_TOKENS" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS}" -fi -if [[ -n "$SCHEDULER_DELAY_FACTOR" ]]; then - MODEL_LEN_ARGS="${MODEL_LEN_ARGS} --scheduler-delay-factor ${SCHEDULER_DELAY_FACTOR}" -fi - -cat < prefill node 0 + router; 1..NODE_OFFSET-1 -> prefill; -# NODE_OFFSET.. -> decode. -if [ "$NODE_RANK" -eq 0 ]; then - echo "NODE INFO =======================================" - echo "${host_name}:${host_ip} is Prefill Node 0 + Router" - echo "Prefill TP=${PREFILL_TP_SIZE}, Decode TP=${DECODE_TP_SIZE}" - echo "Prefill servers: ${PREFILL_ARGS}" - echo "Decode servers: ${DECODE_ARGS}" - echo "================================================" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill0_${host_name}.log & - set +x - prefill0_pid=$! - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for all servers to be up (timeout=${WAIT_SERVER_TIMEOUT}s)..." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for prefill/decode /health endpoints" - else - _deadline=$(( $(date +%s) + WAIT_SERVER_TIMEOUT )) - for _ip in "${PREFILL_IPS[@]}"; do - echo "[wait] prefill http://${_ip}:${PREFILL_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${PREFILL_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] prefill ${_ip}:${PREFILL_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] prefill ${_ip}:${PREFILL_PORT} ready" - done - for _ip in "${DECODE_IPS[@]}"; do - echo "[wait] decode http://${_ip}:${DECODE_PORT}/health" - while ! curl -sf --max-time 10 "http://${_ip}:${DECODE_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_deadline ]]; then - echo "[wait][FAIL] decode ${_ip}:${DECODE_PORT} not ready after ${WAIT_SERVER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] decode ${_ip}:${DECODE_PORT} ready" - done - fi - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "All servers up. Starting atomesh router..." - - ROUTER_CMD="/usr/local/bin/atomesh launch \ - --host 0.0.0.0 --port ${ROUTER_PORT} \ - --pd-disaggregation \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS} \ - --policy random \ - --backend atom \ - --log-level info \ - --disable-health-check \ - --disable-circuit-breaker \ - --prometheus-port 29100" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/tmp/slurm_job-${SLURM_JOB_ID}_router_${host_name}.log" - set -x - eval "$ROUTER_CMD" 2>&1 | tee "$ROUTER_LOG_FILE" & - set +x - proxy_pid=$! - - check_env_vars WAIT_LOCAL_ROUTER_TIMEOUT - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-$WAIT_LOCAL_ROUTER_TIMEOUT}" - echo "[wait] router http://0.0.0.0:${ROUTER_PORT}/v1/models (timeout=${WAIT_ROUTER_TIMEOUT}s)" - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/v1/models" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${ROUTER_PORT}/v1/models not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router /v1/models ready" - - echo "Router is ready for benchmarking" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - cd $ATOM_WS_PATH - - export IS_MTP="false" - if [[ -n "$MODEL_MTP_FLAGS" && "${DECODE_MTP_SIZE}" -gt 0 ]]; then - export IS_MTP="true" - fi - - BENCH_CMD="bash $ATOM_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - if [[ "${RUN_EVAL}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --port ${ROUTER_PORT} (framework=${EVAL_FRAMEWORK}, conc=${EVAL_CONCURRENT_REQUESTS})" - else - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" run_eval --port "${ROUTER_PORT}" - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - export DECODE_NUM_WORKERS="${yD}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - MODEL_NAME="${MODEL_DIR}/${MODEL_NAME}" append_lm_eval_summary - - fi - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - if stage_eval_artifacts \ - "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then - echo "Eval artifacts staged in $EVAL_COPY_DIR" - else - echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2 - EVAL_FAILED=1 - fi - fi - - popd - fi - fi - - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR}/logs" - mkdir -p "$LOGS_OUTPUT" - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Waiting 60s before killing router and prefill server..." - sleep 60 - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing router and prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - kill $proxy_pid - kill $prefill0_pid - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - echo "${host_name}:${host_ip} is Prefill Node (rank ${NODE_RANK})" - - prefill_worker_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_IP="${PREFILL_IPS[$prefill_worker_idx]}" - - PREFILL_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${PREFILL_PORT} \ - --trust-remote-code \ - ${PREFILL_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_producer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - eval "$PREFILL_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log & - set +x - prefill_pid=$! - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - check_env_vars WAIT_REMOTE_ROUTER_TIMEOUT - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-$WAIT_REMOTE_ROUTER_TIMEOUT}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing prefill (pid=$prefill_pid)"; kill $prefill_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing prefill server (rank ${NODE_RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $prefill_pid 2>/dev/null; fi - -else - RANK=$((NODE_RANK - NODE_OFFSET)) - echo "${host_name}:${host_ip} is Decode Node (rank ${RANK})" - - _MAX_CONC=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - CUDAGRAPH_SIZES='[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]' - - DECODE_MAX_NUM_SEQS="${_MAX_CONC}" - - DECODE_CMD="python3 -m atom.entrypoints.openai_server \ - --model ${MODEL_DIR}/${MODEL_NAME} \ - --host 0.0.0.0 --server-port ${DECODE_PORT} \ - --trust-remote-code \ - ${DECODE_PARALLEL_ARGS[*]} \ - ${SPEC_ARGS[*]} \ - ${KV_CACHE_ARG} \ - --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization ${MEM_FRAC_STATIC} \ - --max-num-seqs ${DECODE_MAX_NUM_SEQS} \ - ${MODEL_LEN_ARGS} \ - --no-enable_prefix_caching \ - ${ONLINE_QUANT_ARG} \ - --kv-transfer-config '{\"kv_role\":\"kv_consumer\",\"kv_connector\":\"mooncake\",\"proxy_ip\":\"${host_ip}\",\"handshake_port\":${HANDSHAKE_PORT}}' \ - --cudagraph-capture-sizes "${CUDAGRAPH_SIZES}" \ - ${EXTRA_SERVER_ARGS}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - eval "$DECODE_CMD" \ - 2>&1 | tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log & - set +x - decode_pid=$! - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting for router to be up..." - check_env_vars WAIT_REMOTE_ROUTER_TIMEOUT - WAIT_ROUTER_TIMEOUT="${WAIT_ROUTER_TIMEOUT:-$WAIT_REMOTE_ROUTER_TIMEOUT}" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait for router ${NODE0_ADDR}:${ROUTER_PORT}/health" - else - _router_deadline=$(( $(date +%s) + WAIT_ROUTER_TIMEOUT )) - while ! curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - if [[ $(date +%s) -ge $_router_deadline ]]; then - echo "[wait][FAIL] router ${NODE0_ADDR}:${ROUTER_PORT} not ready after ${WAIT_ROUTER_TIMEOUT}s" >&2 - exit 1 - fi - sleep 10 - done - echo "[wait][OK] router ${NODE0_ADDR}:${ROUTER_PORT} ready" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Waiting until router closes..." - trap 'echo "Caught signal, killing decode (pid=$decode_pid)"; kill $decode_pid 2>/dev/null; exit 0' SIGTERM SIGINT - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: wait until router ${NODE0_ADDR}:${ROUTER_PORT} closes" - else - while curl -sf --max-time 10 "http://${NODE0_ADDR}:${ROUTER_PORT}/health" >/dev/null 2>&1; do - sleep 10 & - wait $! - done - echo "[wait] router ${NODE0_ADDR}:${ROUTER_PORT} closed" - fi - - echo "[-------]" NODE $NODE_RANK "[--------]" - echo "Killing decode server (rank ${RANK})" - if [[ "$DRY_RUN" -eq 0 ]]; then kill $decode_pid 2>/dev/null; fi -fi - -echo "Script completed successfully" -exit 0 \ No newline at end of file diff --git a/benchmarks/multi_node/amd_utils/server_sglang.sh b/benchmarks/multi_node/amd_utils/server_sglang.sh deleted file mode 100755 index 39ab32be20..0000000000 --- a/benchmarks/multi_node/amd_utils/server_sglang.sh +++ /dev/null @@ -1,1494 +0,0 @@ -#!/bin/bash - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars \ - ENABLE_METRICS PREFILL_ROUTER_POLICY SGLANG_ROUTER_STDOUT_LOGS ROUTER_CACHE_THRESHOLD ROUTER_BALANCE_ABS_THRESHOLD \ - ROUTER_BALANCE_REL_THRESHOLD ROUTER_CANARY_TIMEOUT ROUTER_CANARY_REQ_TIMEOUT ROUTER_READINESS_CANARY ROUTER_CB_ARGS - -check_env_vars \ - NODE0_ADDR NODE_RANK MODEL_NAME xP yD \ - IPADDRS PREFILL_TP_SIZE DECODE_TP_SIZE PREFILL_ENABLE_EP PREFILL_ENABLE_DP \ - DECODE_ENABLE_EP DECODE_ENABLE_DP DECODE_MTP_SIZE BENCH_INPUT_LEN BENCH_OUTPUT_LEN \ - BENCH_RANDOM_RANGE_RATIO BENCH_REQUEST_RATE BENCH_NUM_PROMPTS_MULTIPLIER BENCH_MAX_CONCURRENCY DRY_RUN \ - GPUS_PER_NODE RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK BENCHMARK_LOGS_DIR \ - IS_AGENTIC KV_OFFLOADING MODEL_DIR SGLANG_WS_PATH HEADNODE_PORT - -# SGLang Disaggregated Server Launcher with Model-Specific Configurations - -BENCH_MAX_CONC_VALUE=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) -# Exported so the models.yaml config-loader's inline Python (eval_formula) -# can resolve formulas like "BENCH_MAX_CONC_VALUE*2" for max_running_requests. -export BENCH_MAX_CONC_VALUE - -source $SGLANG_WS_PATH/setup_deps.sh -source $SGLANG_WS_PATH/env.sh - -# Install before starting UMBP or serving processes. Early readiness failures must -# close the same owned groups as normal completion, including orphaned workers. -SGLANG_OWNED_PGIDS=() -UMBP_SA_PID="" -trap 'exit_after_background_process_cleanup "$?" 30 5 "$UMBP_SA_PID" "${SGLANG_OWNED_PGIDS[@]}"' EXIT - -host_ip=$(ip route get 1.1.1.1 | awk '/src/ {print $7}') -host_name=$(hostname) - -if [[ -n "${MORI_RDMA_TC}" ]]; then - echo "[INFO] Using MORI_RDMA_TC=$MORI_RDMA_TC for RDMA traffic class configuration" - echo "[INFO] Host '$host_name' configured with MORI_RDMA_TC=$MORI_RDMA_TC" -else - echo "[INFO] MORI_RDMA_TC not set. Skipping RDMA traffic class configuration." - echo "[INFO] This is normal for clusters without QoS requirements." -fi - -# Model-specific configuration from models.yaml -MODELS_YAML="${SGLANG_WS_PATH}/models.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -# Formula evaluation (e.g. "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK * TP * xP") is -# done in Python so bash does not glob-expand the * characters. -eval "$(python3 -c " -import yaml, sys, os - -config_path = '${MODELS_YAML}' -model_name = '${MODEL_NAME}' - -# Select the models.yaml recipe variant by run type: agentic runs (IS_AGENTIC) -# use the '-AgentX' entry, non-agentic disaggregated runs use '-DI'. -# Fall back to the bare model name if the variant-specific key is absent. -is_agentic = '${IS_AGENTIC}'.strip().lower() in ('1', 'true') -model_key = f'{model_name}-AgentX' if is_agentic else f'{model_name}-DI' - -with open(config_path) as f: - models = yaml.safe_load(f) - -if model_key not in models: - if model_name in models: - model_key = model_name - else: - print(f'echo \"ERROR: Model {model_key} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_key] -print(f'echo \"Selected models.yaml entry: {model_key} (IS_AGENTIC={is_agentic})\"') - -def eval_formula(val): - \"\"\"Evaluate chunked_prefill_size: if string, resolve variable names from env and compute.\"\"\" - if isinstance(val, (int, float)): - return int(val) - s = str(val) - # Build a namespace from env vars (convert numeric values to int) - ns = {} - for k, v in os.environ.items(): - try: - ns[k] = int(v) - except (ValueError, TypeError): - pass - try: - return int(eval(s, {'__builtins__': {}}, ns)) - except Exception as e: - print(f'echo \"WARNING: Cannot evaluate formula: {s} ({e})\"', file=sys.stderr) - return val - -def parse_range(cuda_range, default_start, default_end): - if '-' in str(cuda_range): - s, e = str(cuda_range).split('-') - # Resolve formula strings (e.g. "BENCH_MAX_CONC_VALUE/4") the same way - # chunked_prefill_size/max_running_requests do, so a range end like - # "1-BENCH_MAX_CONC_VALUE/4" doesn't reach `seq` as a literal, - # non-numeric string (which fails outright). - return str(eval_formula(s)), str(eval_formula(e)) - return str(default_start), str(default_end) - -# Output shell variables -print(f'MODEL_BASE_FLAGS=\"{m.get(\"base_flags\", \"\")}\"') -print(f'MODEL_MTP_FLAGS=\"{m.get(\"mtp_flags\", \"\")}\"') -print(f'MODEL_DSPARK_FLAGS=\"{m.get(\"dspark_flags\", \"\")}\"') -print(f'MODEL_DP_FLAGS=\"{m.get(\"dp_flags\", \"\")}\"') -print(f'MODEL_EP_FLAGS=\"{m.get(\"ep_flags\", \"\")}\"') - -prefill = m.get('prefill', {}) -decode = m.get('decode', {}) - -print(f'PREFILL_MEM_FRACTION_STATIC=\"{prefill.get(\"mem_fraction_static\", 0.8)}\"') -print(f'PREFILL_DISABLE_RADIX_CACHE=\"{prefill.get(\"disable_radix_cache\", True)}\"') -print(f'PREFILL_DISABLE_CUDA_GRAPH=\"{prefill.get(\"disable_cuda_graph\", False)}\"') - -dp = prefill.get('dp', {}) -no_dp = prefill.get('no_dp', {}) -# Per-bucket mem_fraction_static override (falls back to the role-level -# PREFILL_MEM_FRACTION_STATIC above when a model only sets one value for -# both DP and no-DP, as all but DeepSeek-V4-Pro-AgentX currently do). This is -# NOT routed through eval_formula(): that helper casts its result to int(), -# which would silently truncate a float like 0.92 down to 0. -print(f'PREFILL_MEM_FRACTION_STATIC_DP=\"{dp.get(\"mem_fraction_static\", prefill.get(\"mem_fraction_static\", 0.8))}\"') -print(f'PREFILL_MEM_FRACTION_STATIC_NO_DP=\"{no_dp.get(\"mem_fraction_static\", prefill.get(\"mem_fraction_static\", 0.8))}\"') -print(f'PREFILL_MAX_RUNNING_REQUESTS_DP=\"{eval_formula(dp.get(\"max_running_requests\", 24))}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CUDA_GRAPH_BS_DP=\"{dp.get(\"cuda_graph_bs\", \"1 2 3\")}\"') -print(f'PREFILL_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_DP=\"{dp.get(\"max_total_tokens\", \"\")}\"') -print(f'PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP=\"{dp.get(\"enable_two_batch_overlap\", False)}\"') -print(f'PREFILL_MAX_RUNNING_REQUESTS_NO_DP=\"{eval_formula(no_dp.get(\"max_running_requests\", 128))}\"') -print(f'PREFILL_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'PREFILL_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') -print(f'PREFILL_MAX_TOTAL_TOKENS_NO_DP=\"{no_dp.get(\"max_total_tokens\", \"\")}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'PREFILL_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') - -print(f'DECODE_MEM_FRACTION_STATIC=\"{decode.get(\"mem_fraction_static\", 0.85)}\"') -print(f'DECODE_DISAGG_ENABLE_RADIX_CACHE=\"{decode.get(\"disagg_decode_enable_radix_cache\", False)}\"') - -dp = decode.get('dp', {}) -ep_only = decode.get('ep_only', {}) -no_dp = decode.get('no_dp', {}) - -# Decode DP config -# Per-bucket mem_fraction_static override -- see PREFILL_MEM_FRACTION_STATIC_DP -# comment above for why this bypasses eval_formula(). -print(f'DECODE_MEM_FRACTION_STATIC_DP=\"{dp.get(\"mem_fraction_static\", decode.get(\"mem_fraction_static\", 0.85))}\"') -print(f'DECODE_MEM_FRACTION_STATIC_EP_ONLY=\"{ep_only.get(\"mem_fraction_static\", decode.get(\"mem_fraction_static\", 0.85))}\"') -print(f'DECODE_MEM_FRACTION_STATIC_NO_DP=\"{no_dp.get(\"mem_fraction_static\", decode.get(\"mem_fraction_static\", 0.85))}\"') -print(f'DECODE_MAX_RUNNING_REQUESTS_DP=\"{eval_formula(dp.get(\"max_running_requests\", 4096))}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_DP=\"{eval_formula(dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_DP=\"{dp.get(\"context_length\", \"\")}\"') -s, e = parse_range(dp.get('cuda_graph_bs_range', '1-160'), 1, 160) -print(f'DECODE_CUDA_GRAPH_BS_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_DP_END=\"{e}\"') - -# Decode EP-only config (EP enabled but DP disabled) -print(f'DECODE_MAX_RUNNING_REQUESTS_EP_ONLY=\"{ep_only.get(\"max_running_requests\", 256)}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_EP_ONLY=\"{eval_formula(ep_only.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_EP_ONLY=\"{ep_only.get(\"context_length\", \"\")}\"') -s, e = parse_range(ep_only.get('cuda_graph_bs_range', '1-256'), 1, 256) -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_EP_ONLY_END=\"{e}\"') - -# Decode no-DP config -print(f'DECODE_MAX_RUNNING_REQUESTS_NO_DP=\"{eval_formula(no_dp.get(\"max_running_requests\", 128))}\"') -print(f'DECODE_CHUNKED_PREFILL_SIZE_NO_DP=\"{eval_formula(no_dp.get(\"chunked_prefill_size\", 262144))}\"') -print(f'DECODE_CONTEXT_LENGTH_NO_DP=\"{no_dp.get(\"context_length\", \"\")}\"') -s, e = parse_range(no_dp.get('cuda_graph_bs_range', '1-128'), 1, 128) -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_START=\"{s}\"') -print(f'DECODE_CUDA_GRAPH_BS_NO_DP_END=\"{e}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -if [[ "$PREFILL_ENABLE_DP" == "true" ]]; then - prefill_cuda_graph_bs=($PREFILL_CUDA_GRAPH_BS_DP) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_DP - prefill_enable_two_batch_overlap=$PREFILL_ENABLE_TWO_BATCH_OVERLAP_DP - prefill_mem_fraction_static=$PREFILL_MEM_FRACTION_STATIC_DP -else - prefill_cuda_graph_bs=($(seq $PREFILL_CUDA_GRAPH_BS_NO_DP_START $PREFILL_CUDA_GRAPH_BS_NO_DP_END)) - prefill_max_running_requests=$PREFILL_MAX_RUNNING_REQUESTS_NO_DP - prefill_chunked_prefill_size=$PREFILL_CHUNKED_PREFILL_SIZE_NO_DP - prefill_context_length=$PREFILL_CONTEXT_LENGTH_NO_DP - prefill_max_total_tokens=$PREFILL_MAX_TOTAL_TOKENS_NO_DP - prefill_enable_two_batch_overlap="false" - prefill_mem_fraction_static=$PREFILL_MEM_FRACTION_STATIC_NO_DP -fi - -if [[ "$PREFILL_ENABLE_DP" == "true" ]] && [[ "$PREFILL_ENABLE_EP" == "true" ]]; then - prefill_max_running_requests=$BENCH_MAX_CONC_VALUE - prefill_dp_ranks=$PREFILL_TP_SIZE - echo "[DP+EP override] Prefill: max-running-requests=$prefill_max_running_requests, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" -fi - -if [[ "$DECODE_ENABLE_DP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_DP_START $DECODE_CUDA_GRAPH_BS_DP_END)) - # decode.dp.max_running_requests (YAML) is honored as an upper bound, not - # taken verbatim: the actual admissible concurrency can never exceed what - # the captured CUDA-graph range supports (cuda_graph_bs_end * TP_SIZE -- - # each DP rank runs its own copy of the graph, one request per rank per - # step). Every existing model's YAML value (4096/1024/etc.) is already - # >= this computed ceiling, so taking the min is a no-op for them; it - # only bites for configs (like DeepSeek-V4-Pro-AgentX's - # BENCH_MAX_CONC_VALUE*2 formula) that intentionally want a smaller, - # concurrency-scaled cap. - decode_max_running_requests_computed=$((DECODE_CUDA_GRAPH_BS_DP_END * DECODE_TP_SIZE)) - if [[ "$decode_max_running_requests_computed" -lt "$DECODE_MAX_RUNNING_REQUESTS_DP" ]]; then - decode_max_running_requests=$decode_max_running_requests_computed - else - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_DP - fi - echo "[decode.dp max_running_requests] computed(cuda_graph_bs_end*TP)=$decode_max_running_requests_computed yaml=$DECODE_MAX_RUNNING_REQUESTS_DP -> using $decode_max_running_requests" - decode_context_length=$DECODE_CONTEXT_LENGTH_DP - decode_mem_fraction_static=$DECODE_MEM_FRACTION_STATIC_DP -elif [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_EP_ONLY_START $DECODE_CUDA_GRAPH_BS_EP_ONLY_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_EP_ONLY - decode_context_length=$DECODE_CONTEXT_LENGTH_EP_ONLY - decode_mem_fraction_static=$DECODE_MEM_FRACTION_STATIC_EP_ONLY -else - decode_cuda_graph_bs=($(seq $DECODE_CUDA_GRAPH_BS_NO_DP_START $DECODE_CUDA_GRAPH_BS_NO_DP_END)) - decode_max_running_requests=$DECODE_MAX_RUNNING_REQUESTS_NO_DP - decode_context_length=$DECODE_CONTEXT_LENGTH_NO_DP - decode_mem_fraction_static=$DECODE_MEM_FRACTION_STATIC_NO_DP -fi -# In PD-disaggregation decode must admit requests against the SAME context length -# as prefill; otherwise decode accepts over-length requests that prefill rejects and -# they hang forever waiting for a KV transfer. Fall back to the prefill value. -if [[ -z "$decode_context_length" ]]; then - decode_context_length=$prefill_context_length -fi - -if [[ "$DECODE_ENABLE_DP" == "true" ]] && [[ "$DECODE_ENABLE_EP" == "true" ]]; then - decode_max_running_requests=$BENCH_MAX_CONC_VALUE - decode_dp_ranks=$DECODE_TP_SIZE - MORI_MAX_DISPATCH_TOKENS_DECODE=$((BENCH_MAX_CONC_VALUE / decode_dp_ranks)) - SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD=$((MORI_MAX_DISPATCH_TOKENS_DECODE * 2)) - export SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD - echo "[DP+EP override] Decode: max-running-requests=$decode_max_running_requests, DISPATCH_TOKENS=$MORI_MAX_DISPATCH_TOKENS_DECODE, MOE_MAX_INPUT=$MORI_MOE_MAX_INPUT_TOKENS_DECODE, INTER_KERNEL_SWITCH=$SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD" -fi - -# Build the composed config strings (equivalent to the old MODEL_PREFILL_CONFIGS / MODEL_DECODE_CONFIGS) -# Inspect exact registered options: newer images removed the legacy alias, while -# older images do not recognize the split phase flags. Do not rely on argparse -# prefix matching, which makes --cuda-graph-bs ambiguous on the newer images. -if ! CUDA_GRAPH_FLAGS=$(python3 "$SGLANG_WS_PATH/sglang_cli.py"); then - echo "ERROR: Could not resolve installed SGLang CUDA graph batch-size flags." >&2 - exit 1 -fi -read -r PREFILL_CUDA_GRAPH_FLAG DECODE_CUDA_GRAPH_FLAG <<< "$CUDA_GRAPH_FLAGS" -# disable_cuda_graph (model-level) keeps its existing prefill behavior. -if [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "True" ]] || [[ "$PREFILL_DISABLE_CUDA_GRAPH" == "true" ]]; then - PREFILL_MODE_FLAGS="--mem-fraction-static ${prefill_mem_fraction_static} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} --disable-cuda-graph " -else - PREFILL_MODE_FLAGS="--mem-fraction-static ${prefill_mem_fraction_static} --max-running-requests ${prefill_max_running_requests} --chunked-prefill-size ${prefill_chunked_prefill_size} ${PREFILL_CUDA_GRAPH_FLAG} ${prefill_cuda_graph_bs[*]} " -fi - -if [[ "$PREFILL_DISABLE_RADIX_CACHE" == "True" ]] || [[ "$PREFILL_DISABLE_RADIX_CACHE" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --disable-radix-cache" -fi -# Agentic runs need the radix/prefix cache. -if [[ "${IS_AGENTIC}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - PREFILL_MODE_FLAGS="${PREFILL_MODE_FLAGS//--disable-radix-cache/}" -fi -if [[ -n "$prefill_context_length" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --context-length ${prefill_context_length}" -fi -if [[ -n "$prefill_max_total_tokens" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --max-total-tokens ${prefill_max_total_tokens}" -fi -if [[ "$prefill_enable_two_batch_overlap" == "True" ]] || [[ "$prefill_enable_two_batch_overlap" == "true" ]]; then - PREFILL_MODE_FLAGS="$PREFILL_MODE_FLAGS --enable-two-batch-overlap" - PREFILL_SDMA_ENV="MORI_ENABLE_SDMA=true" -fi - -DECODE_MODE_FLAGS="--mem-fraction-static ${decode_mem_fraction_static} --max-running-requests ${decode_max_running_requests} ${DECODE_CUDA_GRAPH_FLAG} ${decode_cuda_graph_bs[*]} " - -if [[ -n "$decode_context_length" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --context-length ${decode_context_length}" -fi - -if [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "True" ]] || [[ "$DECODE_DISAGG_ENABLE_RADIX_CACHE" == "true" ]]; then - DECODE_MODE_FLAGS="$DECODE_MODE_FLAGS --disaggregation-decode-enable-radix-cache" -fi - -if [[ "$DECODE_MTP_SIZE" -gt 0 ]]; then - MORI_MAX_DISPATCH_TOKENS_DECODE=$((MORI_MAX_DISPATCH_TOKENS_DECODE * (DECODE_MTP_SIZE + 1))) -fi - -# Cluster topology -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_NODES_PER_WORKER=$(((PREFILL_TP_SIZE + 7) / GPUS_PER_NODE)) -DECODE_NODES_PER_WORKER=$(((DECODE_TP_SIZE + 7) / GPUS_PER_NODE)) -NODE_OFFSET=$((PREFILL_NODES_PER_WORKER * xP)) - -PREFILL_HEADNODE_URLS=() -PREFILL_ARGS="" -# Per-worker Prometheus /metrics endpoints for aiperf's --server-metrics scrape; -# the router on :30000 does not serve Prometheus (see ENABLE_METRICS). -SERVER_METRICS_URLS=() -# Per-worker base URLs for cache flushing between concurrency points; the router -# does not fan /flush_cache out, so trace_replay.sh must POST to each worker. -SERVER_FLUSH_URLS=() -for i in $(seq 0 $((xP - 1))); do - prefill_idx=$((i * PREFILL_NODES_PER_WORKER)) - PREFILL_HEADNODE_URLS[$i]="${IP_ARRAY[$prefill_idx]}:${HEADNODE_PORT}" - PREFILL_ARGS="$PREFILL_ARGS --prefill http://${IP_ARRAY[$prefill_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$prefill_idx]}:8000") -done - -DECODE_HEADNODE_URLS=() -DECODE_ARGS="" -for i in $(seq 0 $((yD - 1))); do - decode_idx=$((i * DECODE_NODES_PER_WORKER + NODE_OFFSET)) - DECODE_HEADNODE_URLS[$i]="${IP_ARRAY[$decode_idx]}:${HEADNODE_PORT}" - DECODE_ARGS="$DECODE_ARGS --decode http://${IP_ARRAY[$decode_idx]}:8000" - SERVER_METRICS_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000/metrics") - SERVER_FLUSH_URLS+=("http://${IP_ARRAY[$decode_idx]}:8000") -done - -echo "Prefill worker headnode list: ${PREFILL_HEADNODE_URLS[@]}" -echo "Decode worker headnode list: ${DECODE_HEADNODE_URLS[@]}" -echo "Server metrics endpoints: ${SERVER_METRICS_URLS[@]}" -echo "Server flush endpoints: ${SERVER_FLUSH_URLS[@]}" - -# KV_P2P_TRANSFER (from amd-master.yaml kv-p2p-transfer) overrides the -# --disaggregation-transfer-backend baked into models.yaml base_flags. -apply_kv_p2p_transfer_override() { - local flags="$1" - if [[ -z "${KV_P2P_TRANSFER:-}" ]]; then - printf '%s' "$flags" - return 0 - fi - local stripped - stripped="$(echo "$flags" | sed -E 's/--disaggregation-transfer-backend[[:space:]]+[^[:space:]]+//g')" - stripped="${stripped#"${stripped%%[![:space:]]*}"}" - stripped="${stripped%"${stripped##*[![:space:]]}"}" - echo "[KV_P2P] Using disaggregation-transfer-backend=${KV_P2P_TRANSFER} (KV_P2P_TRANSFER env)" >&2 - printf '%s --disaggregation-transfer-backend %s' "$stripped" "$KV_P2P_TRANSFER" -} - -build_server_config() { - local mode="$1" - local model_name="$2" - local tp_size="$3" - local enable_ep="$4" - local enable_dp="$5" - local decode_mtp_size="$6" - - local ep_size=1 - local dp_size=1 - - if [[ "$enable_ep" == "true" ]]; then - ep_size=$tp_size - fi - - if [[ "$enable_dp" == "true" ]]; then - dp_size=$tp_size - fi - - local parallel_args="--tp-size ${tp_size}" - - if [[ "$enable_ep" == "true" ]]; then - parallel_args="$parallel_args --ep-size ${ep_size}" - fi - - if [[ "$enable_dp" == "true" ]]; then - parallel_args="$parallel_args --dp-size ${dp_size}" - fi - - local base_config - base_config="$(apply_kv_p2p_transfer_override "$MODEL_BASE_FLAGS")" - local mtp_config="" - local dp_config="" - local ep_config="" - local specific_config="" - - # Speculative-decoding config (only if a draft length is set). - # - # DECODE_MTP_SIZE carries the draft length for BOTH algorithms, but the two - # spend it differently: - # EAGLE/MTP -- num-steps = draft length, i.e. that many sequential draft - # forward passes, each producing one token. - # DSPARK -- one draft pass emits a whole block, so num-steps is - # pinned to 1 and the draft length becomes the block size - # (gamma). Passing gamma as num-steps here would ask for - # gamma sequential DSpark passes instead of one gamma-token - # block. - # The verify window (num-draft-tokens = draft length + 1) is the same for - # both, which is also what makes the MORI decode dispatch scaling - # (x (DECODE_MTP_SIZE + 1)) correct for DSpark without further change. - if [ "$decode_mtp_size" -gt 0 ]; then - if [[ "${SPEC_DECODING:-}" == "draft_model" ]]; then - # MODEL_DSPARK_FLAGS is validated at the call site, not here: this - # function is only ever invoked inside $( ), where an exit would - # terminate the subshell and leave the caller with an empty config - # rather than aborting the launch. - mtp_config="${MODEL_DSPARK_FLAGS} --speculative-dspark-block-size ${decode_mtp_size} --speculative-num-steps 1 --speculative-num-draft-tokens $((decode_mtp_size + 1))" - else - mtp_config="${MODEL_MTP_FLAGS} --speculative-num-steps ${decode_mtp_size} --speculative-num-draft-tokens $((decode_mtp_size + 1))" - fi - fi - - if [[ "$enable_dp" == "true" ]]; then - dp_config="$MODEL_DP_FLAGS" - # dp_flags may override a base_flags value (e.g. --swa-full-tokens-ratio); - # strip base_config's copy so the flag appears only once on the command line. - if [[ "$dp_config" == *"--swa-full-tokens-ratio"* ]]; then - base_config="$(echo "$base_config" | sed -E 's/--swa-full-tokens-ratio[[:space:]]+[0-9.]+//')" - fi - # --disable-shared-experts-fusion and base_flags' --enforce-shared-experts-fusion - # are documented by sglang as mutually exclusive (server_args.py). Two-batch - # overlap requires the shared expert NOT be fused into the routed list, so only - # override the base_flags default in that case; strip base_config's copy so - # both flags never land on the same command line. - if [[ "$prefill_enable_two_batch_overlap" == "True" ]] || [[ "$prefill_enable_two_batch_overlap" == "true" ]]; then - dp_config="$dp_config --disable-shared-experts-fusion" - base_config="$(echo "$base_config" | sed -E 's/--enforce-shared-experts-fusion//')" - fi - fi - -# Without EP the a2a backend / deepep mode / ep-dispatch flags are dropped, so the -# MoE runs tensor-parallel even when dp-attention is on. - if [[ "$enable_ep" == "true" ]]; then - ep_config="$MODEL_EP_FLAGS" - fi - - if [[ "$mode" == "prefill" ]]; then - specific_config="$PREFILL_MODE_FLAGS" - elif [[ "$mode" == "decode" ]]; then - specific_config="$DECODE_MODE_FLAGS" - fi - - local full_config="$parallel_args" - if [[ -n "$base_config" ]]; then - full_config="$full_config $base_config" - fi - if [[ -n "$ep_config" ]]; then - full_config="$full_config $ep_config" - fi -# MTP/speculative flags go to BOTH prefill and decode: in PD-disaggregation the -# draft (nextn) layers take part in prefill KV computation, so the PD state component -# count must match. sglang v0.5.15+ rejects a mismatch ("state component count -# mismatch"); older builds silently fed decode uninitialized nextn state (lossy MTP). - if [[ -n "$mtp_config" ]]; then - full_config="$full_config $mtp_config" - fi - if [[ -n "$dp_config" ]]; then - full_config="$full_config $dp_config" - fi - if [[ -n "$specific_config" ]]; then - full_config="$full_config $specific_config" - fi - - echo "$full_config" -} - -# Validate the DSpark path before building either config. This has to happen at -# top level: build_server_config only ever runs inside $( ), so an exit there -# would kill the subshell and hand the caller an empty config string instead of -# stopping the launch. -if [[ "$DECODE_MTP_SIZE" -gt 0 ]] && [[ "${SPEC_DECODING:-}" == "draft_model" ]] && [[ -z "${MODEL_DSPARK_FLAGS// }" ]]; then - echo "FATAL: SPEC_DECODING=draft_model but model '${MODEL_NAME}' has no dspark_flags in models.yaml." >&2 - exit 1 -fi - -PREFILL_SERVER_CONFIG=$(build_server_config "prefill" "$MODEL_NAME" "$PREFILL_TP_SIZE" "$PREFILL_ENABLE_EP" "$PREFILL_ENABLE_DP" "$DECODE_MTP_SIZE") -DECODE_SERVER_CONFIG=$(build_server_config "decode" "$MODEL_NAME" "$DECODE_TP_SIZE" "$DECODE_ENABLE_EP" "$DECODE_ENABLE_DP" "$DECODE_MTP_SIZE") - -if [[ "${ENABLE_METRICS}" == "1" ]]; then - [[ "$PREFILL_SERVER_CONFIG" != *"--enable-metrics"* ]] && PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG --enable-metrics" - [[ "$DECODE_SERVER_CONFIG" != *"--enable-metrics"* ]] && DECODE_SERVER_CONFIG="$DECODE_SERVER_CONFIG --enable-metrics" -fi - -if [[ -n "$MODEL_NAME" ]]; then - echo "Using model-specific configuration for: $MODEL_NAME" -fi - -# sync.py server-up barrier timeout; DSV4 needs more headroom. -if [[ -z "${SYNC_BARRIER_TIMEOUT:-}" ]]; then - case "${MODEL_NAME}" in - *DeepSeek-V4*) SYNC_BARRIER_TIMEOUT=3000 ;; - *) SYNC_BARRIER_TIMEOUT=1800 ;; - esac -fi -echo "SYNC_BARRIER_TIMEOUT=${SYNC_BARRIER_TIMEOUT}s (model=${MODEL_NAME})" - -KV_OFFLOAD_BACKEND="${KV_OFFLOAD_BACKEND:-}" -if [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == "hicache" ]]; then - - # Optional L3 storage tier behind the CPU-DRAM (L2) cache. - # "" -> CPU DRAM only (default) - # "mooncake"-> Mooncake distributed KV store (needs a mooncake_master) - HICACHE_STORAGE_BACKEND="${HICACHE_STORAGE_BACKEND:-}" - - check_env_vars HICACHE_HOST_POOL_COUNT HICACHE_PAGE_SIZE HICACHE_PREFETCH_POLICY \ - HICACHE_IO_BACKEND HICACHE_WRITE_POLICY HICACHE_RATIO FORCE_HICACHE_RATIO \ - HICACHE_L2_MEM_LAYOUT HICACHE_L3_MEM_LAYOUT - if [[ -z "${HICACHE_MEM_LAYOUT:-}" ]]; then -# The mooncake L3 store maps a page-contiguous segment for RDMA/zero-copy, so it -# needs the page_first layout with the direct IO backend; that layout asserts -# host_pool > device_pool, so it needs a large CPU-DRAM budget. - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - HICACHE_MEM_LAYOUT="$HICACHE_L3_MEM_LAYOUT" - else - HICACHE_MEM_LAYOUT="$HICACHE_L2_MEM_LAYOUT" - fi - fi - - check_env_vars \ - MC_MASTER_PORT MC_METADATA_PORT MC_METRICS_PORT MC_MASTER_THREADS MC_EVICTION_HIGH_WATERMARK \ - MC_PROTOCOL MC_GLOBAL_SEG - MC_DEVICE="${MC_DEVICE:-$IBDEVICES}" - MC_MASTER_ADDR="${MC_MASTER_ADDR:-${NODE0_ADDR}:${MC_MASTER_PORT}}" - MC_METADATA_SERVER="${MC_METADATA_SERVER:-http://${NODE0_ADDR}:${MC_METADATA_PORT}/metadata}" - -# The extra-config JSON is single-quoted so it survives the later eval of the -# launch command as a single argument. - build_storage_flags() { - [[ "$HICACHE_STORAGE_BACKEND" != "mooncake" ]] && return 0 - local extra="{\"master_server_address\": \"${MC_MASTER_ADDR}\", \"protocol\": \"${MC_PROTOCOL}\", \"device_name\": \"${MC_DEVICE}\", \"local_hostname\": \"${host_ip}\", \"global_segment_size\": \"${MC_GLOBAL_SEG}\", \"metadata_server\": \"${MC_METADATA_SERVER}\", \"check_server\": false}" - echo "--hicache-storage-backend mooncake --hicache-storage-backend-extra-config '${extra}' --enable-metrics --enable-cache-report" - } - - HICACHE_SIZING_FLAGS="--hicache-ratio ${HICACHE_RATIO}" -# DeepSeek V4's hybrid HiCache pool rejects --hicache-size (ratio only): -# https://github.com/sgl-project/sglang/blob/9dd57ef8c48e2cd82292d849f01e2130c5203e67/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py#L262-L266 - if [[ "${FORCE_HICACHE_RATIO}" != "1" && -n "${TOTAL_CPU_DRAM_GB:-}" && "${TOTAL_CPU_DRAM_GB}" -gt 0 && "${MODEL_NAME}" != *DeepSeek-V4* ]]; then - # TOTAL_CPU_DRAM_GB is the prefill worker's per-node budget; --hicache-size is - # per rank per host pool. A prefill server may span nodes, so divide by the - # ranks that land on one node. - prefill_ranks_per_node=$(( PREFILL_TP_SIZE < GPUS_PER_NODE ? PREFILL_TP_SIZE : GPUS_PER_NODE )) - prefill_hicache_size_gb=$(( TOTAL_CPU_DRAM_GB / prefill_ranks_per_node / HICACHE_HOST_POOL_COUNT )) - if (( prefill_hicache_size_gb < 1 )); then - echo "Error: TOTAL_CPU_DRAM_GB=${TOTAL_CPU_DRAM_GB} / ranks_per_node=${prefill_ranks_per_node} / host_pools=${HICACHE_HOST_POOL_COUNT} rounds below 1 GB" >&2 - exit 1 - fi - HICACHE_SIZING_FLAGS="--hicache-size ${prefill_hicache_size_gb}" - echo "[HiCache] prefill CPU pool capped at ${prefill_hicache_size_gb} GB/rank (budget ${TOTAL_CPU_DRAM_GB} GB / ranks_per_node ${prefill_ranks_per_node} / host_pools ${HICACHE_HOST_POOL_COUNT})" - fi - - build_hicache_flags() { - echo "--page-size ${HICACHE_PAGE_SIZE} --enable-hierarchical-cache ${HICACHE_SIZING_FLAGS} --hicache-io-backend ${HICACHE_IO_BACKEND} --hicache-mem-layout ${HICACHE_MEM_LAYOUT} --hicache-write-policy ${HICACHE_WRITE_POLICY} --hicache-storage-prefetch-policy ${HICACHE_PREFETCH_POLICY} $(build_storage_flags)" - } - - # HiCache requires RadixAttention; strip any --disable-radix-cache. - PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" - DECODE_SERVER_CONFIG="${DECODE_SERVER_CONFIG//--disable-radix-cache/}" - - PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG $(build_hicache_flags "$PREFILL_TP_SIZE")" - - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND} applied to prefill only" - echo "[HiCache] params: io_backend=${HICACHE_IO_BACKEND}, mem_layout=${HICACHE_MEM_LAYOUT}, page_size=${HICACHE_PAGE_SIZE}, write_policy=${HICACHE_WRITE_POLICY}, prefetch_policy=${HICACHE_PREFETCH_POLICY}, storage_backend=${HICACHE_STORAGE_BACKEND:-none}" - if [[ "$HICACHE_STORAGE_BACKEND" == "mooncake" ]]; then - echo "[HiCache] Mooncake store: master=${MC_MASTER_ADDR} metadata=${MC_METADATA_SERVER} protocol=${MC_PROTOCOL} device=${MC_DEVICE} segment=${MC_GLOBAL_SEG} threads=${MC_MASTER_THREADS} eviction_watermark=${MC_EVICTION_HIGH_WATERMARK}" - fi -elif [[ "$KV_OFFLOADING" != "none" && "$KV_OFFLOAD_BACKEND" == umbp-linker* ]]; then - # ========================================================================= - # UMBP as a DIRECT external store for the unified radix tree (PD disagg). - # - # Ported from benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh, - # which is where this arm exists today. It is a SEPARATE sglang code path - # from the HiCache branch above, not a variation of it: the tree loads and - # offloads pages against UMBP with NO host cache tier in between, and - # sglang rejects the combination outright (server_args.py::_handle_hicache - # raises when --enable-hierarchical-cache or --hicache-storage-backend is - # set alongside it). So none of the L2 knobs apply here or may be passed, - # and this branch deliberately shares no code with the one above. - # - # PREFILL ONLY, exactly like HiCache on this path: only the prefill worker - # offloads KV, and the tier metric (sglang:prefill_effective_tokens) is - # emitted by the prefill engine alone. Decode is untouched -- it already - # carries --page-size 256 from the DeepSeek-V4-Pro-AgentX base_flags in - # models.yaml, so it needs no mirror flag the way the HiCache branch does - # (that branch sets HICACHE_PAGE_SIZE and has to restate it). - # ========================================================================= - - # DP-ONLY ON PURPOSE, same refusal the single-node recipe carries. Under - # pure TP the linker's object keys carry a per-rank suffix and MLA KV is - # replicated across TP, so a TP8 prefill worker yields EIGHT keyspaces and - # the tier holds eight copies of the same tokens -- its effective - # distinct-token capacity is an eighth of what the byte budget suggests. - # Under DP attention the keys collapse to tp0 and the tier is one shared - # keyspace. Refuse rather than silently measure a derated tier: a result - # file from the derated arm is indistinguishable from a real one. - if [[ "$PREFILL_ENABLE_DP" != "true" ]]; then - echo "Error: KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' is supported only with prefill dp-attn: true. Under pure TP the linker keyspace is per rank, so the tier holds TP copies of the same tokens and the arm measures a different system than the DP one." >&2 - exit 1 - fi - - # Multi-node prefill workers are not supported here. The tier is a - # per-node process and each node would hold its own keyspace, so a prefill - # worker spanning nodes would silently shard the store by node. - if [[ "$PREFILL_NODES_PER_WORKER" -ne 1 ]]; then - echo "Error: KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' supports single-node prefill workers only (PREFILL_NODES_PER_WORKER=${PREFILL_NODES_PER_WORKER}); the UMBP tier is a per-node process and a multi-node prefill worker would shard its keyspace by node." >&2 - exit 1 - fi - - # Upstream renamed both the flag pair and the class when this line was - # rebased (unified-tree-connector -> unified-cache-external-linker, - # UMBPTreeConnector -> UMBPDirectLinker). Detect rather than pin: the image - # decides which vocabulary is valid, and a pinned name means editing this - # file on every image bump. Search the whole sglang.srt TREE, not - # server_args.py alone. - SGLANG_SRT_DIR="$(python3 -c 'import importlib.util as u, os; s = u.find_spec("sglang.srt.server_args"); print(os.path.dirname(s.origin) if s else "")' 2>/dev/null)" - [[ -d "${SGLANG_SRT_DIR:-}" ]] || { echo "Error: cannot locate the installed sglang.srt tree" >&2; exit 1; } - echo "[UMBP] probing $SGLANG_SRT_DIR for the linker flag vocabulary" - if grep -rqs "enable_unified_cache_external_linker" "$SGLANG_SRT_DIR"; then - UMBP_LINKER_FLAGS="--enable-unified-cache-external-linker --unified-cache-external-linker-backend mori" - elif grep -rqs "enable_unified_tree_connector" "$SGLANG_SRT_DIR"; then - UMBP_LINKER_FLAGS="--enable-unified-tree-connector --unified-tree-connector-backend mori" - else - echo "Error: this image's sglang exposes neither --enable-unified-cache-external-linker nor --enable-unified-tree-connector, so it cannot drive UMBP as a direct external store. Use a linker-capable image (e.g. rocm/mori-dev:sglang-0.5.19-rocm720-mi35x-mori-0908-pr38269-638c6a61)." >&2 - exit 1 - fi - - # ---- Tier capacity ---------------------------------------------------- - # 1.5 TB, the same NODE total the single-node linker arms run with, so a - # PD linker number can be read against them without restating the size. - # TOTAL_CPU_DRAM_GB is NOT the bound here: that is the HiCache budget the - # sweep generator hands down (available-cpu-dram-mib scaled by - # dram-utilization) and the HiCache control arm does not even apply it - # (FORCE_HICACHE_RATIO=1 makes it size by ratio instead). The guard that - # matters is the box's own memory: the 806 GB checkpoint's page cache, the - # sglang ranks and the co-located AIPerf client all live in what is left, - # so refuse a tier above half of MemTotal. - UMBP_DRAM_BYTES="${UMBP_DRAM_BYTES:-1500000000000}" - UMBP_DRAM_GB=$((UMBP_DRAM_BYTES / 1000000000)) - UMBP_HOST_MEMTOTAL_GB=$(awk '/^MemTotal:/ {printf "%d", $2 / 1000000}' /proc/meminfo) - echo "[UMBP] tier sizing: ${UMBP_DRAM_GB} GB requested, host MemTotal ${UMBP_HOST_MEMTOTAL_GB} GB, ceiling $((UMBP_HOST_MEMTOTAL_GB / 2)) GB (TOTAL_CPU_DRAM_GB=${TOTAL_CPU_DRAM_GB:-unset} is the HiCache budget and does not bound this arm)" - if [[ "$UMBP_DRAM_GB" -gt "$((UMBP_HOST_MEMTOTAL_GB / 2))" ]]; then - echo "Error: UMBP tier ${UMBP_DRAM_GB} GB exceeds half of the host's ${UMBP_HOST_MEMTOTAL_GB} GB MemTotal; the checkpoint's page cache and the server's working set need the rest. Lower UMBP_DRAM_BYTES." >&2 - exit 1 - fi - # These nodes run with HugePages_Total=0 and the allocator silently demotes - # to 4 KiB pages. The linker registers the GPU KV buffers, not the host - # pool, so small pages cost locality here, not correctness. - UMBP_DRAM_USE_HUGEPAGES="${UMBP_DRAM_USE_HUGEPAGES:-0}" - - # ---- Standalone server, prefill nodes only ---------------------------- - # server_sglang.sh runs on every node; only the prefill nodes need a tier. - # NODE_RANK < NODE_OFFSET is exactly the prefill-node test the launch - # dispatch further down uses. - if [[ "$NODE_RANK" -lt "$NODE_OFFSET" ]]; then - # The container's own /tmp (the HOST /tmp is bind-mounted at /run_logs), - # so the socket dies with the container and cannot collide with another - # runner on this node. - UMBP_SA_DIR="${UMBP_SA_DIR:-/tmp/umbp_sa_${SLURM_JOB_ID:-$$}}" - mkdir -p "$UMBP_SA_DIR" - export UMBP_STANDALONE_ADDRESS="${UMBP_STANDALONE_ADDRESS:-unix://${UMBP_SA_DIR}/sa.grpc.sock}" - UMBP_SA_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/umbp_standalone_$(hostname).log" - - # Take the standalone server from the mori that is actually importable, - # not a stale copy elsewhere in the image: client and server must agree - # on capabilities or the linker aborts with "requires a standalone - # server whose inner backend advertises ranged multi-buffer I/O - # support" -- which reads like a mori version problem but means the two - # halves disagree. - if [[ -z "${UMBP_SA_BIN:-}" ]]; then - for _cand in \ - "$(python3 -c 'import os, mori; print(os.path.dirname(os.path.realpath(mori.__file__)))' 2>/dev/null)/umbp_standalone_server" \ - /sgl-workspace/mori/python/mori/umbp_standalone_server \ - /sgl-workspace/mori/build_umbp/src/umbp/umbp_standalone_server; do - if [[ -x "$_cand" ]]; then UMBP_SA_BIN="$_cand"; break; fi - done - fi - [[ -x "${UMBP_SA_BIN:-}" ]] || { echo "Error: umbp_standalone_server not found in this image; it does not ship UMBP standalone mode." >&2; exit 1; } - echo "[UMBP] standalone server binary: $UMBP_SA_BIN" - export LD_LIBRARY_PATH="$(dirname "$UMBP_SA_BIN"):${LD_LIBRARY_PATH:-}" - - echo "[UMBP] starting standalone server at $UMBP_STANDALONE_ADDRESS (tier ${UMBP_DRAM_GB} GB, hugepages=${UMBP_DRAM_USE_HUGEPAGES}), log -> $UMBP_SA_LOG" - # UMBP_SSD_ENABLED is atoi()'d by the server (UMBPConfig:: - # FromEnvironment), so it needs 1/0 -- atoi("true") is 0, which happens - # to be right but only by accident. - env UMBP_DRAM_CAPACITY="$UMBP_DRAM_BYTES" \ - UMBP_DRAM_USE_HUGEPAGES="$UMBP_DRAM_USE_HUGEPAGES" \ - UMBP_SSD_ENABLED=0 \ - MORI_UMBP_LOG_LEVEL="${MORI_UMBP_LOG_LEVEL:-info}" \ - "$UMBP_SA_BIN" "$UMBP_STANDALONE_ADDRESS" > "$UMBP_SA_LOG" 2>&1 & - UMBP_SA_PID=$! - echo "[UMBP] standalone server PID: $UMBP_SA_PID" - - # Three waits, all bounded by wall time rather than by a guess at how - # fast this node is. Bind time for a 549 GB tier measured 120 s on - # n08-21 and over 300 s on n09-25 -- same hardware, but n09-25 was - # holding 1.9 TB of page cache and the allocation had to reclaim - # through it. At 1.5 TB that spread only widens, so the ceiling is - # generous; a dead server is still caught in the first second by the - # kill -0 probe, so a generous ceiling costs nothing when something is - # actually broken. - UMBP_SA_WAIT_SECONDS="${UMBP_SA_WAIT_SECONDS:-1800}" - UMBP_SA_SOCK="${UMBP_STANDALONE_ADDRESS#unix://}" - - # 1. The socket appears as soon as grpc listens. - UMBP_SA_T0=$SECONDS - UMBP_SA_READY=false - for _ in $(seq 1 "$UMBP_SA_WAIT_SECONDS"); do - if ! kill -0 "$UMBP_SA_PID" 2>/dev/null; then - echo "[UMBP] standalone server died during startup. Log follows:" >&2 - cat "$UMBP_SA_LOG" >&2 || true - exit 1 - fi - [[ -S "$UMBP_SA_SOCK" ]] && { UMBP_SA_READY=true; break; } - sleep 1 - done - [[ "$UMBP_SA_READY" == "true" ]] || { echo "Error: UMBP standalone server never bound $UMBP_SA_SOCK within ${UMBP_SA_WAIT_SECONDS} s" >&2; cat "$UMBP_SA_LOG" >&2 || true; exit 1; } - echo "[UMBP] bound $UMBP_SA_SOCK after $((SECONDS - UMBP_SA_T0)) s" - - # 2. But the socket is bound before the server can serve: the DRAM tier - # still has to register its host memory. sglang launched into that - # window dies at linker construction with - # RuntimeError: StandaloneProcessClient: server is not ready - # and takes the whole arm with it, minutes in, for a reason that has - # nothing to do with what the run was measuring. "data plane" is the - # first line the server prints once it will answer. - UMBP_SA_T1=$SECONDS - UMBP_SA_SERVING=false - for _ in $(seq 1 "$UMBP_SA_WAIT_SECONDS"); do - if ! kill -0 "$UMBP_SA_PID" 2>/dev/null; then - echo "[UMBP] standalone server died while registering its tier. Log follows:" >&2 - cat "$UMBP_SA_LOG" >&2 || true - exit 1 - fi - grep -q "data plane" "$UMBP_SA_LOG" 2>/dev/null && { UMBP_SA_SERVING=true; break; } - sleep 1 - done - [[ "$UMBP_SA_SERVING" == "true" ]] || { echo "Error: UMBP standalone server bound $UMBP_SA_SOCK but never reached its data plane within ${UMBP_SA_WAIT_SECONDS} s" >&2; cat "$UMBP_SA_LOG" >&2 || true; exit 1; } - echo "[UMBP] data plane up after $((SECONDS - UMBP_SA_T1)) s ($SECONDS s total): $(grep -m1 'data plane' "$UMBP_SA_LOG")" - - # 3. And the data plane answers before the tier is usable from the GPU. - # HostTierRegistration hands hipHostRegister to a worker thread for any - # tier above its sync threshold, so "data plane" can print with the - # region still unpinned -- and mori says what that costs: "the GPU - # gather path stays off and copies fall back to pageable hipMemcpy". An - # arm that starts serving inside that window measures the fallback path - # for its first several minutes. - if [[ "${UMBP_SA_WAIT_REGISTERED:-1}" == "1" ]]; then - UMBP_SA_T2=$SECONDS - UMBP_SA_REGISTERED=false - for _ in $(seq 1 "$UMBP_SA_WAIT_SECONDS"); do - if ! kill -0 "$UMBP_SA_PID" 2>/dev/null; then - echo "[UMBP] standalone server died while registering its tier for GPU access. Log follows:" >&2 - cat "$UMBP_SA_LOG" >&2 || true - exit 1 - fi - if grep -q "host memory registered for GPU access" "$UMBP_SA_LOG" 2>/dev/null; then - UMBP_SA_REGISTERED=true - break - fi - if grep -q "hipHostRegister of .* failed" "$UMBP_SA_LOG" 2>/dev/null; then - echo "Error: hipHostRegister failed for the UMBP tier; every copy would take the pageable fallback path" >&2 - grep -m1 "hipHostRegister of .* failed" "$UMBP_SA_LOG" >&2 || true - exit 1 - fi - sleep 1 - done - [[ "$UMBP_SA_REGISTERED" == "true" ]] || { echo "Error: the UMBP tier was still not registered for GPU access after ${UMBP_SA_WAIT_SECONDS} s" >&2; exit 1; } - echo "[UMBP] tier registered for GPU access after $((SECONDS - UMBP_SA_T2)) s past the data plane: $(grep -m1 'host memory registered for GPU access' "$UMBP_SA_LOG")" - fi - else - echo "[UMBP] node rank ${NODE_RANK} runs decode only; no tier here (offload is prefill-side on this path)" - fi - - # The linker requires RadixAttention, same as HiCache; strip any - # --disable-radix-cache from the prefill config. - PREFILL_SERVER_CONFIG="${PREFILL_SERVER_CONFIG//--disable-radix-cache/}" - - # Device KV pool left at whatever mem-fraction-static profiles, same as the - # HiCache control, so the linker is compared against it at an IDENTICAL - # pool rather than at a capped one. UMBP_MAX_TOTAL_TOKENS caps it if the - # profiled pool swallows the whole working set and the arm ends up - # measuring nothing about UMBP -- sglang takes min(requested, profiled), so - # it can only shrink the pool, and the effective value has to be read back - # from the server log either way. - UMBP_POOL_FLAGS="" - [[ -n "${UMBP_MAX_TOTAL_TOKENS:-}" ]] && UMBP_POOL_FLAGS="--max-total-tokens ${UMBP_MAX_TOTAL_TOKENS}" - - # StandaloneProcess drops the client-side sizing keys: the server owns the - # tier and takes UMBP_DRAM_CAPACITY from its own environment, so the extra - # config is empty. It is still passed because the linker reads the flag. - # Single-quoted so it survives the later `eval` of the launch command as - # one argument, matching build_storage_flags() above. - PREFILL_SERVER_CONFIG="$PREFILL_SERVER_CONFIG ${UMBP_LINKER_FLAGS} ${UMBP_POOL_FLAGS} --hicache-storage-backend-extra-config '{}' --enable-cache-report" - - echo "[UMBP] direct linker on prefill: tier=${UMBP_DRAM_GB} GB, address=${UMBP_STANDALONE_ADDRESS:-}, prefill tp=${PREFILL_TP_SIZE} dp-attn=${PREFILL_ENABLE_DP}, device pool=${UMBP_MAX_TOTAL_TOKENS:-profiled}, no host cache tier" - echo "[UMBP] flags: ${UMBP_LINKER_FLAGS} ${UMBP_POOL_FLAGS}" - echo "[UMBP] decode untouched; --page-size 256 already comes from the models.yaml base_flags" -else - echo "[HiCache] KV_OFFLOADING=${KV_OFFLOADING} backend=${KV_OFFLOAD_BACKEND:-none} (HiCache disabled)" -fi - -if [[ "${EVAL_ONLY}" == "true" ]] || [[ "${RUN_EVAL}" == "true" ]]; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed 's/--ep-dispatch-algorithm fake//g') - unset MORI_MOE_MAX_INPUT_TOKENS_PREFILL - unset MORI_MOE_MAX_INPUT_TOKENS_DECODE -fi - -# sync.py barrier exits 1 on timeout, but without an explicit check the script -# would continue past a timed-out barrier and launch the next stage against -# servers/routers that never came up. -run_barrier_or_die() { - local desc="$1" cmd="$2" - if ! eval "$cmd"; then - echo "FATAL: ${desc} failed — see the sync.py timeout output above for which node/port never became ready." >&2 - exit 1 - fi -} - -echo "Waiting at the container creation barrier on $host_name" -# The 300s default is too tight on the umbp-linker path: rank 0 does not open -# port 5000 until umbp_standalone_server has registered the whole DRAM tier for -# GPU access, which is strongly node-dependent (305.7s on one node vs >780s on -# another). The peer that came up first then times out and kills an otherwise -# healthy run. Raise it per-arm via CONTAINER_BARRIER_TIMEOUT, above -# UMBP_SA_WAIT_SECONDS so UMBP's own wait is the binding one, not the barrier. -# Unset keeps the historical 300s for every other arm. -run_barrier_or_die "container creation barrier" "python3 $SGLANG_WS_PATH/sync.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout ${CONTAINER_BARRIER_TIMEOUT:-300}" - -# Node role assignment and server launch - -# Run a blocking command while watching the local server PID. If the server dies -# the command is aborted and we return non-zero, so SLURM's --kill-on-bad-exit -# tears the job down in seconds instead of waiting out the barrier timeout. -wait_or_die() { # $1 = server pid to watch; rest = blocking command - local watch=$1; shift - "$@" & local cmd=$! - while kill -0 "$cmd" 2>/dev/null; do - kill -0 "$watch" 2>/dev/null || { - echo "FATAL: $(hostname) local sglang server (pid $watch) died; tearing down job" >&2 - kill "$cmd" 2>/dev/null || true - return 1 - } - sleep 5 - done - wait "$cmd" -} - -if [ "$NODE_RANK" -eq 0 ]; then - echo "NODE INFO =======================================" - echo "================================================" - echo "Node List : ${SLURM_JOB_NODELIST}" - echo "Node IPs : ${IPADDRS}" - echo "Model Name : ${MODEL_NAME}" - echo "================================================" - - echo "CLUSTER INFO ====================================" - echo "================================================" - echo "${host_name}:${host_ip} is Proxy Node and Prefill Node" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}, MTP size=${DECODE_MTP_SIZE}" - echo "Prefill servers ($((PREFILL_TP_SIZE/GPUS_PER_NODE)) nodes): ${PREFILL_ARGS}" - echo "Decode servers ($((DECODE_TP_SIZE/GPUS_PER_NODE)) nodes): ${DECODE_ARGS}" - echo "Prefill env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_PREFILL}" - echo "Decode env: SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_MAX_DISPATCH_TOKENS_DECODE} " - echo "Decode env: SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE} " - - echo "================================================" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - echo "# Model: ${MODEL_NAME} Image: ${DOCKER_IMAGE_NAME:-unknown}" >> "$CMD_DUMP" - - if [[ "${KV_OFFLOADING}" != "none" && "${KV_OFFLOAD_BACKEND:-}" == "hicache" && "${HICACHE_STORAGE_BACKEND:-}" == "mooncake" ]]; then - echo "Starting Mooncake master on ${host_ip}:${MC_MASTER_PORT} (metadata :${MC_METADATA_PORT}, metrics :${MC_METRICS_PORT})" - MC_MASTER_CMD="mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port=${MC_METADATA_PORT} \ - --rpc_port=${MC_MASTER_PORT} \ - --rpc_thread_num=${MC_MASTER_THREADS} \ - --metrics_port=${MC_METRICS_PORT} \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio=${MC_EVICTION_HIGH_WATERMARK}" - dump_cmd "MOONCAKE MASTER" "$MC_MASTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $MC_MASTER_CMD" - else - MC_MASTER_LOG="/run_logs/slurm_job-${SLURM_JOB_ID}/mooncake_master_${host_name}.log" - mooncake_master \ - --enable_http_metadata_server=true \ - --http_metadata_server_host=0.0.0.0 \ - --http_metadata_server_port="${MC_METADATA_PORT}" \ - --rpc_port="${MC_MASTER_PORT}" \ - --rpc_thread_num="${MC_MASTER_THREADS}" \ - --metrics_port="${MC_METRICS_PORT}" \ - --enable_metric_reporting=true \ - --eviction_high_watermark_ratio="${MC_EVICTION_HIGH_WATERMARK}" \ - > "${MC_MASTER_LOG}" 2>&1 & - mc_master_pid=$! - sleep 3 - # On shared nodes the Mooncake RPC port may already be held by another - # user's master; the metrics-port check below can then pass against the - # foreign master while our RPC port is dead, and prefill hangs. - if grep -qiE "Address already in use|bind .*error" "${MC_MASTER_LOG}" 2>/dev/null; then - echo "ERROR: mooncake_master failed to bind port ${MC_MASTER_PORT} (already in use)." - echo " Set MC_MASTER_PORT/MC_METRICS_PORT to free ports and resubmit." - grep -iE "Address already in use|bind .*error" "${MC_MASTER_LOG}" | tail -3 - exit 1 - fi - for ((i=3; i<=60; i+=3)); do - if curl -sf "http://127.0.0.1:${MC_METRICS_PORT}/get_all_segments" >/dev/null 2>&1; then - echo " mooncake master OK at ${i}s" - break - fi - sleep 3 - done - fi - fi - - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/$MODEL_NAME \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[0]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank 0" - fi - - dump_cmd "PREFILL (node 0)" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - # setsid puts the server and its TP-scheduler children in one process group so - # teardown can kill -- -$pgid the whole tree. Killing $prefill0_pid alone leaves - # children holding the tee pipe, so the container's outer | tee never gets EOF - # and the container never exits. Process substitution keeps $! as the setsid - # group leader rather than tee's pid. - setsid bash -c "$PREFILL_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log >/dev/null) 2>&1 & - set +x - prefill0_pid=$! - prefill0_pgid=$prefill0_pid - SGLANG_OWNED_PGIDS+=("$prefill0_pgid") - fi - - echo "Waiting for all prefill and decode servers to be up . . ." - - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports 8000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$prefill0_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - echo "Congratulations!!! All prefill and decode servers are up . . ." - - if [[ "${IS_AGENTIC}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - check_env_vars ROUTER_RESILIENCE_FLAGS - ROUTER_PREFILL_POLICY="${PREFILL_ROUTER_POLICY}" - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:---policy ${ROUTER_PREFILL_POLICY} --dp-aware --cache-threshold ${ROUTER_CACHE_THRESHOLD} --balance-abs-threshold ${ROUTER_BALANCE_ABS_THRESHOLD} --balance-rel-threshold ${ROUTER_BALANCE_REL_THRESHOLD}}" - else - check_env_vars ROUTER_DEFAULT_POLICY_FLAGS - ROUTER_POLICY_FLAGS="${ROUTER_POLICY_FLAGS:-$ROUTER_DEFAULT_POLICY_FLAGS}" - ROUTER_RESILIENCE_FLAGS="${ROUTER_RESILIENCE_FLAGS:-${ROUTER_CB_ARGS}}" - fi - - echo "Router config: IS_AGENTIC=${IS_AGENTIC} policy/resilience=${ROUTER_POLICY_FLAGS} ${ROUTER_RESILIENCE_FLAGS}" - - ROUTER_CMD="python -m sglang_router.launch_router \ - --pd-disaggregation \ - --port 30000 \ - ${ROUTER_POLICY_FLAGS} \ - ${ROUTER_RESILIENCE_FLAGS} \ - ${PREFILL_ARGS} \ - ${DECODE_ARGS}" - - dump_cmd "ROUTER" "$ROUTER_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $ROUTER_CMD" - else - ROUTER_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/router_${host_name}.log" - # sgl-router (Rust/tracing) emits ANSI color codes; NO_COLOR asks it to stop and - # the sed strip guarantees a clean file either way. Process substitution keeps $! - # as the router pid. sglang-router >=0.5.14 spawns the Rust worker (binds :30000) - # as a child and lets the python launcher exit, so the worker reparents to init - # but keeps its process group: launch under setsid and record the pgid so teardown - # can kill -- -$proxy_pgid after the launcher is gone. - set -x - if [[ "${SGLANG_ROUTER_STDOUT_LOGS}" == "1" ]]; then - NO_COLOR=1 setsid bash -c "exec $ROUTER_CMD" > >(sed -u -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' | tee "$ROUTER_LOG_FILE") 2>&1 & - else - NO_COLOR=1 setsid bash -c "exec $ROUTER_CMD" > >(sed -u -r 's/\x1b\[[0-9;]*[a-zA-Z]//g' >"$ROUTER_LOG_FILE") 2>&1 & - fi - set +x - proxy_pid=$! - proxy_pgid=$proxy_pid - SGLANG_OWNED_PGIDS+=("$proxy_pgid") - - HEALTH_BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-health \ - --health-endpoint /readiness \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - wait_or_die "$prefill0_pid" bash -c "$HEALTH_BARRIER_CMD" || exit 1 - fi - - # /readiness only proves the router process is up, not that it can reach a - # prefill worker and complete a generation; an eval started on /readiness alone - # 503'd every request ("all circuits open or unhealthy") and produced no results. - # Gate on one successful generation through the router. Runs under wait_or_die - # so a prefill crash right after /readiness aborts in seconds instead of burning - # ROUTER_CANARY_TIMEOUT on repeated 503s. - run_router_canary() { - local canary_url="http://${NODE0_ADDR}:30000/v1/chat/completions" - local canary_model="${MODEL_DIR}/${MODEL_NAME}" - local canary_deadline=$(( $(date +%s) + ${ROUTER_CANARY_TIMEOUT} )) - local canary_code - while [ "$(date +%s)" -lt "$canary_deadline" ]; do - canary_code=$(curl -s -o /tmp/router_canary.out -w '%{http_code}' \ - -m "${ROUTER_CANARY_REQ_TIMEOUT}" \ - -X POST "$canary_url" -H 'Content-Type: application/json' \ - -d "{\"model\":\"${canary_model}\",\"messages\":[{\"role\":\"user\",\"content\":\"ping\"}],\"max_tokens\":1,\"temperature\":0}" 2>/dev/null) - if [ "$canary_code" = "200" ] && \ - ! grep -qE "circuits open|server_selection_failed|No available" /tmp/router_canary.out 2>/dev/null; then - echo "Router readiness canary passed (end-to-end generation OK)" - return 0 - fi - echo "Router readiness canary not ready yet (http=${canary_code}); retrying in 5s . . ." - sleep 5 - done - echo "ERROR: router readiness canary failed after ${ROUTER_CANARY_TIMEOUT}s -- the router cannot complete a generation through a prefill worker (all circuits open/unhealthy). Refusing to start the eval against a non-serving router." - head -c 800 /tmp/router_canary.out 2>/dev/null - return 1 - } - if [[ "${ROUTER_READINESS_CANARY}" == "1" ]]; then - wait_or_die "$prefill0_pid" run_router_canary || exit 1 - fi - - echo "Router is ready for benchmarking" - fi - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - - echo "Benchmarking on ${host_name}:${host_ip}" - cd $SGLANG_WS_PATH - - if [ "$DECODE_MTP_SIZE" -gt 0 ]; then - export IS_MTP=true - else - export IS_MTP=false - fi - - if [[ "${IS_AGENTIC}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - # aiperf auto-detects the router from --url, which does not expose Prometheus; - # point the scrape at the per-worker /metrics endpoints or every server-side - # cache/KV field comes out null. - if [[ "${ENABLE_METRICS}" == "1" && "${#SERVER_METRICS_URLS[@]}" -gt 0 ]]; then - AIPERF_SERVER_METRICS_URLS=$(IFS=,; echo "${SERVER_METRICS_URLS[*]}") - export AIPERF_SERVER_METRICS_URLS - echo "AIPERF_SERVER_METRICS_URLS=${AIPERF_SERVER_METRICS_URLS}" - fi - # trace_replay.sh flushes these workers directly when CLEAR_CACHE_BETWEEN_CONC=1. - if [[ "${#SERVER_FLUSH_URLS[@]}" -gt 0 ]]; then - SERVER_FLUSH_URLS_CSV=$(IFS=,; echo "${SERVER_FLUSH_URLS[*]}") - export SERVER_FLUSH_URLS_CSV - echo "SERVER_FLUSH_URLS_CSV=${SERVER_FLUSH_URLS_CSV}" - fi - # trace_replay.sh signature: model_path model_name concurrency_list log_path - BENCH_CMD="bash $SGLANG_WS_PATH/trace_replay.sh \ - $MODEL_DIR $MODEL_NAME $BENCH_MAX_CONCURRENCY /run_logs/slurm_job-${SLURM_JOB_ID}" - echo "Benchmark runner: trace_replay.sh (agentic, KV_OFFLOADING=${KV_OFFLOADING}, backend=${KV_OFFLOAD_BACKEND:-none}, CONC=${BENCH_MAX_CONCURRENCY})" - else - # bench.sh signature: - # n_prefill n_decode prefill_gpus decode_gpus model_dir model_name log_path - # isl osl concurrency_list req_rate random_range_ratio num_prompts_multiplier - BENCH_CMD="bash $SGLANG_WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - echo "Benchmark runner: bench.sh (fixed-seq-len)" - fi - - IS_AGENTIC_RUN=0 - if [[ "${IS_AGENTIC}" == "1" || "${IS_AGENTIC:-}" == "true" ]]; then - IS_AGENTIC_RUN=1 - fi - - BENCHMARK_EXIT_CODE=0 - if [[ "${EVAL_ONLY}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - elif [[ -n "${CLIENT_IMAGE:-}" && "$IS_AGENTIC_RUN" == "1" ]]; then - # With CLIENT_IMAGE set, the aiperf trace replay runs in a sibling container - # (pre-baked aiperf) on this node against the router over --network host. - # job.slurm mounts the docker socket and forwards HOST_REPO_DIR / HOST_MODEL_DIR / - # HOST_BENCH_LOGS / CLIENT_CONT_NAME for this. - CLIENT_ENV_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/client.env" - mkdir -p "/run_logs/slurm_job-${SLURM_JOB_ID}" - check_env_vars INFERENCEX_RUNTIME_ENV_VARS - # Unset vars are skipped so the client keeps its own defaults. - { - for _v in $INFERENCEX_RUNTIME_ENV_VARS \ - ENGINE MODEL_NAME MODEL_PREFIX PRECISION FRAMEWORK SPEC_DECODING \ - DURATION MAX_MODEL_LEN RESULT_FILENAME RUNNER_NAME RUNNER_TYPE IMAGE MODEL_PATH \ - AIPERF_SERVER_METRICS_URLS SERVER_FLUSH_URLS_CSV \ - ENABLE_METRICS IS_AGENTIC CLEAR_CACHE_BETWEEN_CONC FLUSH_DRAIN_TIMEOUT \ - DISAGG IS_MULTINODE \ - TP EP_SIZE DP_ATTENTION DCP_SIZE PCP_SIZE \ - PREFILL_NUM_WORKERS PREFILL_TP PREFILL_EP PREFILL_DP_ATTN PREFILL_ENABLE_DP PREFILL_HARDWARE \ - DECODE_NUM_WORKERS DECODE_TP DECODE_EP DECODE_DP_ATTN DECODE_ENABLE_DP DECODE_HARDWARE \ - KV_OFFLOADING KV_OFFLOAD_BACKEND KV_OFFLOAD_BACKEND_METADATA TOTAL_CPU_DRAM_GB KV_P2P_TRANSFER \ - WEKA_LOADER_OVERRIDE AIPERF_FAILED_REQUEST_THRESHOLD \ - AIPERF_WARMUP_REQUESTS_PER_LANE AIPERF_TRACE_IDLE_GAP_CAP_SECONDS \ - AIPERF_EXPERIMENTAL_FAST AIPERF_UNSAFE_OVERRIDE \ - AIPERF_TRAJECTORY_START_MIN_RATIO AIPERF_TRAJECTORY_START_MAX_RATIO \ - AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES ROUTER_PORT TQDM_MININTERVAL; do - if [[ -n "${!_v+x}" ]]; then - _val="${!_v}" - # docker --env-file needs one KEY=VALUE per line; KV_OFFLOAD_BACKEND_METADATA - # carries pretty-printed multi-line JSON, so re-serialize it compact via - # json.loads/json.dumps. Empty/"none"/"null" means no metadata (job.slurm - # always sets the var) and must pass through untouched, matching - # optional_kv_offload_backend_metadata() in process_agentic_result.py. - if [[ "$_v" == "KV_OFFLOAD_BACKEND_METADATA" && -n "$_val" && "$_val" != "null" ]]; then - _val="$(python3 -c 'import json, sys -print(json.dumps(json.loads(sys.stdin.read())))' <<<"$_val")" || { - echo "KV_OFFLOAD_BACKEND_METADATA must contain valid JSON" >&2 - exit 1 - } - fi - printf '%s=%s\n' "$_v" "$_val" - fi - done - echo "INFMAX_CONTAINER_WORKSPACE=/workspace" - # AGENTIC_OUTPUT_DIR is deliberately not pinned: it must default to /workspace - # (the host repo mount) so ${RESULT_FILENAME}_conc.json lands where the - # workflow guard globs it. - echo "HF_HOME=/run_logs/hf_cache" - echo "MODEL_DIR=/models" - # Without a pre-baked venv (CLIENT_AIPERF_VENV unset, e.g. reusing the server - # image) trace_replay builds aiperf from /workspace/utils/aiperf. - if [[ -n "${CLIENT_AIPERF_VENV:-}" ]]; then - echo "AIPERF_USE_PREBUILT=1" - echo "AIPERF_VENV=${CLIENT_AIPERF_VENV}" - fi - } > "$CLIENT_ENV_FILE" - - echo "Launching agentic benchmark in separate client container: ${CLIENT_IMAGE}" - docker rm -f "${CLIENT_CONT_NAME}" 2>/dev/null || true - set -x - docker run --rm --network host \ - --name "${CLIENT_CONT_NAME}" \ - --shm-size 32G \ - -v "${HOST_REPO_DIR}:/workspace" \ - -v "${HOST_MODEL_DIR}:/models" \ - -v /tmp:/run_logs \ - -v "${HOST_BENCH_LOGS}:/benchmark_logs" \ - --env-file "${CLIENT_ENV_FILE}" \ - --entrypoint "" \ - "${CLIENT_IMAGE}" \ - bash -lc "cd /workspace/benchmarks/multi_node/amd_utils && bash trace_replay.sh /models ${MODEL_NAME} \"${BENCH_MAX_CONCURRENCY}\" /run_logs/slurm_job-${SLURM_JOB_ID}" - BENCHMARK_EXIT_CODE=$? - set +x - else - set -x - eval "$BENCH_CMD" - BENCHMARK_EXIT_CODE=$? - set +x - fi - - if [[ "${RUN_EVAL}" == "true" ]]; then - echo "Running lm-eval (GSM8K) evaluation on Node 0..." - - # The throughput benchmark may have crashed decode workers; skip eval if so. - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:30000/readiness" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - # Must run from repo root so infx/evals/gsm8k.yaml resolves - pushd /workspace - - # Match the disaggregation router launched above. - export PORT=30000 - source /workspace/benchmarks/benchmark_lib.sh - - # CONC must be exported before run_eval so meta_env.json matches validate_scores.py. - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - export CONC="${EVAL_CONCURRENT_REQUESTS}" - - if [[ -n "$prefill_context_length" ]]; then - export EVAL_MAX_MODEL_LEN="$prefill_context_length" - fi - - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - bridge_disagg_eval_metadata - # IS_MULTINODE, FRAMEWORK, PRECISION, MODEL_PREFIX, RUNNER_TYPE, RESULT_FILENAME - # arrive via Docker -e flags from job.slurm. - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --port ${PORT} (framework=${EVAL_FRAMEWORK}, conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - run_eval --port "$PORT" - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2 - EVAL_FAILED=1 - else - # Always rewrite meta_env.json so EP/DPA match the workflow - # topology even when run_eval() staged artifacts internally. - rewrite_lm_eval_meta_env - - # Fixed-seq-len post-bench eval still needs append to move - # results out of the temp EVAL_RESULT_DIR. - if [[ "${EVAL_ONLY}" != "true" || "$IS_AGENTIC_RUN" != "1" ]]; then - append_lm_eval_summary - fi - - fi - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - if stage_eval_artifacts \ - "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then - echo "Eval artifacts staged in $EVAL_COPY_DIR" - else - echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2 - EVAL_FAILED=1 - fi - fi - - popd - fi - fi - - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - node_exit_status=$BENCHMARK_EXIT_CODE - if [[ "${EVAL_FAILED:-0}" -eq 1 && "$node_exit_status" -eq 0 ]]; then - node_exit_status=1 - fi - exit "$node_exit_status" - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$NODE_OFFSET" ]; then - echo "${host_name}:${host_ip} is Prefill Node (Model: ${MODEL_NAME})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - echo "Prefill parallelism: TP=${PREFILL_TP_SIZE}, EP enabled: ${PREFILL_ENABLE_EP}, DP enabled: ${PREFILL_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - PREFILL_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_PREFILL" ]]; then - PREFILL_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_PREFILL}" - fi - set +x - PREFILL_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_PREFILL} ${PREFILL_SDMA_ENV} ${PREFILL_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_PREFILL:-${MORI_MAX_DISPATCH_TOKENS_PREFILL}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path $MODEL_DIR/${MODEL_NAME} \ - --disaggregation-mode prefill \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${PREFILL_SERVER_CONFIG} " - - if [ "$PREFILL_NODES_PER_WORKER" -gt 1 ]; then - rank=$((NODE_RANK % PREFILL_NODES_PER_WORKER)) - prefill_idx=$((NODE_RANK / PREFILL_NODES_PER_WORKER)) - PREFILL_CMD="$PREFILL_CMD --dist-init-addr ${PREFILL_HEADNODE_URLS[$prefill_idx]} --nnodes ${PREFILL_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "PREFILL (rank ${NODE_RANK})" "$PREFILL_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - set -x - # setsid isolates the server tree so teardown can group-kill python + TP-scheduler - # children; otherwise they hold the tee pipe and the container never exits. - setsid bash -c "$PREFILL_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log >/dev/null) 2>&1 & - set +x - prefill_pid=$! - prefill_pgid=$prefill_pid - SGLANG_OWNED_PGIDS+=("$prefill_pgid") - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$prefill_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - wait_or_die "$prefill_pid" bash -c "$WAIT_CMD" || exit 1 - fi - - echo "Killing the rank $NODE_RANK prefill server" - - if [[ "$DRY_RUN" -eq 0 ]]; then - exit 0 - fi - -else - RANK=$((NODE_RANK - xP * PREFILL_NODES_PER_WORKER)) - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - echo "Decode node rank: $RANK" - echo "Decode parallelism: TP=${DECODE_TP_SIZE}, EP enabled: ${DECODE_ENABLE_EP}, DP enabled: ${DECODE_ENABLE_DP}" - - CMD_DUMP="/run_logs/slurm_job-${SLURM_JOB_ID}/commands_${host_name}.txt" - dump_cmd() { echo -e "\n# ── $1 ──\n$2" >> "$CMD_DUMP"; } - echo "# Commands dump — $(date -u '+%Y-%m-%d %H:%M:%S UTC')" > "$CMD_DUMP" - echo "# Host: ${host_name} (${host_ip}) Node rank: ${NODE_RANK}" >> "$CMD_DUMP" - - DECODE_MORI_MOE_ENV="" - set -x - if [[ -n "$MORI_MOE_MAX_INPUT_TOKENS_DECODE" ]]; then - DECODE_MORI_MOE_ENV="SGLANG_MORI_MOE_MAX_INPUT_TOKENS=${MORI_MOE_MAX_INPUT_TOKENS_DECODE}" - fi - set +x - - # Agentic trace replay doesn't reproduce real token-by-token traffic, so - # measured MTP/EAGLE acceptance there isn't representative (PR #2309 - # review: https://github.com/SemiAnalysisAI/InferenceX/pull/2309#pullrequestreview-4778348624). - # Per the AgentX fairness guidelines (golden_al_distribution/README.md), - # agentic throughput benchmarks simulate acceptance at the model's - # committed golden AL instead of measuring real (non-representative) - # acceptance. Eval runs (RUN_EVAL / EVAL_ONLY) need real acceptance so - # GSM8K scores reflect actual MTP behavior. AgentX uses one golden curve - # per checkpoint, thinking mode, and draft length, including when the - # supported PD draft implementation differs from the calibration engine. - # Sources (thinking_on): dsv4_mtp.yaml for the original checkpoint and - # golden_al_distribution/dsv4-pro-0813-dspark.yaml for Pro-0813. - DECODE_SIM_ACC_ENV="" - if [[ "$DECODE_MTP_SIZE" -gt 0 ]] && { [[ "${IS_AGENTIC}" == "1" ]] || [[ "${IS_AGENTIC:-}" == "true" ]]; }; then - if [[ "${EVAL_ONLY}" == "true" ]] || [[ "${RUN_EVAL}" == "true" ]]; then - echo "[INFO] Eval mode: synthetic MTP disabled (using real acceptance)" - else - DSV4_GOLDEN_AL="" - case "${MODEL_NAME}:${DECODE_MTP_SIZE}" in - DeepSeek-V4-Pro-0813:1) DSV4_GOLDEN_AL=1.84 ;; - DeepSeek-V4-Pro-0813:2) DSV4_GOLDEN_AL=2.51 ;; - DeepSeek-V4-Pro-0813:3) DSV4_GOLDEN_AL=3.01 ;; - DeepSeek-V4-Pro-0813:*) - echo "ERROR: Pro-0813 draft length ${DECODE_MTP_SIZE} has no golden AL wired here; refusing to use the original V4 curve." >&2 - exit 1 - ;; - *DeepSeek-V4*:1) DSV4_GOLDEN_AL=1.79 ;; - *DeepSeek-V4*:2) DSV4_GOLDEN_AL=2.27 ;; - *DeepSeek-V4*:3) DSV4_GOLDEN_AL=2.49 ;; - esac - if [[ -n "$DSV4_GOLDEN_AL" ]]; then - DECODE_SIM_ACC_ENV="SGLANG_SIMULATE_ACC_LEN=${DSV4_GOLDEN_AL} SGLANG_SIMULATE_ACC_METHOD=match-expected SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token" - else - echo "WARNING: agentic spec-decoding run (model=${MODEL_NAME}, algorithm=${SPEC_DECODING:-mtp}, DECODE_MTP_SIZE=${DECODE_MTP_SIZE}) has no golden AL wired in server_sglang.sh -- falling back to real (unsimulated, non-representative) acceptance. Add a case in server_sglang.sh and golden_al_distribution/ before shipping this arm. See golden_al_distribution/README.md." >&2 - fi - fi - fi - - DECODE_CMD="SGLANG_MORI_COMBINE_DTYPE=${MORI_COMBINE_DTYPE_DECODE} ${DECODE_MORI_MOE_ENV} SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK=${MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK_DECODE:-${MORI_MAX_DISPATCH_TOKENS_DECODE}} MORI_IO_SQ_BACKOFF_TIMEOUT_US=${MORI_IO_SQ_BACKOFF_TIMEOUT_US} MORI_IO_QP_MAX_SEND_WR=${MORI_IO_QP_MAX_SEND_WR} ${DECODE_SIM_ACC_ENV} ${LAUNCH_PREFIX:-} python3 -m sglang.launch_server \ - --model-path ${MODEL_DIR}/${MODEL_NAME} \ - --disaggregation-mode decode \ - --disaggregation-ib-device ${IBDEVICES} \ - --host 0.0.0.0 \ - --port 8000 \ - --trust-remote-code \ - ${DECODE_SERVER_CONFIG} " - - if [ "$DECODE_NODES_PER_WORKER" -gt 1 ]; then - rank=$((RANK % DECODE_NODES_PER_WORKER)) - decode_idx=$((RANK / DECODE_NODES_PER_WORKER)) - DECODE_CMD="$DECODE_CMD --dist-init-addr ${DECODE_HEADNODE_URLS[$decode_idx]} --nnodes ${DECODE_NODES_PER_WORKER} --node-rank $rank" - fi - - dump_cmd "DECODE (rank ${NODE_RANK})" "$DECODE_CMD" - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - set -x - # setsid isolates the server tree so teardown can group-kill python + TP-scheduler - # children; otherwise they hold the tee pipe and the container never exits. - setsid bash -c "$DECODE_CMD" \ - > >(tee /run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log >/dev/null) 2>&1 & - - set +x - decode_pid=$! - decode_pgid=$decode_pid - SGLANG_OWNED_PGIDS+=("$decode_pgid") - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $SGLANG_WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports 30000 \ - --wait-for-all-ports \ - --timeout ${SYNC_BARRIER_TIMEOUT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - wait_or_die "$decode_pid" bash -c "$BARRIER_CMD" || exit 1 - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $SGLANG_WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port 30000" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - wait_or_die "$decode_pid" bash -c "$WAIT_CMD" || exit 1 - fi - - echo "Killing the rank $RANK decode server" - if [[ "$DRY_RUN" -eq 0 ]]; then - exit 0 - fi - -fi - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/amd_utils/server_vllm.sh b/benchmarks/multi_node/amd_utils/server_vllm.sh deleted file mode 100755 index 158bdefc62..0000000000 --- a/benchmarks/multi_node/amd_utils/server_vllm.sh +++ /dev/null @@ -1,485 +0,0 @@ -#!/bin/bash - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only - -check_env_vars \ - NODE0_ADDR NODE_RANK MODEL_NAME xP yD \ - IPADDRS PREFILL_TP_SIZE DECODE_TP_SIZE PREFILL_ENABLE_EP PREFILL_ENABLE_DP \ - DECODE_ENABLE_EP DECODE_ENABLE_DP BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_RANDOM_RANGE_RATIO \ - BENCH_REQUEST_RATE BENCH_NUM_PROMPTS_MULTIPLIER BENCH_MAX_CONCURRENCY DRY_RUN GPUS_PER_NODE \ - RUN_EVAL EVAL_ONLY EVAL_FRAMEWORK BENCHMARK_LOGS_DIR MODEL_DIR \ - WS_PATH ROUTER_PORT SERVER_PORT PROXY_PING_PORT MODEL_PATH - -# vLLM Disaggregated Server Launcher with Model-Specific Configurations -# -# Node role assignment (by NODE_RANK): -# 0 -> Proxy/Router + first Prefill node (kv_producer) -# 1..xP-1 -> Additional Prefill nodes (kv_producer) -# xP..xP+yD-1 -> Decode nodes (kv_consumer) -# -# Total nodes = xP + yD (router co-located with first prefill, like SGLang). - -# setup_deps.sh is idempotent; required on the base vLLM image. -source "$(dirname "${BASH_SOURCE[0]}")/setup_deps.sh" - -# Prefer MODEL_PATH from job.slurm (handles HF cache snapshot resolution) - -source $WS_PATH/env.sh - -host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7}') -# RDMA IP for Nixl KV transfer (prefer 192.168.x.x subnet if available) -rdma_ip=$(hostname -I | tr ' ' '\n' | grep '^192\.168\.' | head -1) -rdma_ip="${rdma_ip:-$host_ip}" -host_name=$(hostname) - -echo "[INFO] Management IP (barriers/proxy): $host_ip" -echo "[INFO] RDMA IP (Nixl KV transfer): $rdma_ip" - -setup_rdma_env() { - # Pensando ionic (RoCEv2) point-to-point /31 route fix. - # Each benic interface has a /31 to the TOR switch. Without explicit routes, - # traffic to other nodes' RDMA IPs falls through to the management network. - if [[ "$rdma_ip" =~ ^192\.168\.([0-9]+)\.([0-9]+)$ ]]; then - local rdma_subnet="${BASH_REMATCH[1]}" - local rdma_host="${BASH_REMATCH[2]}" - local rdma_gw="192.168.${rdma_subnet}.$(( rdma_host | 1 ))" - local rdma_iface - rdma_iface=$(ip -o addr show | awk -v ip="$rdma_ip" '$4 ~ ip {print $2}' | head -1) - if [[ -n "$rdma_iface" ]]; then - ip route replace "192.168.${rdma_subnet}.0/24" via "$rdma_gw" dev "$rdma_iface" 2>/dev/null && \ - echo "[RDMA-ROUTE] Added 192.168.${rdma_subnet}.0/24 via $rdma_gw dev $rdma_iface" || \ - echo "[RDMA-ROUTE] Route add failed for 192.168.${rdma_subnet}.0/24" - fi - fi - - # Nixl UCX backend: ucx_error_handling_mode=none. Under high concurrency (C512+) - # UCX's default UCP_ERR_HANDLING_MODE_PEER runs transport-level error recovery on - # ibv_post_send failures, which stops RIXL RDMA READ retries from recovering; the - # prefill KV cache then fills to 100% and the pipeline deadlocks. Needed on every - # NIC type, not just ionic. - local nixl_api - nixl_api=$(python3 -c "import rixl._api; print(rixl._api.__file__)" 2>/dev/null) - if [[ -n "$nixl_api" ]]; then - if ! grep -q 'ucx_error_handling_mode' "$nixl_api"; then - sed -i '/self\.create_backend(bknd, init)/i\ init["ucx_error_handling_mode"] = "none"' "$nixl_api" - echo "[PATCH] Added ucx_error_handling_mode=none to $nixl_api (IBDEVICES=${IBDEVICES:-unset})" - else - echo "[PATCH] ucx_error_handling_mode already set in $nixl_api" - fi - fi -} - -setup_rdma_env - -if [[ -z "$UCX_NET_DEVICES" ]]; then - echo "Error: UCX_NET_DEVICES is empty after env.sh detection" >&2 - exit 1 -fi - -MODELS_YAML="${WS_PATH}/models_vllm.yaml" - -if [[ ! -f "$MODELS_YAML" ]]; then - echo "ERROR: models.yaml not found at $MODELS_YAML" - exit 1 -fi - -if [[ -z "$MODEL_NAME" ]]; then - echo "ERROR: MODEL_NAME is not set"; exit 1 -fi - -eval "$(python3 -c " -import yaml, sys - -with open('${MODELS_YAML}') as f: - models = yaml.safe_load(f) - -model_name = '${MODEL_NAME}' -if model_name not in models: - print(f'echo \"ERROR: Model {model_name} not in models.yaml\"; exit 1') - sys.exit(0) - -m = models[model_name] - -def bash_escape(s): - \"\"\"Escape a value for safe embedding in a bash double-quoted assignment.\"\"\" - return s.replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"').replace('\$', '\\\\\$').replace('\`', '\\\\\`') - -pf = bash_escape(m.get('prefill_flags', '--tensor-parallel-size 8')) -df = bash_escape(m.get('decode_flags', '--tensor-parallel-size 8')) -ev = bash_escape(m.get('env', '')) -dev = bash_escape(m.get('decode_env', '')) -pev = bash_escape(m.get('prefill_env', '')) -print(f'PREFILL_SERVER_CONFIG=\"{pf}\"') -print(f'DECODE_SERVER_CONFIG=\"{df}\"') -print(f'MODEL_ENVS=\"{ev}\"') -print(f'DECODE_MODEL_ENVS=\"{dev}\"') -print(f'PREFILL_MODEL_ENVS=\"{pev}\"') -")" - -echo "Loaded model configuration for: $MODEL_NAME" - -if [[ -n "${PREFILL_TP_SIZE:-}" ]]; then - if echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - PREFILL_SERVER_CONFIG=$(echo "$PREFILL_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${PREFILL_TP_SIZE}/g") - else - PREFILL_SERVER_CONFIG+=" --tensor-parallel-size ${PREFILL_TP_SIZE}" - fi -fi -if [[ -n "${DECODE_TP_SIZE:-}" ]]; then - if echo "$DECODE_SERVER_CONFIG" | grep -q -- '--tensor-parallel-size'; then - DECODE_SERVER_CONFIG=$(echo "$DECODE_SERVER_CONFIG" | sed -E "s/--tensor-parallel-size[[:space:]]+[0-9]+/--tensor-parallel-size ${DECODE_TP_SIZE}/g") - else - DECODE_SERVER_CONFIG+=" --tensor-parallel-size ${DECODE_TP_SIZE}" - fi -fi -if [[ "${PREFILL_ENABLE_EP}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - PREFILL_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${PREFILL_ENABLE_DP}" == "true" ]] && ! echo "$PREFILL_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - PREFILL_SERVER_CONFIG+=" --enable-dp-attention" -fi -if [[ "${DECODE_ENABLE_EP}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-expert-parallel'; then - DECODE_SERVER_CONFIG+=" --enable-expert-parallel" -fi -if [[ "${DECODE_ENABLE_DP}" == "true" ]] && ! echo "$DECODE_SERVER_CONFIG" | grep -q -- '--enable-dp-attention'; then - DECODE_SERVER_CONFIG+=" --enable-dp-attention" -fi - -echo "PREFILL_SERVER_CONFIG (after TP/EP/DP): $PREFILL_SERVER_CONFIG" -echo "DECODE_SERVER_CONFIG (after TP/EP/DP): $DECODE_SERVER_CONFIG" - -echo "Waiting at the container creation barrier on $host_name" -python3 $WS_PATH/sync.py barrier \ - --local-ip ${host_ip} \ - --local-port 5000 \ - --enable-port \ - --node-ips ${IPADDRS} \ - --node-ports 5000 \ - --wait-for-all-ports \ - --timeout 600 - -IFS=',' read -ra IP_ARRAY <<< "$IPADDRS" - -PREFILL_ARGS="" -DECODE_ARGS="" - -for ((i=0; i "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for all prefill and decode servers to be up . . ." - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: skipping barrier (wait-for-all-ports)" - else - python3 $WS_PATH/sync.py barrier \ - --node-ips ${IPADDRS} \ - --node-ports $SERVER_PORT \ - --wait-for-all-ports \ - --timeout 1800 - fi - - echo "Congratulations!!! All prefill and decode servers are up . . ." - - HEALTH_BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-health \ - --health-endpoint /health \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $HEALTH_BARRIER_CMD" - else - eval "$HEALTH_BARRIER_CMD" - echo "MoRI-IO proxy is ready for benchmarking" - fi - - echo "Ready for benchmarking on ${host_name}:${host_ip}" - echo "Benchmarking on ${host_name}:${host_ip}" - cd $WS_PATH - - export ROUTER_PORT=$ROUTER_PORT - BENCH_CMD="bash $WS_PATH/bench.sh ${xP} ${yD} $((PREFILL_TP_SIZE*xP)) $((DECODE_TP_SIZE*yD)) \ - $MODEL_DIR $MODEL_NAME /run_logs/slurm_job-${SLURM_JOB_ID} ${BENCH_INPUT_LEN} \ - ${BENCH_OUTPUT_LEN} \"${BENCH_MAX_CONCURRENCY}\" ${BENCH_REQUEST_RATE} \ - ${BENCH_RANDOM_RANGE_RATIO} ${BENCH_NUM_PROMPTS_MULTIPLIER}" - - if [[ "${EVAL_ONLY}" == "true" ]]; then - echo "EVAL_ONLY mode: skipping throughput benchmark" - elif [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BENCH_CMD" - else - set -x - eval "$BENCH_CMD" - set +x - fi - - if [[ "${RUN_EVAL}" == "true" ]]; then - echo "Running lm-eval evaluation on Node 0..." - - EVAL_HEALTH_OK=false - for _attempt in 1 2 3; do - if curl -sf --max-time 10 "http://0.0.0.0:${ROUTER_PORT}/health" >/dev/null 2>&1; then - EVAL_HEALTH_OK=true - break - fi - echo "Eval health check attempt $_attempt failed, retrying in 10s..." - sleep 10 - done - - if [[ "$EVAL_HEALTH_OK" != "true" ]]; then - echo "WARNING: Router health check failed after 3 attempts. Skipping eval." - else - pushd /workspace - - source /workspace/benchmarks/benchmark_lib.sh - - if [[ -n "${EVAL_CONC:-}" ]]; then - export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" - else - export EVAL_CONCURRENT_REQUESTS=$(echo "$BENCH_MAX_CONCURRENCY" | tr 'x' '\n' | sort -n | tail -1) - fi - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: run_eval --port $ROUTER_PORT (framework=${EVAL_FRAMEWORK}, conc=${EVAL_CONCURRENT_REQUESTS}, ctx=${EVAL_MAX_MODEL_LEN:-auto})" - else - run_eval --port "$ROUTER_PORT" - eval_rc=$? - - if [[ $eval_rc -ne 0 ]]; then - echo "ERROR: run_eval exited rc=$eval_rc; preserving failure artifacts" >&2 - EVAL_FAILED=1 - else - export TP="${PREFILL_TP_SIZE}" - export CONC="${EVAL_CONCURRENT_REQUESTS}" - export EP_SIZE=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && EP_SIZE="${PREFILL_TP_SIZE}" - export PREFILL_TP="${PREFILL_TP_SIZE}" - export PREFILL_EP=1 - [[ "${PREFILL_ENABLE_EP}" == "true" ]] && PREFILL_EP="${PREFILL_TP_SIZE}" - export PREFILL_NUM_WORKERS="${xP}" - export DECODE_TP="${DECODE_TP_SIZE}" - export DECODE_EP=1 - [[ "${DECODE_ENABLE_EP}" == "true" ]] && DECODE_EP="${DECODE_TP_SIZE}" - export DECODE_NUM_WORKERS="${yD}" - export DP_ATTENTION="${PREFILL_ENABLE_DP}" - export PREFILL_DP_ATTENTION="${PREFILL_ENABLE_DP}" - export DECODE_DP_ATTENTION="${DECODE_ENABLE_DP}" - export ISL="${BENCH_INPUT_LEN}" - export OSL="${BENCH_OUTPUT_LEN}" - - append_lm_eval_summary - - fi - - EVAL_COPY_DIR="/run_logs/slurm_job-${SLURM_JOB_ID}/eval_results" - if stage_eval_artifacts \ - "$EVAL_COPY_DIR" /workspace "${EVAL_RESULT_DIR:-}"; then - echo "Eval artifacts staged in $EVAL_COPY_DIR" - else - echo "ERROR: failed to stage eval artifacts in $EVAL_COPY_DIR" >&2 - EVAL_FAILED=1 - fi - fi - - popd - fi - fi - - LOGS_OUTPUT="${BENCHMARK_LOGS_DIR}/logs" - mkdir -p "$LOGS_OUTPUT" - - if [[ "$DRY_RUN" -eq 0 ]]; then - cp -r /run_logs/slurm_job-${SLURM_JOB_ID} "$LOGS_OUTPUT/" - echo "Copied results to $LOGS_OUTPUT/slurm_job-${SLURM_JOB_ID}" - fi - - echo "Killing the prefill server" - if [[ "$DRY_RUN" -eq 0 ]]; then - [[ -n "${prefill_pid:-}" ]] && kill $prefill_pid 2>/dev/null || true - sleep 2 - pkill -f "vllm serve" 2>/dev/null || true - fi - - if [[ "${EVAL_FAILED:-0}" -eq 1 ]]; then - echo "ERROR: eval failed; exiting node-0 with rc=1" - exit 1 - fi - -elif [ "$NODE_RANK" -gt 0 ] && [ "$NODE_RANK" -lt "$xP" ]; then - echo "${host_name}:${host_ip} is Additional Prefill Node (Model: ${MODEL_NAME})" - echo "Using prefill config: $PREFILL_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${PREFILL_MODEL_ENVS}; do - export "$env_pair" - echo "[PREFILL_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - PREFILL_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_producer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${PREFILL_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $PREFILL_CMD" - else - PREFILL_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/prefill_${host_name}.log" - set -x - eval "$PREFILL_CMD" > "$PREFILL_LOG_FILE" 2>&1 & - set +x - prefill_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the prefill server" - [[ "$DRY_RUN" -eq 0 ]] && kill $prefill_pid 2>/dev/null || true - -else - echo "${host_name}:${host_ip} is Decode Node (Model: ${MODEL_NAME})" - echo "Using decode config: $DECODE_SERVER_CONFIG" - - setup_vllm_env - - for env_pair in ${DECODE_MODEL_ENVS}; do - export "$env_pair" - echo "[DECODE_ENV] $env_pair" - done - - SERVED_MODEL="${MODEL_NAME}" - DECODE_CMD="vllm serve ${MODEL_PATH} \ - --served-model-name ${SERVED_MODEL} \ - --port $SERVER_PORT \ - --trust-remote-code \ - --kv-transfer-config '{\"kv_connector\": \"MoRIIOConnector\", \"kv_role\": \"kv_consumer\", \"kv_connector_extra_config\": {\"proxy_ip\": \"${NODE0_ADDR}\", \"proxy_ping_port\": \"${PROXY_PING_PORT}\", \"http_port\": \"${SERVER_PORT}\", \"read_mode\": true}}' \ - ${DECODE_SERVER_CONFIG}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $DECODE_CMD" - else - DECODE_LOG_FILE="/run_logs/slurm_job-${SLURM_JOB_ID}/decode_${host_name}.log" - set -x - eval "$DECODE_CMD" > "$DECODE_LOG_FILE" 2>&1 & - set +x - decode_pid=$! - fi - - echo "Waiting for proxy server to be up..." - BARRIER_CMD="python3 $WS_PATH/sync.py barrier \ - --node-ips ${NODE0_ADDR} \ - --node-ports ${ROUTER_PORT} \ - --wait-for-all-ports \ - --timeout 1800" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $BARRIER_CMD" - else - eval "$BARRIER_CMD" - fi - - echo "Waiting until proxy server closes..." - WAIT_CMD="python3 $WS_PATH/sync.py wait \ - --remote-ip ${NODE0_ADDR} \ - --remote-port ${ROUTER_PORT}" - - if [[ "$DRY_RUN" -eq 1 ]]; then - echo "DRY RUN: $WAIT_CMD" - else - eval "$WAIT_CMD" - fi - - echo "Killing the decode server" - [[ "$DRY_RUN" -eq 0 ]] && kill $decode_pid 2>/dev/null || true -fi - -echo "Script completed successfully" -exit 0 diff --git a/benchmarks/multi_node/amd_utils/setup_deps.sh b/benchmarks/multi_node/amd_utils/setup_deps.sh deleted file mode 100644 index 1cc50a15ea..0000000000 --- a/benchmarks/multi_node/amd_utils/setup_deps.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars ROCM_PATH UCX_HOME RIXL_HOME -# Install missing disagg dependencies at container start; sourced by server_vllm.sh -# and server_sglang.sh so PATH / LD_LIBRARY_PATH exports persist. Each installer is -# idempotent and gated on $ENGINE (vllm-disagg / sglang-disagg). - -_SETUP_START=$(date +%s) -_SETUP_INSTALLED=() - -# ibv_devinfo (ibverbs-utils) and ip (iproute2) for in-container NIC/RDMA checks. -install_recipe_deps() { - if command -v ibv_devinfo >/dev/null 2>&1 && command -v ip >/dev/null 2>&1; then - echo "[SETUP] Container RDMA/net tools already present" - return 0 - fi - - echo "[SETUP] Installing ibv_devinfo + iproute2 in container..." - apt-get update -q -y && apt-get install -q -y \ - ibverbs-utils iproute2 \ - && rm -rf /var/lib/apt/lists/* - - if ! command -v ibv_devinfo >/dev/null 2>&1 || ! command -v ip >/dev/null 2>&1; then - echo "[SETUP] ERROR: Failed to install ibv_devinfo/iproute2"; exit 1 - fi - _SETUP_INSTALLED+=("ibverbs-utils+iproute2") -} - -# ROCm vLLM lacks the quark dependency needed for MXFP4 models: -# https://github.com/vllm-project/vllm/issues/35633 -install_amd_quark() { - if python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] amd-quark already present" - return 0 - fi - - echo "[SETUP] Installing amd-quark for MXFP4 quantization support..." - pip install --quiet amd-quark - - if ! python3 -c "import quark" 2>/dev/null; then - echo "[SETUP] WARN: amd-quark install failed (non-fatal for non-MXFP4 models)" - return 0 - fi - _SETUP_INSTALLED+=("amd-quark") -} - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - install_recipe_deps - install_amd_quark - - export ROCM_PATH - export UCX_HOME - export RIXL_HOME - export PATH="${UCX_HOME}/bin:/usr/local/bin/etcd:/root/.cargo/bin:${PATH}" - export LD_LIBRARY_PATH="${UCX_HOME}/lib:${RIXL_HOME}/lib:${RIXL_HOME}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" -fi - -_SETUP_END=$(date +%s) -if [[ ${#_SETUP_INSTALLED[@]} -eq 0 ]]; then - echo "[SETUP] All dependencies already present ($(( _SETUP_END - _SETUP_START ))s wallclock)" -else - echo "[SETUP] Installed: ${_SETUP_INSTALLED[*]} in $(( _SETUP_END - _SETUP_START ))s" -fi diff --git a/benchmarks/multi_node/amd_utils/sglang_cli.py b/benchmarks/multi_node/amd_utils/sglang_cli.py deleted file mode 100644 index f8faa6076d..0000000000 --- a/benchmarks/multi_node/amd_utils/sglang_cli.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Resolve CLI spellings from the installed SGLang parser.""" - -from __future__ import annotations - -import argparse - - -def cuda_graph_flags(parser: argparse.ArgumentParser) -> tuple[str, str]: - """Preserve legacy graph settings, or use the explicit phase options.""" - options = parser._option_string_actions - if "--cuda-graph-bs" in options: - return "--cuda-graph-bs", "--cuda-graph-bs" - split = "--cuda-graph-bs-prefill", "--cuda-graph-bs-decode" - if all(option in options for option in split): - return split - raise ValueError("Installed SGLang has no supported CUDA graph batch-size flags") - - -def main() -> None: - from sglang.srt.server_args import ServerArgs - - parser = argparse.ArgumentParser() - ServerArgs.add_cli_args(parser) - print(*cuda_graph_flags(parser)) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/multi_node/amd_utils/stage_node_logs.sh b/benchmarks/multi_node/amd_utils/stage_node_logs.sh deleted file mode 100755 index f1e67814e4..0000000000 --- a/benchmarks/multi_node/amd_utils/stage_node_logs.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail - -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " >&2 - exit 2 -fi - -SOURCE_LOGS=$1 -SHARED_LOGS=$2 - -if [[ ! -d "$SOURCE_LOGS" ]]; then - echo "[logs][ERROR] no node-local logs found on $(hostname): $SOURCE_LOGS" >&2 - exit 1 -fi - -# Server containers create the source tree as root, and node 0 may have already -# created the shared destination as root. The Slurm nodes provide passwordless -# sudo for the same Docker lifecycle used by job.slurm. -sudo mkdir -p "$SHARED_LOGS" -sudo cp -r "$SOURCE_LOGS"/. "$SHARED_LOGS"/ -echo "[logs] staged $(hostname):$SOURCE_LOGS -> $SHARED_LOGS" diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh deleted file mode 100755 index 428f8e0db4..0000000000 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ /dev/null @@ -1,306 +0,0 @@ -#!/bin/bash - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -# -# Submit a multi-node disaggregated benchmark job to SLURM. The router is co-located -# with the first prefill node, so NUM_NODES = PREFILL_NODES + DECODE_NODES. - -usage() { - cat << 'USAGE' -Usage: - bash submit.sh \ - \ - \ - \ - \ - [NODE_LIST] - -Arguments: - PREFILL_NODES Number of prefill nodes - PREFILL_WORKERS Number of prefill workers (usually 1) - DECODE_NODES Number of decode nodes - DECODE_WORKERS Number of decode workers (usually 1) - ISL Input sequence length - OSL Output sequence length - CONCURRENCIES Concurrency levels, delimited by 'x' (e.g., "8x16x32") - REQUEST_RATE Request rate ("inf" for max throughput) - PREFILL_ENABLE_EP true/false or 1/0 (expert parallelism on prefill) - PREFILL_ENABLE_DP true/false or 1/0 (data-parallel attention on prefill) - DECODE_ENABLE_EP true/false or 1/0 (expert parallelism on decode) - DECODE_ENABLE_DP true/false or 1/0 (data-parallel attention on decode) - PREFILL_TP Tensor parallel size per prefill node - DECODE_TP Tensor parallel size per decode node - RANDOM_RANGE_RATIO Random range ratio for benchmark client - NODE_LIST Optional: comma-separated hostnames (must match NUM_NODES) - -Required environment variables: - SLURM_ACCOUNT SLURM account name - SLURM_PARTITION SLURM partition - TIME_LIMIT Job time limit (e.g., "08:00:00") - MODEL_PATH Path to model directory (e.g., /nfsdata) - MODEL_NAME Model name directory - CONTAINER_IMAGE Docker image name (e.g., vllm_disagg_pd:latest) - RUNNER_NAME Runner identifier (for job name) - -Required environment variables (continued): - DRY_RUN 1 = echo composed server/router launch commands instead of - running them (preview a recipe against a real allocation). -USAGE -} - -check_env_vars \ - SLURM_ACCOUNT SLURM_PARTITION TIME_LIMIT MODEL_PATH MODEL_NAME \ - CONTAINER_IMAGE RUNNER_NAME FRAMEWORK GPUS_PER_NODE PREFILL_EP \ - PREFILL_DP_ATTN PREFILL_NUM_WORKERS PREFILL_PP_SIZE PREFILL_DCP_SIZE PREFILL_PCP_SIZE \ - DECODE_EP DECODE_DP_ATTN DECODE_NUM_WORKERS DECODE_PP_SIZE DECODE_DCP_SIZE \ - DECODE_PCP_SIZE DECODE_MTP_SIZE BENCH_NUM_PROMPTS_MULTIPLIER DRY_RUN RUN_EVAL \ - EVAL_ONLY EVAL_FRAMEWORK IS_MULTINODE SWEBENCH_USE_MODAL BENCHMARK_LOGS_DIR \ - KEEP_CONTAINERS ROUTER_TYPE ROUTER_PORT PROXY_PING_PORT HEADNODE_PORT \ - SERVER_PORT IS_AGENTIC KV_OFFLOADING - -if [[ $# -lt 15 || $# -gt 16 ]]; then - usage >&2 - exit 1 -fi - -PREFILL_NODES=$1 -PREFILL_WORKERS=${2} -DECODE_NODES=$3 -DECODE_WORKERS=${4} -ISL=$5 -OSL=$6 -CONCURRENCIES=$7 -REQUEST_RATE=$8 -PREFILL_ENABLE_EP=${9} -PREFILL_ENABLE_DP=${10} -DECODE_ENABLE_EP=${11} -DECODE_ENABLE_DP=${12} -PREFILL_TP=${13} -DECODE_TP=${14} -RANDOM_RANGE_RATIO=${15} -NODE_LIST=${16:-} - -NUM_NODES=$((PREFILL_NODES + DECODE_NODES)) -profiler_args="${ISL} ${OSL} ${CONCURRENCIES} ${REQUEST_RATE}" - -export ENGINE="${FRAMEWORK}" -export MODEL_DIR=$MODEL_PATH -export DOCKER_IMAGE_NAME=$CONTAINER_IMAGE -export PROFILER_ARGS=$profiler_args - -if [[ "$ENGINE" == "vllm-disagg" ]]; then - check_env_vars PROXY_STREAM_IDLE_TIMEOUT - export PROXY_STREAM_IDLE_TIMEOUT -fi -# xP = prefill workers, yD = decode workers (may span multiple nodes) -export xP=$PREFILL_WORKERS -export yD=$DECODE_WORKERS -export PREFILL_TP_SIZE=$(( $PREFILL_NODES * $PREFILL_TP / $PREFILL_WORKERS )) -export PREFILL_ENABLE_EP -export PREFILL_ENABLE_DP -export PREFILL_TP -export PREFILL_EP -export PREFILL_DP_ATTN -export PREFILL_NUM_WORKERS -export PREFILL_PP_SIZE -export PREFILL_DCP_SIZE -export PREFILL_PCP_SIZE -export DECODE_TP_SIZE=$(( $DECODE_NODES * $DECODE_TP / $DECODE_WORKERS )) -export DECODE_ENABLE_EP -export DECODE_ENABLE_DP -export DECODE_TP -export DECODE_EP -export DECODE_DP_ATTN -export DECODE_NUM_WORKERS -export DECODE_PP_SIZE -export DECODE_DCP_SIZE -export DECODE_PCP_SIZE -export DECODE_MTP_SIZE - -export NUM_NODES=$NUM_NODES -export GPUS_PER_NODE=$GPUS_PER_NODE -export MODEL_NAME=$MODEL_NAME -export BENCH_INPUT_LEN=${ISL} -export BENCH_OUTPUT_LEN=${OSL} -export BENCH_NUM_PROMPTS_MULTIPLIER -export BENCH_MAX_CONCURRENCY=${CONCURRENCIES} -export BENCH_REQUEST_RATE=${REQUEST_RATE} -export BENCH_RANDOM_RANGE_RATIO=${RANDOM_RANGE_RATIO} - -# sbatch defaults to --export=ALL, so exporting DRY_RUN carries it through job.slurm -# and Docker (-e) into server_sglang.sh. -export DRY_RUN - -export RUN_EVAL -export EVAL_ONLY -export EVAL_CONC="${EVAL_CONC:-}" -export EVAL_FRAMEWORK -export EVAL_SUITE="${EVAL_SUITE:-}" -export SWEBENCH_GEN_MODE="${SWEBENCH_GEN_MODE:-}" -export FRAMEWORK="${FRAMEWORK:-}" -export PRECISION="${PRECISION:-}" -export MODEL_PREFIX="${MODEL_PREFIX:-}" -export RUNNER_TYPE="${RUNNER_TYPE:-}" -export RESULT_FILENAME="${RESULT_FILENAME:-}" -export SPEC_DECODING="${SPEC_DECODING:-}" -export IS_MULTINODE -export SWEBENCH_USE_MODAL -export MODAL_TOKEN_ID="${MODAL_TOKEN_ID:-}" -export MODAL_TOKEN_SECRET="${MODAL_TOKEN_SECRET:-}" -export HF_TOKEN="${HF_TOKEN:-}" -export SCENARIO_TYPE="${SCENARIO_TYPE:-}" -export EVAL_LIMIT="${EVAL_LIMIT:-}" - -# Log directory: must be on NFS (shared filesystem) so the submit host can read SLURM output. -export BENCHMARK_LOGS_DIR -mkdir -p "$BENCHMARK_LOGS_DIR" - -NODELIST_OPT=() -if [[ -n "${NODE_LIST//[[:space:]]/}" ]]; then - IFS=',' read -r -a NODE_ARR <<< "$NODE_LIST" - if [[ "${#NODE_ARR[@]}" -ne "$NUM_NODES" ]]; then - echo "Error: NODE_LIST has ${#NODE_ARR[@]} nodes but NUM_NODES=${NUM_NODES}" >&2 - echo "Error: NODE_LIST='${NODE_LIST}'" >&2 - exit 1 - fi - NODELIST_CSV="$(IFS=,; echo "${NODE_ARR[*]}")" - NODELIST_OPT=(--nodelist "$NODELIST_CSV") -fi - -# Exclude known-bad nodes for this (FRAMEWORK, MODEL_NAME) combo (e.g. broken Docker -# sockets) from node_excludes.yaml; SLURM_EXCLUDE_NODES overrides with an explicit -# list. Resolution must fail loudly rather than silently yield an empty exclude list, -# or a missing python3/PyYAML would quietly reintroduce the known-bad nodes. -EXCLUDE_OPT=() -NODE_EXCLUDES_YAML="$(dirname "$0")/node_excludes.yaml" -if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then - RESOLVED_EXCLUDE_NODES="$SLURM_EXCLUDE_NODES" -elif [[ -f "$NODE_EXCLUDES_YAML" ]]; then - if command -v python3 >/dev/null 2>&1 && python3 -c "import yaml" >/dev/null 2>&1; then - RESOLVED_EXCLUDE_NODES=$(python3 -c " -import yaml - -with open('${NODE_EXCLUDES_YAML}') as f: - cfg = yaml.safe_load(f) or {} - -framework = '${FRAMEWORK}' -model = '${MODEL_NAME}' -for rule in cfg.get('rules', []): - if rule.get('framework') == framework and model in (rule.get('models') or []): - print(rule.get('exclude_nodes', '')) - break -") - PYTHON_EXCLUDE_RC=$? - if [[ $PYTHON_EXCLUDE_RC -ne 0 ]]; then - echo "Error: python3 failed (exit ${PYTHON_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 - echo "Error: fix the YAML, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 - exit 1 - fi - else - # awk fallback matched to node_excludes.yaml's fixed rule/models/exclude_nodes - # shape, for submit hosts without python3 or PyYAML. - echo "Warning: python3/PyYAML unavailable on submit host; falling back to awk parsing of ${NODE_EXCLUDES_YAML}" >&2 - RESOLVED_EXCLUDE_NODES=$(awk -v fw="$FRAMEWORK" -v model="$MODEL_NAME" ' - /^ - framework:/ { - line = $0 - sub(/^ - framework: */, "", line) - fw_match = (line == fw) - model_match = 0 - next - } - fw_match && /^ - / { - m = $0 - sub(/^ - */, "", m) - gsub(/^"|"$/, "", m) - if (m == model) model_match = 1 - next - } - fw_match && model_match && /^ exclude_nodes:/ { - val = $0 - sub(/^ *exclude_nodes: */, "", val) - gsub(/^"|"$/, "", val) - print val - exit - } - ' "$NODE_EXCLUDES_YAML") - AWK_EXCLUDE_RC=$? - if [[ $AWK_EXCLUDE_RC -ne 0 ]]; then - echo "Error: awk fallback failed (exit ${AWK_EXCLUDE_RC}) parsing ${NODE_EXCLUDES_YAML}" >&2 - echo "Error: fix the YAML/parser, or set SLURM_EXCLUDE_NODES to bypass this lookup." >&2 - exit 1 - fi - fi -else - RESOLVED_EXCLUDE_NODES="" -fi -if [[ -n "$RESOLVED_EXCLUDE_NODES" ]]; then - EXCLUDE_OPT=(--exclude "$RESOLVED_EXCLUDE_NODES") -fi - -# SLURM_REUSE_JOBID: run job.slurm directly in the current shell against the existing -# allocation. Inner srun calls pick it up via SLURM_JOB_ID; SLURM_OVERLAP=1 lets them -# share task slots with the interactive shell already holding the allocation. -if [[ -n "${SLURM_REUSE_JOBID:-}" ]]; then - REUSE_JID="$SLURM_REUSE_JOBID" - echo "Reusing existing Slurm allocation ${REUSE_JID} (skipping sbatch)" >&2 - - ALLOC_NODELIST="${SLURM_JOB_NODELIST:-$(squeue -h -j "$REUSE_JID" -o '%N' 2>/dev/null)}" - if [[ -z "$ALLOC_NODELIST" ]]; then - echo "Error: could not resolve nodelist for job ${REUSE_JID}" >&2 - exit 1 - fi - ALLOC_NNODES=$(scontrol show hostnames "$ALLOC_NODELIST" | wc -l) - if [[ "$ALLOC_NNODES" -lt "$NUM_NODES" ]]; then - echo "Error: allocation ${REUSE_JID} has ${ALLOC_NNODES} nodes, need ${NUM_NODES}" >&2 - exit 1 - fi - - export SLURM_JOB_ID="$REUSE_JID" - export SLURM_JOBID="$REUSE_JID" - export SLURM_JOB_NODELIST="$ALLOC_NODELIST" - export SLURM_NODELIST="$ALLOC_NODELIST" - export SLURM_NNODES="$ALLOC_NNODES" - export SLURM_JOB_NUM_NODES="$ALLOC_NNODES" - export SLURM_NTASKS="$ALLOC_NNODES" - export SLURM_NPROCS="$ALLOC_NNODES" - export SLURM_NTASKS_PER_NODE=1 - export SLURM_TASKS_PER_NODE="1(x${ALLOC_NNODES})" - export SLURM_OVERLAP=1 - export SLURM_SUBMIT_DIR="$(pwd)" - - STDOUT_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.out" - STDERR_LOG="${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.err" - rm -f "$STDOUT_LOG" "$STDERR_LOG" - - nohup bash "$(dirname "$0")/job.slurm" >"$STDOUT_LOG" 2>"$STDERR_LOG" & - INLINE_PID=$! - echo "$INLINE_PID" > "${BENCHMARK_LOGS_DIR}/slurm_job-${REUSE_JID}.pid" - echo "Started job.slurm (pid=${INLINE_PID}); logs: ${STDOUT_LOG}" >&2 - - echo "$REUSE_JID" - exit 0 -fi - -sbatch_cmd=( - sbatch - --parsable - --exclusive - -N "$NUM_NODES" - -n "$NUM_NODES" - "${NODELIST_OPT[@]}" - "${EXCLUDE_OPT[@]}" - --time "$TIME_LIMIT" - --partition "$SLURM_PARTITION" - --account "$SLURM_ACCOUNT" - --job-name "$RUNNER_NAME" - --output "${BENCHMARK_LOGS_DIR}/slurm_job-%j.out" - --error "${BENCHMARK_LOGS_DIR}/slurm_job-%j.err" - "$(dirname "$0")/job.slurm" -) - -JOB_ID=$("${sbatch_cmd[@]}") -if [[ $? -ne 0 ]]; then - echo "Error: Failed to submit job with sbatch" >&2 - exit 1 -fi -echo "$JOB_ID" diff --git a/benchmarks/multi_node/amd_utils/sync.py b/benchmarks/multi_node/amd_utils/sync.py deleted file mode 100755 index 96e94c1b08..0000000000 --- a/benchmarks/multi_node/amd_utils/sync.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python3 -""" -Multi-node synchronization utilities for disaggregated inference. - -Subcommands: - barrier - Wait until all specified nodes have opened their ports (TCP barrier) - Optionally wait for HTTP health endpoints to return 200 - wait - Block until a remote port closes (shutdown coordination) -""" - -import socket -import time -import threading -import argparse -import sys -import urllib.request -import urllib.error - - -def is_port_open(ip, port, timeout=2): - """Check if a given IP and port are accessible.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.settimeout(timeout) - return s.connect_ex((ip, port)) == 0 - - -def check_health(ip, port, path="/health", timeout=2): - """Return True if http://ip:port/path returns HTTP 200.""" - try: - url = f"http://{ip}:{port}{path}" - req = urllib.request.Request(url) - with urllib.request.urlopen(req, timeout=timeout) as resp: - return getattr(resp, "status", 200) == 200 - except (urllib.error.URLError, urllib.error.HTTPError, OSError): - return False - - -# ============================================================================= -# barrier subcommand -# ============================================================================= - -def cmd_barrier(args): - """Wait until all nodes have opened the specified ports.""" - NODE_IPS = [ip.strip() for ip in args.node_ips.split(",") if ip.strip()] - NODE_PORTS = [int(p.strip()) for p in args.node_ports.split(",") if p.strip()] - - if not NODE_IPS: - print("Error: NODE_IPS argument is empty or not set.") - sys.exit(1) - - if len(NODE_PORTS) == 1: - NODE_PORTS *= len(NODE_IPS) - elif len(NODE_PORTS) != len(NODE_IPS): - print("Error: Number of ports must match number of node IPs or only one port should be given for all.") - sys.exit(1) - - server_socket = None - - def open_port(): - nonlocal server_socket - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind((args.local_ip, args.local_port)) - server_socket.listen(5) - print(f"Port {args.local_port} is now open on {args.local_ip}.") - while True: - conn, addr = server_socket.accept() - conn.close() - - def close_port(): - nonlocal server_socket - if server_socket: - server_socket.close() - print(f"Port {args.local_port} has been closed on {args.local_ip}.") - - if args.enable_port: - threading.Thread(target=open_port, daemon=True).start() - - # Wait for all ports (TCP check) - if args.wait_for_all_ports: - start_time = time.time() - timeout = args.timeout - seen_open = set() - - while True: - status = {(ip, port): is_port_open(ip, port) - for ip, port in zip(NODE_IPS, NODE_PORTS)} - - # Fail fast: a port that was open and is now closed means that - # server died/was killed; don't wait out the full timeout. - regressed = [t for t in seen_open if not status[t]] - if regressed: - print("ERROR: the following ports were open then went down (server died):", flush=True) - for ip, port in regressed: - print(f" - {ip}:{port}", flush=True) - sys.exit(1) - seen_open.update(t for t, ok in status.items() if ok) - - if all(status.values()): - break - - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_open = [t for t, ok in status.items() if not ok] - print(f"ERROR: Timeout after {timeout} seconds waiting for ports to open.", flush=True) - print("The following nodes/ports are still not responding:", flush=True) - for ip, port in not_open: - print(f" - {ip}:{port}", flush=True) - sys.exit(1) - remaining = timeout - (time.time() - start_time) - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . . ({remaining:.0f}s remaining)", flush=True) - else: - print(f"Waiting for nodes.{NODE_PORTS},{NODE_IPS} . .", flush=True) - time.sleep(5) - - # Wait for all health endpoints (HTTP check) - if args.wait_for_all_health: - health_path = args.health_endpoint - start_time = time.time() - timeout = args.timeout - seen_ready = set() - - while True: - status = { - (ip, port): check_health(ip, port, health_path) - for ip, port in zip(NODE_IPS, NODE_PORTS) - } - - # Fail fast: an endpoint that was healthy and is now down means the - # server died/was killed; don't wait out the full timeout. - regressed = [t for t in seen_ready if not status[t]] - if regressed: - print(f"ERROR: the following ({health_path}) were healthy then went down (server died):", flush=True) - for ip, port in regressed: - print(f" - http://{ip}:{port}{health_path}", flush=True) - sys.exit(1) - seen_ready.update(t for t, ok in status.items() if ok) - - if all(status.values()): - break - - if timeout > 0: - elapsed = time.time() - start_time - if elapsed >= timeout: - not_ready = [t for t, ok in status.items() if not ok] - print(f"ERROR: Timeout after {timeout} seconds waiting for health endpoints.", flush=True) - print(f"The following (http://ip:port{health_path}) are still not responding:", flush=True) - for ip, port in not_ready: - print(f" - http://{ip}:{port}{health_path}", flush=True) - sys.exit(1) - remaining = timeout - (time.time() - start_time) - print( - f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) .. ({remaining:.0f}s remaining)", - flush=True, - ) - else: - print(f"Waiting for health on {list(zip(NODE_IPS, NODE_PORTS))} ({health_path}) ..", flush=True) - time.sleep(30) - - if args.enable_port: - # Keep the port open long enough for slow nodes to pass their barrier. - # The previous 30s was too short when setup times vary by minutes. - grace = max(60, args.timeout // 2) if args.timeout > 0 else 300 - time.sleep(grace) - close_port() - - -# ============================================================================= -# wait subcommand -# ============================================================================= - -def cmd_wait(args): - """Wait while a remote port remains open, exit when it closes.""" - print( - f"Waiting while port {args.remote_port} on {args.remote_ip} is open...", - flush=True, - ) - while is_port_open(args.remote_ip, args.remote_port): - time.sleep(5) - print(f"Port {args.remote_port} on {args.remote_ip} is now closed.", flush=True) - - -# ============================================================================= -# CLI -# ============================================================================= - -def main(): - parser = argparse.ArgumentParser(description="Multi-node synchronization utilities.") - subparsers = parser.add_subparsers(dest="command", required=True) - - # barrier subcommand - bp = subparsers.add_parser("barrier", help="Wait for all nodes to open specified ports.") - bp.add_argument("--local-ip", required=False, help="Local IP address to bind the server.") - bp.add_argument("--local-port", type=int, required=False, help="Port number to bind the server.") - bp.add_argument("--enable-port", action="store_true", help="Enable opening and closing of local port.") - bp.add_argument("--node-ips", required=True, help="Comma-separated list of node IPs.") - bp.add_argument("--node-ports", required=True, help="Comma-separated list of ports to check.") - bp.add_argument("--timeout", type=int, default=600, - help="Timeout in seconds (default: 600). Set to 0 for no timeout.") - bp.add_argument("--wait-for-all-ports", action="store_true", - help="Wait until all node ports are open (TCP).") - bp.add_argument("--wait-for-all-health", action="store_true", - help="Wait until http://ip:port/health returns 200 for all nodes.") - bp.add_argument("--health-endpoint", default="/health", - help="Path for health check (default: /health).") - bp.set_defaults(func=cmd_barrier) - - # wait subcommand - wp = subparsers.add_parser("wait", help="Wait while a remote port remains open.") - wp.add_argument("--remote-ip", required=True, help="Remote server IP address.") - wp.add_argument("--remote-port", type=int, required=True, help="Remote port number.") - wp.set_defaults(func=cmd_wait) - - args = parser.parse_args() - args.func(args) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/multi_node/amd_utils/trace_replay.sh b/benchmarks/multi_node/amd_utils/trace_replay.sh deleted file mode 100644 index 42fa16ca3f..0000000000 --- a/benchmarks/multi_node/amd_utils/trace_replay.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash -# Agentic trace-replay runner for the disaggregated servers. -# -# Usage: bash trace_replay.sh - -source "$(dirname "${BASH_SOURCE[0]}")/../../benchmark_lib.sh" --validation-only -check_env_vars ENGINE MODEL_PATH MODEL_NAME ROUTER_PORT -if [[ $# -ne 4 ]]; then - echo "Error: trace_replay.sh requires 4 positional arguments" >&2 - exit 1 -fi - -model_path=$1 -model_name=$2 -concurrency_list=${3} -# vllm-disagg uses --served-model-name MODEL_NAME; sglang defaults to MODEL_PATH -if [[ "$ENGINE" == "vllm-disagg" ]]; then - MODEL="${MODEL_NAME}" -else - MODEL="${MODEL_PATH}" -fi -log_path=${4} - -IFS='x' read -r -a chosen_concurrencies <<< "${concurrency_list}" - -export TRANSFORMERS_VERBOSITY=error -export TOKENIZERS_PARALLELISM=false - -RESULT_DIR="${RESULT_DIR:-${log_path}/agentic}" -mkdir -p "$RESULT_DIR" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Wipe every KV cache tier on each backend worker before a concurrency point so it -# is measured cold. Hits each worker directly (the router does not fan /flush_cache -# out) using the URLs server_sglang.sh resolved into SERVER_FLUSH_URLS_CSV. -# L1 (GPU radix) + L2 (host hicache): POST /flush_cache, a NO-OP while any request -# is in flight, so drain-retry until "Cache flushed" or FLUSH_DRAIN_TIMEOUT. -# L3 (umbp / mooncake store): POST /hicache/storage-backend/clear, non-200 when -# L3 is off. -# Best-effort: never hard-fails the sweep. -clear_kv_caches() { - local drain_tmo="${FLUSH_DRAIN_TIMEOUT}" - local urls_csv="${SERVER_FLUSH_URLS_CSV:-}" - if [[ -z "$urls_csv" ]]; then - echo "[clear_caches] WARN: SERVER_FLUSH_URLS_CSV unset; skipping cache flush" >&2 - return 0 - fi - local -a urls - IFS=',' read -r -a urls <<< "$urls_csv" - local url start ok resp code - for url in "${urls[@]}"; do - [[ -n "$url" ]] || continue - # L1 + L2: drain-retry until flushed (no-op while requests in flight). - start=$(date +%s); ok=0; resp="" - while :; do - resp=$(curl -sf -m 10 -X POST "${url}/flush_cache" 2>/dev/null || true) - echo "$resp" | grep -qi "Cache flushed" && { ok=1; break; } - (( $(date +%s) - start >= drain_tmo )) && break - sleep 3 - done - if [[ "$ok" == 1 ]]; then - echo "[clear_caches] ${url}: L1+L2 flushed" - else - echo "[clear_caches] WARN ${url}: L1+L2 flush NOT confirmed after ${drain_tmo}s (resp='${resp:0:80}')" >&2 - fi - # L3: storage-backend clear (umbp / mooncake). 200 when a backend is attached. - code=$(curl -s -m 60 -o /dev/null -w '%{http_code}' -X POST "${url}/hicache/storage-backend/clear" 2>/dev/null || echo 000) - if [[ "$code" == 200 ]]; then - echo "[clear_caches] ${url}: L3 store cleared" - else - echo "[clear_caches] ${url}: L3 clear http=${code} (no storage backend / L3 off — ok)" - fi - done -} - -PORT="${ROUTER_PORT}" -check_env_vars DURATION RESULT_FILENAME FLUSH_DRAIN_TIMEOUT CLEAR_CACHE_BETWEEN_CONC -export MODEL DURATION MAX_MODEL_LEN -# The workflow guard / upload steps expect one "${RESULT_FILENAME}_conc.json" per -# concurrency, so each conc below is suffixed with _conc (as agentic_srt.sh does). -RESULT_FILENAME_BASE="${RESULT_FILENAME}" - -mkdir -p "$RESULT_DIR" - -if [ "$PREFILL_ENABLE_DP" = "true" ]; then - set -x - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - set +x -fi - -resolve_trace_source -install_agentic_deps - -ANY_FAILED=0 -for max_concurrency in "${chosen_concurrencies[@]}"; do - - echo "==========================================" - echo "Agentic trace replay: conc=$max_concurrency" - echo "==========================================" - - # Measure each conc point cold (no prefix reuse from the previous conc). - # CLEAR_CACHE_BETWEEN_CONC=0 disables; best-effort, never fails the run. - if [[ "${CLEAR_CACHE_BETWEEN_CONC}" == "1" ]]; then - echo "conc=$max_concurrency: clearing L1/L2/L3 on all backends (no server restart)" - clear_kv_caches || echo "WARNING: cache clear had issues for conc=$max_concurrency" >&2 - fi - - # benchmark-multinode-tmpl.yml expects the per-conc nesting (LOGS/agentic/conc_*/...) - # even though CI runs one concurrency per job; nesting also keeps local multi-conc - # sweeps from overwriting each other (same layout as agentic_srt.sh). - CONC_RESULT_DIR="$RESULT_DIR/conc_${max_concurrency}" - mkdir -p "$CONC_RESULT_DIR" - - CONC="$max_concurrency" - USERS="$max_concurrency" - export CONC USERS - build_replay_cmd "$CONC_RESULT_DIR" - - # Must match the workflow guard's "${RESULT_FILENAME}_conc*.json" glob and the - # agg / checkpoint upload steps. - export RESULT_FILENAME="${RESULT_FILENAME_BASE}_conc${max_concurrency}" - if ! run_agentic_replay_and_write_outputs "$CONC_RESULT_DIR"; then - echo "WARNING: agentic trace replay for conc=$max_concurrency failed (replay or validation) after writing available results" >&2 - ANY_FAILED=1 - fi - - echo "-----------------------------------------" - -done - -export RESULT_FILENAME="$RESULT_FILENAME_BASE" - -if [ "$ANY_FAILED" -ne 0 ]; then - echo "WARNING: at least one conc had a non-zero exit; per-conc result files were still written when possible." >&2 -fi diff --git a/benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index d17d1a3238..0000000000 --- a/benchmarks/multi_node/deprecated/dsv4_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index d92dad7503..0000000000 --- a/benchmarks/multi_node/deprecated/glm5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# Launch jobs based on ISL/OSL -# Replace ' ' in CONC_LIST with 'x' such that the concurrency list is represented -# by a list of numbers delimited by 'x'. This is because of how the underlying launch script -# expects the concurrencies. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index 1505b905de..0000000000 --- a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh deleted file mode 100755 index d7995fb250..0000000000 --- a/benchmarks/multi_node/deprecated/kimik2.5_fp4_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -# Same EP/DP booleans as dsr1_fp8_mi355x_sglang-disagg.sh → amd_utils/submit.sh -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -# Parameter order matches SGLang disagg submit.sh; arg 16 is optional NODELIST. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index a9a28d889b..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm2.5_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh deleted file mode 100644 index e86e044a45..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh deleted file mode 100755 index ac0f0a27d7..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp4_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODE_LIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh deleted file mode 100644 index e86e044a45..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_atom-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -# Use upstreamed multi_node scripts (no external clone needed) -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -# Set up SGL launch script-specific environment variables -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh b/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh deleted file mode 100644 index 91628d632f..0000000000 --- a/benchmarks/multi_node/deprecated/minimaxm3_fp8_mi355x_vllm-disagg.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -# MiniMax-M3 MXFP8 (~414 GB) is pre-staged in this cluster's shared HF cache -# (/it-share/hf-hub-cache/models--MiniMaxAI--MiniMax-M3-MXFP8), not the default -# /it-share/data the launcher sets. Point the disagg model dir there for M3 only; -# submit.sh exports MODEL_DIR=$MODEL_PATH and job.slurm resolves the snapshot under -# it and bind-mounts MODEL_DIR into the prefill/decode serving containers. -export MODEL_PATH=/it-share/hf-hub-cache -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP:-1}" -eq 1 ]]; then - export PREFILL_ENABLE_EP=false -else - export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then - export PREFILL_ENABLE_DP=true -else - export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP:-1}" -eq 1 ]]; then - export DECODE_ENABLE_EP=false -else - export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then - export DECODE_ENABLE_DP=true -else - export DECODE_ENABLE_DP=false -fi - -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - "${NODELIST:-}") - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh deleted file mode 100644 index a7e5df573d..0000000000 --- a/benchmarks/multi_node/dsr1_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh deleted file mode 100644 index aed3695934..0000000000 --- a/benchmarks/multi_node/dsr1_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" \ No newline at end of file diff --git a/benchmarks/multi_node/llm-d/README.md b/benchmarks/multi_node/llm-d/README.md index 81dbd51995..e39e84ff51 100644 --- a/benchmarks/multi_node/llm-d/README.md +++ b/benchmarks/multi_node/llm-d/README.md @@ -1,8 +1,7 @@ # llmd-vllm multi-node SLURM scaffolding This directory holds the SLURM-side orchestration for the `llmd-vllm` -benchmark framework. It mirrors the AMD `sglang-disagg` pattern under -`benchmarks/multi_node/amd_utils/` (NOT the Dynamo / srt-slurm pattern): +benchmark framework. Unlike the srt-slurm-managed AMD and Dynamo paths, InferenceX itself owns the SLURM job, no vendor multi-node tool involved. | File | Role | diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index c5d4948d7a..2f5d271699 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -517,7 +517,7 @@ PY # Benchmark sweep. BENCH_MAX_CONCURRENCY is 'x'-delimited from submit.sh (e.g. "1024x512"). IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" # GPU counts are embedded in the result filename as _gpus_/_ctx_/_gen_ so the CI - # "Process result" step can parse them (same convention as amd_utils/bench.sh). + # "Process result" step can parse them. # ctx = prefill GPUs, gen = decode GPUs. _bench_prefill_gpus=$(( PREFILL_NODES * GPUS_PER_NODE )) _bench_decode_gpus=$(( DECODE_NODES * GPUS_PER_NODE )) diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index ab1098a116..83d8ec1255 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # # Submit a multi-node llmd-vllm wide-EP P/D disagg benchmark job to SLURM. -# Modeled after benchmarks/multi_node/amd_utils/submit.sh; prints JOB_ID on +# Prints JOB_ID on # stdout so the runner can poll for completion. # # Topology (matches the llm-d wide-EP guide reference): diff --git a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh deleted file mode 100755 index 560daa8673..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp4_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh b/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh deleted file mode 100755 index 560daa8673..0000000000 --- a/benchmarks/multi_node/qwen3.5_fp8_mi355x_sglang-disagg.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -source "$(dirname "$0")/../benchmark_lib.sh" - -check_env_vars \ - CONC_LIST \ - ISL \ - OSL \ - IMAGE \ - SPEC_DECODING \ - MODEL_PATH \ - PREFILL_NUM_WORKERS \ - PREFILL_TP \ - PREFILL_EP \ - PREFILL_DP_ATTN \ - DECODE_NUM_WORKERS \ - DECODE_TP \ - DECODE_EP \ - DECODE_DP_ATTN \ - PREFILL_NODES \ - DECODE_NODES \ - RANDOM_RANGE_RATIO \ - FRAMEWORK - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -set -x - -cd "$GITHUB_WORKSPACE/benchmarks/multi_node/amd_utils" || exit 1 - -export TIME_LIMIT="08:00:00" -export MODEL_PATH=$MODEL_PATH -export MODEL_NAME=$MODEL_NAME -export CONTAINER_IMAGE=$IMAGE - -if [[ "${PREFILL_EP}" -eq 1 ]]; then -export PREFILL_ENABLE_EP=false -else -export PREFILL_ENABLE_EP=true -fi - -if [[ "$PREFILL_DP_ATTN" == "true" ]]; then -export PREFILL_ENABLE_DP=true -else -export PREFILL_ENABLE_DP=false -fi - -if [[ "${DECODE_EP}" -eq 1 ]]; then -export DECODE_ENABLE_EP=false -else -export DECODE_ENABLE_EP=true -fi - -if [[ "$DECODE_DP_ATTN" == "true" ]]; then -export DECODE_ENABLE_DP=true -else -export DECODE_ENABLE_DP=false -fi - -# submit.sh wants the concurrency list 'x'-delimited. -JOB_ID=$(bash ./submit.sh $PREFILL_NODES \ - $PREFILL_NUM_WORKERS \ - $DECODE_NODES \ - $DECODE_NUM_WORKERS \ - $ISL $OSL "${CONC_LIST// /x}" inf \ - ${PREFILL_ENABLE_EP} ${PREFILL_ENABLE_DP} \ - ${DECODE_ENABLE_EP} ${DECODE_ENABLE_DP} \ - ${PREFILL_TP} ${DECODE_TP} \ - ${RANDOM_RANGE_RATIO} \ - ${NODE_LIST:-}) - -if [[ $? -ne 0 ]]; then - echo "Failed to submit job" >&2 - exit 1 -fi - -echo "$JOB_ID" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml new file mode 100644 index 0000000000..94cf4c3d11 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml @@ -0,0 +1,97 @@ +# Minimal MI355X aggregate validation. It exercises native SGLang Router and +# one aggregate SGLang worker; this is orchestration validation, not tuning. + +schema: 2 +name: "mi355x-sglang-qwen3-0.6b-agg-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + +slurm: + time_limit: "00:30:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 +frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 180 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in ${CONC_LIST}; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-agg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml new file mode 100644 index 0000000000..524bc45fc3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml @@ -0,0 +1,111 @@ +# Minimal two-node MI355X P/D validation. SGLang Router provides the request +# plane and the bundled AMD MoRI transport moves KV between prefill and decode. + +schema: 2 +name: "mi355x-sglang-qwen3-0.6b-disagg-1p1d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" + frameworks: + sglang: "0.5.17.dev20260809+g7120f3ee13" + sglang-router: "0.3.2" + amd-mori: "0.5.17.dev20260809+g7120f3ee13" + +slurm: + time_limit: "00:45:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 1 +frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 1 + env: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + MORI_RDMA_TC: "104" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" + args: &worker_config + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + disaggregation-transfer-backend: mori + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + decode: + nodes: 1 + workers: 1 + gpus: 1 + + env: *worker_environment + args: *worker_config + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 240 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in ${CONC_LIST}; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-disagg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index bae2efc100..28484ef8b6 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -222,49 +222,6 @@ qwen3.5-fp8-mi355x-atom-mtp: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } -qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.16-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP4P+TP8D/EP1 baseline (no speculative decoding). - # TP4 prefill saves 4 GPUs vs TP8P while delivering identical decode - # interactivity and 24-31% better throughput/GPU (12 vs 16 GPUs). - # dp-attn intentionally false: with --enable-dp-attention + - # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size, - # but is_deepep_class_backend() excludes MoRI, so - # num_shared_slots stays at the global value (1) and the - # (num_experts - num_shared_slots) % moe_ep_size assertion in - # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913 model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 @@ -329,40 +286,6 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - { tp: 2, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [20, 24, 28, 32, 36, 40] } -qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -426,156 +349,6 @@ dsr1-fp8-mi355x-atom-mtp: search-space: - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } -dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (2 prefill worker at DEP8 and 1 decode worker at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TP8 and 2 decode workers at TP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4, 2 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - kimik3-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-rocm100-af1c01499b289be555c475669ba50a88e96d846e model: moonshotai/Kimi-K3 @@ -647,306 +420,6 @@ minimaxm3-fp4-mi355x-atom-agentic-mtp: - { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp } - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.4.5" }, conc-list: [40, 48], spec-decoding: mtp } -dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D pure TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 32, 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 640, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 64 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - dsr1-fp8-mi325x-sglang-mtp: image: lmsysorg/sglang:v0.5.12-rocm700-mi30x model: deepseek-ai/DeepSeek-R1-0528 @@ -1014,137 +487,6 @@ dsv4-fp4-mi355x-atom-agentic-mtp: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: draft_model, conc-list: [48, 64, 96, 128, 256] } -dsr1-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 128, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 1*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 64, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - # 2*DEP8 + 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - minimaxm3-fp8-mi300x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:v0.29.0 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1245,114 +587,6 @@ dsv41flash-fp4-mi325x-vllm-agentic-dspark: # recipe halves the indexer buffer and caps the scheduler here. - { tp: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } -dsv4-fp4-mi355x-sglang-disagg-agentic-umbp-dspark: - # Renamed from dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp when this arm - # moved from EAGLE/MTP to DSpark; earlier perf-changelog entries are recorded - # under the old key. - # 20260913 rather than 20260911: it is the first tag carrying the DSpark - # optimizations, which is the whole point of this arm. CLIENT_IMAGE stays at - # 20260907 -- that container only runs the load generator, so the - # server-side DSpark work does not reach it. - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913 - model: deepseek-ai/DeepSeek-V4-Pro-0813 - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: "draft_model" - conc-list: [ 4 ] - kv-offloading: none - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - spec-decoding: "draft_model" - conc-list: [ 16 ] - kv-offloading: none - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - spec-decoding: "draft_model" - conc-list: [ 32, 48 ] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "HICACHE_RATIO=2.5" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - spec-decoding: "draft_model" - conc-list: [ 128, 192, 256 ] - kv-offloading: dram - kv-offload-backend: { name: umbp-linker } - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - - "PREFILL_ROUTER_POLICY=consistent_hashing" - # Rank 0 opens the barrier port only after umbp_standalone_server has - # registered the 1.5 TB tier for GPU access; that took 305.7s on n08-25 - # but had not finished after 780s on n09-29, which killed job 29871 at - # the 900s default. Sit above UMBP_SA_WAIT_SECONDS (1800) so UMBP's own - # wait is the binding one rather than the barrier. - - "CONTAINER_BARRIER_TIMEOUT=2400" - - "CLIENT_IMAGE=lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - minimaxm3-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-2a02f6efe319c885e3ccbcecde402e0028f9ec1e model: amd/MiniMax-M3-MXFP4 @@ -1480,3 +714,58 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: search-space: # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + +qwen3-0.6b-fp16-mi355x-sglang-srt-agg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang + multinode: true + disagg: false + router: { name: sglang-router, version: "0.3.2" } + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1, 4] + num-nodes: 1 + worker: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml" + +qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi355x-amds + precision: fp16 + framework: sglang-disagg + multinode: true + disagg: true + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: [1, 4] + prefill: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false diff --git a/configs/deprecated/amd-master.yaml b/configs/deprecated/amd-master.yaml index 0bf2a275e1..faa8e91e3d 100644 --- a/configs/deprecated/amd-master.yaml +++ b/configs/deprecated/amd-master.yaml @@ -305,41 +305,6 @@ qwen3.5-fp8-mi355x-atom-mtp: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } - { tp: 8, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } -qwen3.5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang:v0.5.14-rocm720-mi35x - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Matches qwen3.5-fp8-mi355x-sglang TP8/EP1 low-concurrency sweep - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 model: amd/Qwen3.5-397B-A17B-MXFP4 @@ -388,41 +353,6 @@ qwen3.5-fp4-mi355x-sglang-mtp: - { tp: 2, conc-start: 4, conc-end: 128, spec-decoding: mtp } - { tp: 4, conc-start: 4, conc-end: 16, spec-decoding: mtp } -qwen3.5-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: amd/Qwen3.5-397B-A17B-MXFP4 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP8/EP1, dp-attn false; MoRI conn.py overlay via job.slurm. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - qwen3.5-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm720-mi30x model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -472,42 +402,6 @@ glm5-fp8-mi355x-sglang-mtp-deprecated-1k1k: - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } # Original config key: glm5-fp8-mi355x-sglang-disagg -glm5-fp8-mi355x-sglang-disagg-deprecated-1k1k: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep (aligned with glm5-fp8-mi355x-sglang conc range) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# Original config key: glm5-fp8-mi355x-atom glm5-fp8-mi355x-atom-deprecated-1k1k: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: zai-org/GLM-5-FP8 @@ -670,454 +564,6 @@ dsr1-fp8-mi355x-atom-mtp: search-space: - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } -dsr1-fp8-mi355x-sglang-disagg: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16) - - spec-decoding: "none" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Middle of curve" (1 prefill workers each at TP8 and 2 decode workers at DEP8) - - spec-decoding: "none" - conc-list: [ 1536, 1024, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "none" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp8-mi355x-sglang-disagg-mtp: - image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 - model: deepseek-ai/DeepSeek-R1-0528 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # "Top of curve" (1 prefill worker at DEP8 and 1 decode worker at DEP16) - - spec-decoding: "mtp" - conc-list: [ 1024, 2048 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Middle of curve" (1 prefill worker at TP8 and 2 decode workers each at DEP8) - - spec-decoding: "mtp" - conc-list: [ 1536, 1024, 512, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=1" - - # "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8) - - spec-decoding: "mtp" - conc-list: [ 256, 128, 64, 32, 16, 8, 4 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - - spec-decoding: "mtp" - conc-list: [ 64, 32, 16, 8, 4, 2, 1 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - -# Original config key: kimik2.5-fp4-mi355x-vllm-disagg -kimik2.5-fp4-mi355x-vllm-disagg-deprecated-1k1k: - image: vllm/vllm-openai-rocm:v0.24.0 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -dsr1-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D TP8 - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP8 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1P2D TP4 - - spec-decoding: "none" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=0" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "none" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -dsr1-fp4-mi355x-sglang-disagg-1k1k-mtp: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260529 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - -# Original config key: dsv4-fp4-mi355x-sglang dsv4-fp4-mi355x-sglang-deprecated-1k1k: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro @@ -1298,155 +744,6 @@ glm5-fp8-mi325x-sglang-mtp-deprecated-1k1k: search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } -dsr1-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260519 - model: amd/DeepSeek-R1-0528-MXFP4-v2 - model-prefix: dsr1 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # MTP configurations - # 1P1D TP8 - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 2, 4, 8, 16, 32 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=3" - - # 1P2D TP8 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1P2D TP4 - - spec-decoding: "mtp" - conc-list: [ 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - "DECODE_MTP_SIZE=2" - - # 1*DEP4+ 1*DEP8 - - spec-decoding: "mtp" - conc-list: [ 1024, 2048, 4096 ] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - -# Original config key: dsv4-fp4-mi355x-atom-disagg -dsv4-fp4-mi355x-atom-disagg-deprecated-1k1k: - image: rocm/atom-dev:nightly_202606101403 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - - conc-list: [ 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - -# MiniMax-M3 MXFP8 MI355X recipe: -# https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 -# MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. -# Original config key: minimaxm3-fp8-mi355x-vllm minimaxm3-fp8-mi355x-vllm-deprecated-1k1k: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1584,84 +881,6 @@ minimaxm3-fp8-mi355x-atom-mtp-deprecated-1k1k: - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } # Original config key: minimaxm3-fp8-mi355x-atom-disagg -minimaxm3-fp8-mi355x-atom-disagg-deprecated-1k1k: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# Original config key: minimaxm3-fp4-mi355x-atom-disagg -minimaxm3-fp4-mi355x-atom-disagg-deprecated-1k1k: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# MiniMax-M3 MXFP8 MI300X day-zero recipe. Reuse the dedicated ROCm image and -# MI355X serving shape, but retain the default BF16 KV cache because this -# checkpoint lacks calibrated ROCm FP8 attention scales. TP8-only, plain -# (non-expert-parallel) search space across the full conc range: EP8 -# (--enable-expert-parallel) produces garbage/incoherent output on this -# MXFP8+gfx942 combination (confirmed locally: TP8/EP8 returns garbled tokens -# even on trivial prompts, TP8/EP1 answers correctly), and was already the -# lower-throughput topology where measured. -# Original config key: minimaxm3-fp8-mi300x-vllm minimaxm3-fp8-mi300x-vllm-deprecated-1k1k: image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -1735,64 +954,6 @@ minimaxm3-fp8-mi325x-vllm-mtp-deprecated-1k1k: - { tp: 8, ep: 8, dp-attn: true, conc-start: 256, conc-end: 512, spec-decoding: mtp } # Original config key: minimaxm3-fp8-mi355x-vllm-disagg -minimaxm3-fp8-mi355x-vllm-disagg-deprecated-1k1k: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Decode-heavy 1P TP4 + 2D TP4 (3 nodes): double the decode engines to - # absorb the decode-bound 1k1k tail at high concurrency. - - spec-decoding: "none" - conc-list: [ 256, 512, 1024, 2048 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - # 8k1k is prefill-bound (8192-token prompts vs 1024 decode steps): pair the - # balanced layout with a prefill-heavy 2P/1D layout. Concurrency is capped at - # 512 so the multi-node eval policy (8k1k + conc >= 16, highest eligible conc) - # marks lm-eval at conc 512 — matching the range NVIDIA's aggregated 8k1k - # sweep tops out at and keeping the lm-eval async client stable. - -# Original config key: dsv4-fp8-mi325x-vllm dsv4-fp8-mi325x-vllm-deprecated-1k1k: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: deepseek-ai/DeepSeek-V4-Pro @@ -1880,161 +1041,6 @@ dsv4-fp8-mi300x-vllm-mtp: # Cap 8k1k MTP at 128 (normal holds 256, 1k1k holds 512). - { tp: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } -dsv4-fp4-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260701 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # non-MTP configurations - # 1P1D pure TP8 (mori KV transfer) - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - - spec-decoding: "none" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - - spec-decoding: "none" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - -# MTP variant of dsv4-fp4-mi355x-sglang-disagg (base run 29821778206). Mirrors -# the base disagg search space and topologies exactly (same 20260701 image, -# mori KV transfer + MoE a2a, sglang-router --pd-disaggregation) and enables -# DeepSeek-V4 built-in MTP on the decode side via spec-decoding: mtp — the same -# enablement path as dsr1-fp8-mi355x-sglang-disagg-mtp. DECODE_MTP_SIZE=1 (draft -# depth 1) on every arm. Depth 2 was tried on the pure-TP8 (bottom-of-curve) arm, -# but this 20260701 build's 2-step EAGLE verification is lossy: greedy gsm8k fell -# to ~0.85 (below the 0.91 dsv4 floor), while depth 1 is lossless (~0.96, matching -# the spec-none base). So all arms use depth 1. The dsv4 ROCm MTP -# CUDA-graph fix (sgl-project/sglang#26383, merged 2026-05-27) is already -# present in the pinned 20260701 nightly, so no image bump is needed. -dsv4-fp4-mi355x-sglang-disagg-mtp: - image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # MTP configurations - # 1P1D pure TP8 (mori KV transfer) - - spec-decoding: "mtp" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=2" - # 1P1D DEP8 (mori KV transfer + mori MoE a2a, dp-attention) - - spec-decoding: "mtp" - conc-list: [ 256 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - - - spec-decoding: "mtp" - conc-list: [ 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=1" - dsv4-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro @@ -2180,64 +1186,6 @@ dsv4-fp4-mi355x-atom-mtp: # MiniMax-M3 MXFP8 MI355X recipe: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. -dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: mi355x - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 2P1D DPA+TP8 - - conc-list: [ 256, 512, 768, 1024, 2048 ] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 - - conc-list: [ 4, 8, 16, 32, 64, 128 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 - -# DeepSeek-V4-Pro FP8 single-node on MI325X (gfx942) via vLLM. -# EXTRAPOLATED bring-up. Same rationale as dsv4-fp8-mi300x-vllm: sglang has no -# gfx942 build of the dsv4 nvfp4 MoE / TileLang-MLA kernels, so vLLM runs the -# checkpoint in FP8 via --quantization deepseek_v4_fp8 (dequant FP4 MoE -> FP8), -# the H200 dsv4 vLLM path. Config mirrors the same-model dsv4-fp4-mi355x-vllm -# (TP8, conc 4-512); 8x256GB (2TB) has ample headroom for the ~1.05TB FP8 -# footprint. Launch script dsv4_fp8_mi325x.sh carries the deepseek_v4 + gfx942 -# AITER flags. dsv4-fp8-mi325x-vllm: image: vllm/vllm-openai-rocm:nightly-09663abde0f50944a8d5ea30120666024b503faa model: deepseek-ai/DeepSeek-V4-Pro @@ -2317,42 +1265,6 @@ glm5-fp8-mi355x-sglang-mtp: - { tp: 8, conc-start: 4, conc-end: 8, spec-decoding: mtp } -glm5-fp8-mi355x-sglang-disagg: - image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 - model: zai-org/GLM-5-FP8 - model-prefix: glm5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang-disagg - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P+1D TP8/EP1 CI smoke sweep; dp-attn false (NSA / MoRI path) - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - - glm5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: zai-org/GLM-5-FP8 @@ -2624,118 +1536,6 @@ kimik2.5-fp4-mi355x-atom: search-space: - { tp: 4, conc-start: 4, conc-end: 128 } -kimik2.5-fp4-mi355x-atom-disagg: - image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atomesh_202607121715 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # 1P(tp4)1D(tp4) - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P(tp4)2D(tp8) - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - # All workers TP4 (real-weight sweep: TP8 decode is no better than TP4). - # Split across P/D topologies: 1D is stable at low conc; the high-conc tail - # runs on 2D so decode load is spread across two engines. - - isl: 8192 - osl: 1024 - search-space: - # 1P(TP4) 1D(TP4) = 2 nodes. Low concurrency only. - - spec-decoding: "none" - conc-list: [ 1, 2, 4 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P(TP4) 2D(TP4) = 3 nodes. High concurrency (decode KV/load spread over 2D). - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -# Archive source: amd-minimaxm2.5-m2.7-master.yaml -# Deprecated MiniMax M2.5/M2.7 entries archived from amd-master.yaml. -# Removed from the active master config so sweep generation no longer selects them. - - minimaxm2.5-fp8-mi355x-vllm: image: vllm/vllm-openai-rocm:v0.22.0 model: MiniMaxAI/MiniMax-M2.5 @@ -2882,68 +1682,6 @@ minimaxm2.5-fp8-mi325x-vllm: # the original minimaxm2.5-fp8-mi325x-vllm entry is left identical to origin/main so # its fixed-seq-len sweep is unaffected. # - image: 'vllm/vllm-openai-rocm:v0.18.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf' -minimaxm2.5-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-a6682d1d259cca69a9ae737ea5608fbbe7520031 - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 1024 - osl: 1024 - search-space: - # 1P2D: 1 prefill node (co-located with proxy) + 2 decode nodes = 3 nodes total - # Prefill also needs EP=8: MiniMax M2.5 expert intermediate_size=1536, - # TP8 shards to 192 which is not divisible by FP8 block_n=128. - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - - - isl: 8192 - osl: 1024 - search-space: - - spec-decoding: "none" - conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "VLLM_MORIIO_CONNECTOR_READ_MODE=1" - decode: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "DECODE_NODES=2" - -# Archive source: amd-minimaxm3-8k1k-master.yaml -# Deprecated MiniMax-M3 Single-turn 8k1k entries archived from amd-master.yaml. -# The scenario was deprecated after Monday, August 3, 2026 (see the Deprecation -# Notice in MODELS.md). Agentic coding remains active for this model. -# Removed from the active master config so sweep generation no longer selects them. - minimaxm3-fp8-mi355x-vllm: image: vllm/vllm-openai-rocm:nightly-9e57de7197f234f9d9187715d96e07e007048c0f model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -2985,44 +1723,6 @@ minimaxm3-fp8-mi355x-vllm-mtp: - { tp: 4, conc-start: 1, conc-end: 512, spec-decoding: mtp } # MiniMax-M3 MXFP4 MI355X vLLM disaggregated (prefill/decode) config. -minimaxm3-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P TP4 + 1D TP4 (2 nodes total), conc sweep 1..256 (single job, looped) - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - -# MiniMax-M3 MXFP4 MI355X vLLM recipe. The pinned nightly includes upstream -# MiniMax-M3 Quark MXFP4 support (vllm-project/vllm#45794). Use the text-only -# language-model path and mirror the MXFP8 MI355X search space for a direct -# precision comparison. minimaxm3-fp4-mi355x-vllm: image: vllm/vllm-openai-rocm:nightly-69715823df89b11ee684b84066390cbb9092d5c1 model: amd/MiniMax-M3-MXFP4 @@ -3124,110 +1824,6 @@ minimaxm3-fp8-mi355x-atom-mtp: search-space: - { tp: 4, conc-start: 1, conc-end: 256, spec-decoding: mtp } -minimaxm3-fp8-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 2P1D, DPA TP4 - - conc-list: [ 256, 512, 768, 1024 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D TP4 - -minimaxm3-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202607011530 - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "04b120d3040e0dd1a6915e88a3a3c1f588e6684a" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # 1P1D TP4 - - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 2P1D, DPA TP4 - - conc-list: [ 256, 512, 768, 1024 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - - "DECODE_MTP_SIZE=0" - # 1P1D TP4 - minimaxm3-fp8-mi300x-vllm: image: vllm/vllm-openai-rocm:nightly-4559c43a9526597c00cbcc4f59979496500268d1 model: MiniMaxAI/MiniMax-M3-MXFP8 @@ -3328,66 +1924,6 @@ minimaxm3-fp8-mi325x-vllm-mtp: # - 2P-TP4 / 1D-TP4 (3 nodes): prefill-heavy, for the prefill-bound 8k1k tail. # Per-worker serve flags live in # benchmarks/multi_node/amd_utils/models_vllm.yaml (MiniMax-M3-MXFP8). -minimaxm3-fp8-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp8 - framework: vllm-disagg - router: { name: vllm-router, version: "0.1.14" } - kv-p2p-transfer: moriio - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # Balanced 1P TP4 + 1D TP4 (2 nodes) across the full curve. - - spec-decoding: "none" - conc-list: [ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 ] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # Prefill-heavy 2P TP4 + 1D TP4 (3 nodes): two half-node TP4 prefill workers - # keep the single TP4 decode engine fed for the prefill-bound 8k1k tail. - - spec-decoding: "none" - conc-list: [ 128, 256, 512 ] - prefill: - num-worker: 2 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=2" - decode: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - -# Archive source: amd-qwen3.5-bf16-master.yaml -# Deprecated Qwen3.5-397B-A17B bf16 entries archived from amd-master.yaml. -# All bf16 recipes, in every scenario, were deprecated after Monday, -# August 3, 2026 (see the Deprecation Notice in MODELS.md). fp8 and fp4 -# recipes remain active. -# Removed from the active master config so sweep generation no longer selects them. - - qwen3.5-bf16-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260517 model: Qwen/Qwen3.5-397B-A17B diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index de9f1cda5c..6477619e8e 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -21,11 +21,11 @@ Use this page for benchmark configuration, recipe, image, and runner changes. It | [`perf-changelog.yaml`](../perf-changelog.yaml) | Append-only benchmark trigger log | | [`AGENTS.md`](../AGENTS.md) | Repository-wide config, MTP, changelog, and sweep rules | -Archive deprecated entries in [`configs/deprecated/amd-master.yaml`](../configs/deprecated/amd-master.yaml) or [`configs/deprecated/nvidia-master.yaml`](../configs/deprecated/nvidia-master.yaml). Use only these two vendor archives, not separate files per deprecation. Preserve historical settings and comments; disambiguate colliding keys with a descriptive suffix and an original-key comment. For partial retirements, move only the retired scenarios. Keep archives out of active sweep inputs. Retired AMD server-registry entries and model-specific setup belong in `benchmarks/multi_node/amd_utils/deprecated/`, outside the active server lookup. Preserve shared dependencies needed by retained SPEED-Bench collectors, including their scheduling scores. See the [deprecation rules](../AGENTS.md#deprecating-benchmark-configs). +Archive deprecated entries in [`configs/deprecated/amd-master.yaml`](../configs/deprecated/amd-master.yaml) or [`configs/deprecated/nvidia-master.yaml`](../configs/deprecated/nvidia-master.yaml). Use only these two vendor archives, not separate files per deprecation. Preserve historical settings and comments; disambiguate colliding keys with a descriptive suffix and an original-key comment. For partial retirements, move only the retired scenarios. Keep archives out of active sweep inputs. Archived AMD srt-slurm ports use native recipes rather than the retired `amd_utils` server registry. Preserve shared dependencies needed by retained SPEED-Bench collectors, including their scheduling scores. See the [deprecation rules](../AGENTS.md#deprecating-benchmark-configs). ## Dependency submodules -Git records the exact dependency commits. [`.gitmodules`](../.gitmodules) defines the repositories: AIPerf at `utils/aiperf`, NVIDIA srt-slurm at `utils/srt-slurm`. TileRT is a documented manual fork checkout in `setup_srt_slurm()`, not a separate submodule. +Git records the exact dependency commits. [`.gitmodules`](../.gitmodules) defines the repositories: AIPerf at `utils/aiperf`, NVIDIA srt-slurm at `utils/srt-slurm`. TileRT and the MI355X AMD port use pinned fork checkouts in `setup_srt_slurm()`, not separate submodules. The MI355X fork and revision are selected in `runners/runtime_settings.sh`; other runners keep their existing runtime selection. Initialize them before running benchmarks locally: @@ -56,6 +56,14 @@ parsed YAML scalars so quotes and punctuation remain data, not YAML or shell syn Keep model selection, cache preparation, and workload-dependent time limits in the launcher. Do not add profiles for non-srt-slurm launchers or change their routing here. +The MI355X port uses the same cluster renderer through `infx.workflows.srt_slurm`. +Its profile provides the ROCm visibility mask, fabric, Slurm directives, mounts, +and host prerequisites. The adapter renders job-local paths and forwards workflow +inputs; `srtctl` owns allocation, worker/router/service coordination, and completion. +Recipes use schema 2 (`engine` and `roles`) and `apply_srt_recipe` selects measured +AgentX acceptance from the golden curves. Fixed-sequence recipes consume the +matrix's `CONC_LIST`, so the benchmark points and result collector agree. + ## Procedure index 1. [Prepare a worktree](#prepare-a-worktree) diff --git a/infx/srt_slurm/synthetic_acceptance.py b/infx/srt_slurm/synthetic_acceptance.py index 95d06b7763..102f957590 100644 --- a/infx/srt_slurm/synthetic_acceptance.py +++ b/infx/srt_slurm/synthetic_acceptance.py @@ -19,6 +19,9 @@ GOLDEN_DIR = Path(__file__).resolve().parents[2] / "golden_al_distribution" ENGINES = { + "sglang": "sglang", + "sglang-disagg": "sglang", + "vllm-disagg": "vllm", "vllm": "vllm", "dynamo-vllm": "vllm", "dynamo-sglang": "sglang", diff --git a/infx/workflows/srt_slurm.py b/infx/workflows/srt_slurm.py new file mode 100644 index 0000000000..5f9dd3ec4e --- /dev/null +++ b/infx/workflows/srt_slurm.py @@ -0,0 +1,354 @@ +"""InferenceX workflow inputs and artifact contract for srt-slurm. + +This module does not allocate nodes, launch processes, poll Slurm, or repair +hosts. The cluster launcher supplies paths; srt-slurm owns the job lifecycle. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import re +import shutil +import tarfile +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +import yaml + +from infx.srt_slurm.cluster_config import render_cluster_config + +_FORWARDED_ENV = ( + "AIPERF_EXPERIMENTAL_FAST", + "AIPERF_DRAIN_TIMEOUT_SECONDS", + "AIPERF_DRAIN_POLL_SECONDS", + "CLEAR_CACHE_BETWEEN_CONC", + "FLUSH_DRAIN_TIMEOUT", + "CONC", + "CONC_LIST", + "DECODE_DP_ATTN", + "DECODE_EP", + "DECODE_NUM_WORKERS", + "DECODE_PCP_SIZE", + "DECODE_PP_SIZE", + "DECODE_TP", + "DECODE_HARDWARE", + "DISAGG", + "DURATION", + "EVAL_CONC", + "EVAL_FRAMEWORK", + "EVAL_LIMIT", + "EVAL_ONLY", + "EVAL_SUITE", + "FRAMEWORK", + "IS_AGENTIC", + "ISL", + "KV_OFFLOADING", + "KV_OFFLOAD_BACKEND", + "KV_OFFLOAD_BACKEND_METADATA", + "KV_P2P_TRANSFER", + "MAX_MODEL_LEN", + "MODEL", + "MODEL_PREFIX", + "PREFILL_DP_ATTN", + "PREFILL_EP", + "PREFILL_NUM_WORKERS", + "PREFILL_PCP_SIZE", + "PREFILL_PP_SIZE", + "PREFILL_TP", + "PREFILL_HARDWARE", + "PRECISION", + "RANDOM_RANGE_RATIO", + "REQUIRE_POWER", + "RESULT_FILENAME", + "RUN_EVAL", + "RUNNER_TYPE", + "RUNNER_NAME", + "SCENARIO_TYPE", + "OSL", + "SPEC_DECODING", + "SWEBENCH_GEN_MODE", + "SWEBENCH_USE_MODAL", + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "THINKING_MODE", + "TOTAL_CPU_DRAM_GB", + "WEKA_LOADER_OVERRIDE", +) + +_EVAL_COMMAND = r""" +set -eo pipefail +source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only +check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST +eval_root="/results/${SLURM_JOB_ID}/eval" +mkdir -p "${eval_root}" +cd "${eval_root}" +source /infmax-workspace/benchmarks/benchmark_lib.sh +export EVAL_SERVER_HOST="${SRT_FRONTEND_HOST}" +if [[ -n "${EVAL_CONC:-}" ]]; then + export EVAL_CONCURRENT_REQUESTS="${EVAL_CONC}" +else + export EVAL_CONCURRENT_REQUESTS="$(printf '%s\n' "$CONC_LIST" | tr ' ' '\n' | sort -n | tail -1)" +fi +export CONC="${EVAL_CONCURRENT_REQUESTS}" +bridge_disagg_eval_metadata +run_eval --port "${SRT_FRONTEND_PORT}" +append_lm_eval_summary +""".strip() + + +def prepare_recipe( + recipe: dict[str, Any], + profile: dict[str, Any], + environment: Mapping[str, str], + *, + workspace: Path, + results_root: Path, + aiperf_cache: Path, + image_cache: Path, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Adapt CI metadata without changing the recipe's serving contract.""" + recipe = copy.deepcopy(recipe) + if "base" in recipe: + from srtctl.core.config import generate_override_configs + + _, separator, selector = environment["CONFIG_FILE"].partition(":") + if not separator: + raise ValueError("CONFIG_FILE must select one recipe variant") + variants = generate_override_configs(recipe, selector=selector) + if len(variants) != 1: + raise ValueError("Each CI job must select exactly one recipe variant") + recipe = variants[0][1] + profile = render_cluster_config(profile, dict(environment), {}) + profile.setdefault("default_mounts", {}).update( + { + str(workspace): "/infmax-workspace", + str(results_root): "/results", + str(aiperf_cache): "/aiperf_mmap_cache", + } + ) + image = environment["IMAGE"] + cached_image = image_cache / (image.replace("/", "_").replace(":", "_") + ".sqsh") + # Reuse a provisioned image when available. Otherwise Pyxis imports the + # recipe's image during its normal container lifecycle, not a staging job. + profile.setdefault("containers", {})[recipe["model"]["container"]] = ( + str(cached_image) if cached_image.is_file() else image + ) + benchmark_env = recipe.setdefault("benchmark", {}).setdefault("env", {}) + if environment.get("CLIENT_IMAGE"): + recipe["benchmark"]["container_image"] = environment["CLIENT_IMAGE"] + for key in ( + *_FORWARDED_ENV, + *environment.get("INFERENCEX_RUNTIME_ENV_VARS", "").split(), + ): + value = environment.get(key) + if value: + benchmark_env[key] = value + + _configure_sglang_contract(recipe, environment) + _configure_evaluation(recipe, environment) + return recipe, profile + + +def _configure_sglang_contract(recipe: dict[str, Any], environment: Mapping[str, str]) -> None: + if recipe.get("engine") != "sglang": + return + roles = recipe.get("roles", {}) + prefill = roles.get("prefill", {}).get("args", {}) + decode = roles.get("decode", {}).get("args", {}) + # The Pro-0813 recipe leaves this range empty for the workflow's actual + # concurrency, just as models.yaml sizes each legacy allocation at launch. + if decode.get("cuda-graph-bs-decode") != []: + if prefill.get("enable-dp-attention") and int(environment.get("PREFILL_EP", "1")) > 1: + concurrency = max(int(value) for value in environment["CONC_LIST"].split()) + prefill["max-running-requests"] = concurrency + decode["max-running-requests"] = concurrency + decode_env = roles["decode"].setdefault("env", {}) + dispatch_tokens = max(1, concurrency // int(environment["DECODE_TP"])) + decode_env.setdefault( + "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK", str(dispatch_tokens) + ) + if "MORI_MAX_DISPATCH_TOKENS_DECODE" in decode_env: + mtp_size = int(environment.get("DECODE_MTP_SIZE", "0")) + decode_env["MORI_MAX_DISPATCH_TOKENS_DECODE"] = str( + dispatch_tokens * (mtp_size + 1) + ) + decode_env["SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD"] = str( + 2 * dispatch_tokens + ) + return + concurrency = max(int(value) for value in environment["CONC_LIST"].split()) + if concurrency <= 0: + raise ValueError("CONC_LIST must contain positive concurrency values") + for role, config in (("PREFILL", prefill), ("DECODE", decode)): + if int(environment[f"{role}_TP"]) != config["tp-size"]: + raise ValueError(f"{role}_TP disagrees with the selected recipe") + dp = environment[f"{role}_DP_ATTN"].lower() == "true" + if dp != config.get("enable-dp-attention", False): + raise ValueError(f"{role}_DP_ATTN disagrees with the selected recipe") + config["max-running-requests"] = concurrency * 2 + if environment.get("DISABLE_CUSTOM_ALL_REDUCE") == "1": + config["disable-custom-all-reduce"] = True + graph_max = concurrency // 4 if decode.get("enable-dp-attention") else concurrency * 2 + if graph_max < 1: + raise ValueError("Concurrency is too small for the DP decode graph range") + decode["cuda-graph-bs-decode"] = list(range(1, graph_max + 1)) + if decode.get("enable-dp-attention"): + decode["max-running-requests"] = min(concurrency * 2, graph_max * decode["tp-size"]) + if prefill.get("enable-hierarchical-cache") and "HICACHE_RATIO" in environment: + prefill["hicache-ratio"] = float(environment["HICACHE_RATIO"]) + if "PREFILL_ROUTER_POLICY" in environment: + recipe["frontend"]["args"]["policy"] = environment["PREFILL_ROUTER_POLICY"] + + +def _configure_evaluation(recipe: dict[str, Any], environment: Mapping[str, str]) -> None: + benchmark_env = recipe["benchmark"]["env"] + eval_only = environment.get("EVAL_ONLY", "false").lower() == "true" + run_eval = environment.get("RUN_EVAL", "false").lower() == "true" + if eval_only or run_eval: + roles = recipe.get("roles", {}) + decode_env = roles.get("decode", {}).get("env", {}) + for key in ( + "SGLANG_SIMULATE_ACC_LEN", + "SGLANG_SIMULATE_ACC_METHOD", + "SGLANG_SIMULATE_ACC_TOKEN_MODE", + ): + decode_env.pop(key, None) + server_config = {name: role.get("args", {}) for name, role in roles.items()} + for mode in ("prefill", "decode"): + server_config.get(mode, {}).pop("ep-dispatch-algorithm", None) + + prefill = server_config.get("prefill", server_config.get("agg", {})) + decode = server_config.get("decode", prefill) + + def topology_value(config: dict[str, Any], *keys: str, default: int = 1) -> int: + for key in keys: + if key in config: + return int(config[key]) + return default + + topology_defaults = { + "IS_MULTINODE": "true", + "MODEL_NAME": environment["MODEL"], + "EVAL_MAX_MODEL_LEN": str( + prefill.get("context-length", environment.get("MAX_MODEL_LEN", "16384")) + ), + "PREFILL_TP": str(topology_value(prefill, "tp-size", "tensor-parallel-size")), + "PREFILL_EP": str(topology_value(prefill, "ep-size", "expert-parallel-size")), + "PREFILL_NUM_WORKERS": str( + roles.get("prefill", roles.get("agg", {})).get("workers", 1) + ), + "DECODE_TP": str(topology_value(decode, "tp-size", "tensor-parallel-size")), + "DECODE_EP": str(topology_value(decode, "ep-size", "expert-parallel-size")), + "DECODE_NUM_WORKERS": str(roles.get("decode", roles.get("agg", {})).get("workers", 1)), + "PREFILL_DP_ATTN": str(prefill.get("enable-dp-attention", False)).lower(), + "DECODE_DP_ATTN": str(decode.get("enable-dp-attention", False)).lower(), + } + for key, value in topology_defaults.items(): + benchmark_env.setdefault(key, value) + + if eval_only: + recipe["benchmark"]["command"] = _EVAL_COMMAND + else: + recipe["benchmark"]["command"] = ( + recipe["benchmark"]["command"].rstrip() + "\n" + _EVAL_COMMAND + ) + + +def collect_results( + submission: dict[str, Any], + environment: Mapping[str, str], + *, + workspace: Path, + results_root: Path, +) -> None: + """Collect only this allocation's artifacts into the workflow workspace.""" + job_id = str(submission["slurm_job_id"]) + if not job_id.isdecimal(): + raise ValueError("Submission must identify one numeric Slurm job") + log_dir = Path(submission["output_dir"]) / "logs" + result_dir = results_root / job_id + if log_dir.is_dir(): + with tarfile.open(workspace / "multinode_server_logs.tar.gz", "w:gz") as archive: + archive.add(log_dir, arcname=".") + if result_dir.is_dir(): + shutil.copytree(result_dir, workspace / "LOGS", dirs_exist_ok=True) + + filename = environment["RESULT_FILENAME"] + eval_only = environment.get("EVAL_ONLY", "false").lower() == "true" + if not eval_only and environment.get("IS_AGENTIC", "0") == "1": + if not list(workspace.glob(f"{filename}_conc*.json")): + raise ValueError(f"No AgentX aggregate results found for {filename}") + elif not eval_only: + results = sorted((result_dir / "fixed-seq").glob("*.json")) + if not results: + raise ValueError(f"No fixed-sequence results found in {result_dir}") + prefill_gpus = int(environment["PREFILL_NUM_WORKERS"]) * int(environment["PREFILL_TP"]) + if environment.get("DISAGG", "false").lower() == "true": + decode_gpus = int(environment["DECODE_NUM_WORKERS"]) * int(environment["DECODE_TP"]) + suffix = f"gpus_{prefill_gpus + decode_gpus}_ctx_{prefill_gpus}_gen_{decode_gpus}" + else: + total = ( + prefill_gpus + * int(environment.get("PREFILL_PP_SIZE", "1")) + * int(environment.get("PREFILL_PCP_SIZE", "1")) + ) + suffix = f"gpus_{total}" + for result in results: + match = re.search(r"-c([0-9]+)\.json$", result.name) + if not match: + raise ValueError(f"Cannot parse concurrency from {result}") + destination = workspace / f"{filename}_srt-{job_id}_conc{match[1]}_{suffix}.json" + shutil.copy2(result, destination) + print(f"Collected {destination}") + + if eval_only or environment.get("RUN_EVAL", "false").lower() == "true": + eval_dir = log_dir / "eval_results" + if eval_only and not (eval_dir / "meta_env.json").is_file(): + raise ValueError(f"No eval metadata found in {eval_dir}") + for artifact in eval_dir.glob("*"): + if artifact.is_file(): + shutil.copy2(artifact, workspace / artifact.name) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + prepare = commands.add_parser("prepare") + prepare.add_argument("--recipe", type=Path, required=True) + prepare.add_argument("--profile", type=Path, required=True) + prepare.add_argument("--work-dir", type=Path, required=True) + prepare.add_argument("--aiperf-cache", type=Path, required=True) + prepare.add_argument("--image-cache", type=Path, required=True) + collect = commands.add_parser("collect") + collect.add_argument("--submission", type=Path, required=True) + for command in (prepare, collect): + command.add_argument("--workspace", type=Path, required=True) + command.add_argument("--results-root", type=Path, required=True) + args = parser.parse_args() + if args.command == "prepare": + recipe, profile = prepare_recipe( + yaml.safe_load(args.recipe.read_text()), + yaml.safe_load(args.profile.read_text()), + os.environ, + workspace=args.workspace, + results_root=args.results_root, + aiperf_cache=args.aiperf_cache, + image_cache=args.image_cache, + ) + (args.work_dir / "recipe.yaml").write_text(yaml.safe_dump(recipe, sort_keys=False)) + (args.work_dir / "srtslurm.yaml").write_text(yaml.safe_dump(profile, sort_keys=False)) + else: + collect_results( + json.loads(args.submission.read_text()), + os.environ, + workspace=args.workspace, + results_root=args.results_root, + ) + + +if __name__ == "__main__": + main() diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 86eb8dbb63..eb5584fd0b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8587,3 +8587,20 @@ - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" - "新增 --no-swa-bounded-replay：vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启，其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失，曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃；gfx942 使用同一路径。prefix caching 保持开启" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3337 + +- config-keys: + - qwen3-0.6b-fp16-mi355x-sglang-srt-agg + - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg + - '*-mi355x-*-disagg*' + description: + - 'Refresh the shared AMD srt-slurm integration onto current InferenceX: v2 roles, native cluster profiles, golden + AgentX acceptance, and the synchronized AMD/MoRI/ATOM/runtime stack; retire the replaced manual launcher.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - '*-mi355x-*-disagg*' + - qwen3-0.6b-fp16-mi355x-sglang-srt-agg + - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg + description: + - 'Update the AMD srt-slurm runtime to c3a5cecd5d4513aa69444323cd6bce668dcbdc88: native connector resolution, allocator-owned MoRI-IO listener ports, and discovery-aware vLLM Router readiness. Preserve workload options and engine images.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 diff --git a/runners/check_mi355x_rdma.sh b/runners/check_mi355x_rdma.sh new file mode 100644 index 0000000000..780f72ee69 --- /dev/null +++ b/runners/check_mi355x_rdma.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Fast, per-node fabric preflight for MI355X srt-slurm allocations. This keeps +# the meaningful QoS/DCQCN gate from the retired amd_utils launcher without its +# Docker or job-control plumbing. + +log() { printf '[%s] %s\n' "$(hostname -s)" "$*"; } +fail() { log "RDMA preflight failed: $*" >&2; exit 1; } + +source "$(dirname "${BASH_SOURCE[0]}")/../benchmarks/benchmark_lib.sh" --validation-only +check_env_vars IBDEVICES +expected_devices="$IBDEVICES" +IFS=',' read -r -a devices <<< "$expected_devices" +for device in "${devices[@]}"; do + [[ -d "/sys/class/infiniband/${device}" ]] || fail "missing device ${device}" +done +log "found all ${#devices[@]} expected RDMA devices: ${expected_devices}" + +if ! command -v nicctl >/dev/null 2>&1; then + log "nicctl is unavailable; device presence passed, QoS/DCQCN checks skipped" + exit 0 +fi + +probe=$(sudo -n nicctl show version firmware 2>&1 || true) +if grep -qiE 'No AMD NICs|Invalid card handle|Failed to get NIC' <<< "$probe"; then + fail "nicctl cannot access the AMD NICs" +fi + +qos=$(sudo -n nicctl show qos 2>/dev/null) || fail "nicctl show qos failed" +classification=$(awk '/Classification type/ {print $NF; exit}' <<< "$qos") +[[ "$classification" == "DSCP" ]] || fail "classification is ${classification:-unset}, expected DSCP" + +priorities=$(awk '/PFC no-drop priorities/ {print $NF; exit}' <<< "$qos") +bitmap=$(awk '/PFC priority bitmap/ {print $NF; exit}' <<< "$qos") +[[ -n "$priorities" ]] || fail "PFC no-drop priorities are missing" +[[ -n "$bitmap" && "$bitmap" != "0x0" ]] || fail "PFC is disabled" +IFS=',' read -r -a priority_values <<< "$priorities" +for priority in "${priority_values[@]}"; do + priority="${priority//[^0-9]/}" + [[ -n "$priority" ]] || fail "invalid PFC priority list: ${priorities}" + (( bitmap & (1 << priority) )) || fail "PFC bitmap ${bitmap} does not cover priority ${priority}" +done + +dcqcn=$(sudo -n nicctl show dcqcn 2>/dev/null) || fail "nicctl show dcqcn failed" +device_count=$(grep -c 'ROCE device' <<< "$dcqcn" || true) +(( device_count > 0 )) || fail "no RoCE devices reported by nicctl" +if grep 'Status' <<< "$dcqcn" | grep -qv 'Enabled'; then + fail "DCQCN is disabled on at least one RoCE device" +fi +cnp_count=$(awk '/DSCP value used for CNP/ {print $NF}' <<< "$dcqcn" | sort -u | grep -c . || true) +(( cnp_count == 1 )) || fail "CNP DSCP is inconsistent across NICs" + +log "RDMA QoS/DCQCN preflight passed" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index e27c0178d0..6bd157c483 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -1,221 +1,79 @@ #!/usr/bin/env bash source "$(dirname "${BASH_SOURCE[0]}")/../benchmarks/benchmark_lib.sh" --validation-only || exit 1 +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || exit 1 check_env_vars EVAL_ONLY IS_AGENTIC IS_MULTINODE KEEP_LOGS RUN_EVAL -scancel_sync() { - local jobid=$1 - local timeout=${2:-600} - local interval=10 - local start - start=$(date +%s) - - echo "[scancel_sync] Requesting cancel of job $jobid" - scancel "$jobid" || true - - while [[ -n "$(squeue -j "$jobid" --noheader 2>/dev/null)" ]]; do - local now - now=$(date +%s) - if (( now - start >= timeout )); then - echo "[scancel_sync][WARN] job $jobid still present after ${timeout}s" - return 1 - fi - echo "[scancel_sync] waiting for job $jobid to exit. $((timeout-(now-start))) secs remaining..." - sleep "$interval" - done - echo "[scancel_sync] job $jobid exited" - return 0 -} +# One cluster entry point. Migrated recipes delegate the entire allocation and +# serving lifecycle to srt-slurm; the adapter only maps CI inputs and artifacts. +if [[ -n "${CONFIG_FILE:-}" ]]; then + set -eo pipefail + check_env_vars GITHUB_WORKSPACE RESULT_FILENAME IMAGE MODEL FRAMEWORK \ + SRT_SLURM_REPOSITORY SRT_SLURM_COMMIT SRT_SLURM_SHARED_BASE \ + SRT_SLURM_CLUSTER_CONFIG SRT_SLURM_COMPUTE_ARCH AIPERF_MMAP_CACHE_HOST_PATH \ + GITHUB_RUN_ID GITHUB_RUN_ATTEMPT RUNNER_NAME HOME + + SHARED_BASE="$SRT_SLURM_SHARED_BASE" + SHARED_AIPERF_CACHE="$AIPERF_MMAP_CACHE_HOST_PATH" + SHARED_RESULTS="${SHARED_BASE}/results" + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RUNNER_NAME}" + WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" + SRT_REPO_DIR="${WORK_DIR}/srt-slurm" + CONFIG_PATH="${CONFIG_FILE%%:*}" + LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" + CLUSTER_PROFILE="$SRT_SLURM_CLUSTER_CONFIG" + ADAPTER=infx.workflows.srt_slurm + + # Only shared, user-owned directories are prepared here. Pyxis manages + # container imports/lifetime; engines use the cache mounted by the profile. + mkdir -p "$WORK_DIR" "$SHARED_RESULTS" "$SHARED_AIPERF_CACHE" + setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" 0 + + export PATH="$HOME/.local/bin:$PATH" + cd "$SRT_REPO_DIR" + uv venv --python 3.12 + uv pip install -e . + source .venv/bin/activate + + PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" python -m "$ADAPTER" prepare \ + --recipe "$LOCAL_RECIPE" --profile "$CLUSTER_PROFILE" \ + --work-dir "$WORK_DIR" --workspace "$GITHUB_WORKSPACE" \ + --results-root "$SHARED_RESULTS" --aiperf-cache "$SHARED_AIPERF_CACHE" \ + --image-cache "${SHARED_BASE}/containers" + export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" + # Use upstream's normal setup with the prepared cluster configuration; + # providing it first also keeps setup non-interactive in Actions. + cp "$SRTSLURM_CONFIG" "$SRT_REPO_DIR/srtslurm.yaml" + make setup ARCH="$SRT_SLURM_COMPUTE_ARCH" + PREPARED_RECIPE="${WORK_DIR}/recipe.yaml" + + # Do not leak the CI host's bytecode-cache location into the containers. + # Recipes may still explicitly set their own container cache environment. + unset PYTHONPYCACHEPREFIX + apply_srt_recipe "$PREPARED_RECIPE" "$FRAMEWORK" -f "$PREPARED_RECIPE" \ + --tags "mi355x,inferencex,github-actions,${RUN_KEY}" --json \ + > "${WORK_DIR}/submission.json" + JOB_ID=$(jq -er '.slurm_job_id' "${WORK_DIR}/submission.json") + OUTPUT_DIR=$(jq -er '.output_dir' "${WORK_DIR}/submission.json") + echo "SRT_SLURM_JOB_ID=$JOB_ID" + + # srt-slurm handles queue/accounting transitions and propagates failures. + # Always collect available diagnostics, including on an unsuccessful job. + job_rc=0 + srtctl wait "$JOB_ID" --log-file "${OUTPUT_DIR}/logs/sweep_${JOB_ID}.log" || job_rc=$? + collect_rc=0 + PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" python -m "$ADAPTER" collect --submission "${WORK_DIR}/submission.json" \ + --workspace "$GITHUB_WORKSPACE" --results-root "$SHARED_RESULTS" || collect_rc=$? + [[ "$job_rc" -eq 0 ]] || exit "$job_rc" + [[ "$collect_rc" -eq 0 ]] || exit "$collect_rc" + printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" + echo "MI355X srt-slurm validation completed successfully" + exit 0 +fi if [[ "$IS_MULTINODE" == "true" ]]; then - - set -x - - export SLURM_ACCOUNT="$USER" - export SLURM_PARTITION="compute" - export SLURM_JOB_NAME="benchmark-sglang-disagg.job" - - export MODEL_NAME=${MODEL##*/} - export MODEL_PATH="/it-share/data" - export IBDEVICES="rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" - export MORI_RDMA_TC=104 - - export MODEL_DIR="$MODEL_PATH" # job.slurm uses MODEL_DIR - export GPUS_PER_NODE=8 # MI355X has 8 GPUs (set to 4 for MI325X) - - export ISL="$ISL" - export OSL="$OSL" - - check_env_vars BENCHMARK_LOGS_DIR - mkdir -p "$BENCHMARK_LOGS_DIR" - sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true - - # Root-owned container output must go even on early exit, or the next job's - # checkout hits EACCES; slurm logs are saved as artifacts first. KEEP_LOGS=1 - # disables the trap for local debugging. - cleanup_and_save_logs() { - if [[ -n "${GITHUB_ACTIONS:-}" && -n "${JOB_ID:-}" ]]; then - local art_dir="$GITHUB_WORKSPACE/benchmark_artifacts" - mkdir -p "$art_dir" - cp -r "$BENCHMARK_LOGS_DIR"/slurm_job-${JOB_ID}.{out,err} "$art_dir/" 2>/dev/null || true - fi - local err_file="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID:-unknown}.err" - if [[ -s "$err_file" ]]; then - echo "=== Slurm job stderr ===" - tail -100 "$err_file" - echo "========================" - fi - sudo rm -rf "$BENCHMARK_LOGS_DIR" 2>/dev/null || true - } - if [[ "${KEEP_LOGS}" == "1" ]]; then - trap '' EXIT - else - trap cleanup_and_save_logs EXIT - fi - - SCRIPT_NAME="${EXP_NAME%%_*}_${PRECISION}_mi355x_${FRAMEWORK}.sh" - if [[ "$FRAMEWORK" == "sglang-disagg" ]] || [[ "$FRAMEWORK" == "vllm-disagg" ]] || [[ "$FRAMEWORK" == "atom-disagg" ]]; then - # Agentic recipes under multi_node/agentic/ export the HiCache tunables; - # fixed-seq-len recipes live at the multi_node/ root. - if [[ "${SCENARIO_SUBDIR}" == "agentic/" ]]; then - BENCHMARK_SUBDIR="multi_node/agentic" - else - BENCHMARK_SUBDIR="multi_node" - fi - else - BENCHMARK_SUBDIR="single_node/fixed_seq_len" - fi - JOB_ID=$(bash "benchmarks/${BENCHMARK_SUBDIR}/${SCRIPT_NAME}") - - LOG_FILE="$BENCHMARK_LOGS_DIR/slurm_job-${JOB_ID}.out" - - sleep 10 - - while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -u "$USER" --noheader --format='%i' | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - sleep 5 - done - - set +x - - ( - while squeue -u $USER --noheader --format='%i' | grep -q "$JOB_ID"; do - sleep 10 - done - ) & - POLL_PID=$! - - # -F follows by name and polls; inotify does not work on NFS. - tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - - wait $POLL_PID - - set -x - - - - - if [[ "${EVAL_ONLY}" != "true" && "${IS_AGENTIC}" != "1" ]]; then - cat > collect_latest_results.py <<'PY' -import os, sys -job_dir, isl, osl, nexp, framework = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] -logs_root = f"{job_dir}/logs/" -candidates = [] -if os.path.isdir(logs_root): - for name in os.listdir(logs_root): - subdir = f"{logs_root}{name}/{framework}_isl_{isl}_osl_{osl}" - if os.path.isdir(subdir): - candidates.append(subdir) -for path in sorted(candidates, key=os.path.getmtime, reverse=True)[:nexp]: - print(path) -PY - - LOGS_DIR=$(python3 collect_latest_results.py "$BENCHMARK_LOGS_DIR" "$ISL" "$OSL" 1 "$FRAMEWORK") - if [ -z "$LOGS_DIR" ]; then - echo "No logs directory found for ISL=${ISL}, OSL=${OSL}" - exit 1 - fi - - echo "Found logs directory: $LOGS_DIR" - ls -la "$LOGS_DIR" - - for result_file in $(find $LOGS_DIR -type f); do - file_name=$(basename $result_file) - if [ -f $result_file ]; then - WORKSPACE_RESULT_FILE="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${file_name}" - echo "Found result file ${result_file}. Copying it to ${WORKSPACE_RESULT_FILE}" - cp $result_file $WORKSPACE_RESULT_FILE - fi - done - fi - - if [[ "${RUN_EVAL}" == "true" ]]; then - EVAL_DIR=$(find "$BENCHMARK_LOGS_DIR/logs" -type d -name eval_results 2>/dev/null | head -1) - if [ -n "$EVAL_DIR" ] && [ -d "$EVAL_DIR" ]; then - echo "Extracting eval results from $EVAL_DIR" - shopt -s nullglob - for eval_file in "$EVAL_DIR"/*; do - [ -f "$eval_file" ] || continue - eval_dest="$GITHUB_WORKSPACE/$(basename "$eval_file")" - rm -f "$eval_dest" - # Eval artifacts are root-owned from the container; sudo overwrites - # stale root-owned files left by prior runs. - if sudo cp "$eval_file" "$eval_dest"; then - sudo chown "$(id -u):$(id -g)" "$eval_dest" 2>/dev/null || true - echo "Copied eval artifact: $(basename "$eval_file")" - else - echo "ERROR: failed to copy eval artifact: $(basename "$eval_file")" >&2 - exit 1 - fi - done - shopt -u nullglob - else - echo "WARNING: RUN_EVAL=true but no eval results found under $BENCHMARK_LOGS_DIR/logs" - fi - fi - - # benchmark-multinode-tmpl.yml uploads LOGS/agentic/conc_*/... and - # multinode_server_logs.tar.gz, so preserve trace_replay.sh's conc_/ - # nesting before the logs dir is removed below. - if [[ "${IS_AGENTIC}" == "1" ]]; then - JOB_LOGS_DIR="$BENCHMARK_LOGS_DIR/logs/slurm_job-${JOB_ID}" - if [ -d "$JOB_LOGS_DIR" ]; then - AGENTIC_SRC="$JOB_LOGS_DIR/agentic" - if [ -d "$AGENTIC_SRC" ] && find "$AGENTIC_SRC" -mindepth 1 -maxdepth 1 -type d -name 'conc_*' -print -quit 2>/dev/null | grep -q .; then - echo "Staging agentic raw artifacts from $AGENTIC_SRC" - mkdir -p "$GITHUB_WORKSPACE/LOGS/agentic" - cp -r "$AGENTIC_SRC"/. "$GITHUB_WORKSPACE/LOGS/agentic/" - # Container artifacts arrive root-owned; later jobs, possibly a - # different runner user, must be able to remove LOGS/. - sudo chown -R "$(id -u):$(id -g)" "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true - chmod -R a+rwX "$GITHUB_WORKSPACE/LOGS" 2>/dev/null || true - ls -laR "$GITHUB_WORKSPACE/LOGS/agentic" - else - echo "WARNING: no agentic conc_*/ artifacts found under $JOB_LOGS_DIR/agentic" - fi - if tar czf "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -C "$JOB_LOGS_DIR" . 2>/dev/null; then - echo "Created multinode_server_logs.tar.gz" - else - echo "WARNING: failed to create multinode_server_logs.tar.gz" - fi - else - echo "WARNING: agentic staging skipped; $JOB_LOGS_DIR not found" - fi - fi - - echo "All result files processed" - # Synchronous cancel so the NFS file handles are released before cleanup. - set +x - scancel_sync $JOB_ID - set -x - echo "Canceled the slurm job $JOB_ID" - - sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true - + echo "MI355X multi-node jobs require a CONFIG_FILE srt-slurm recipe." >&2 + exit 1 else export HF_HUB_CACHE_MOUNT="/var/lib/hf-hub-cache/" diff --git a/runners/prepare_mi355x_srt_host.sh b/runners/prepare_mi355x_srt_host.sh new file mode 100644 index 0000000000..792a62f65a --- /dev/null +++ b/runners/prepare_mi355x_srt_host.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -eo pipefail + +bash "$(dirname "${BASH_SOURCE[0]}")/check_mi355x_rdma.sh" + +# Preserve the legacy bare-process GPU drain gate. Slurm owns the node, but a +# process left outside the prior job's container can still retain VRAM and make +# the next model load fail much later with a misleading OOM. +( + unset IS_AGENTIC SCENARIO_TYPE + # shellcheck source=benchmarks/benchmark_lib.sh + source "$(dirname "${BASH_SOURCE[0]}")/../benchmarks/benchmark_lib.sh" + wait_for_amd_gpu_clean +) + +# Some MI355X experiments reserve large 2 MiB HugeTLB pools and leave the +# reservation behind after their Slurm allocation exits. Those free hugepages +# remain unavailable to ordinary host allocations, which can make a later +# unchanged SGLang HiCache recipe fail even on a 3 TiB node. Reclaim only free +# pages; pages currently used or reserved by host services are preserved. +meminfo=/proc/meminfo +nr_hugepages=/proc/sys/vm/nr_hugepages + +read_hugepage_value() { + local key="$1" + awk -v key="${key}:" '$1 == key {print $2}' "$meminfo" +} + +total=$(read_hugepage_value HugePages_Total) +free=$(read_hugepage_value HugePages_Free) +reserved=$(read_hugepage_value HugePages_Rsvd) +used=$((total - free)) +target=$((used + reserved)) + +echo "MI355X host memory before preparation:" +grep -E '^(MemAvailable|HugePages_Total|HugePages_Free|HugePages_Rsvd|HugePages_Surp|Hugetlb):' "$meminfo" + +if (( target < total )); then + printf '%s\n' "$target" | sudo -n tee "$nr_hugepages" >/dev/null +fi + +after_total=$(read_hugepage_value HugePages_Total) +after_free=$(read_hugepage_value HugePages_Free) +echo "MI355X host memory after preparation:" +grep -E '^(MemAvailable|HugePages_Total|HugePages_Free|HugePages_Rsvd|HugePages_Surp|Hugetlb):' "$meminfo" + +if (( after_total - after_free < used )); then + echo "Host preparation released hugepages that were in use" >&2 + exit 1 +fi +if (( after_free > reserved )); then + echo "Host preparation could not reclaim all unused hugepages" >&2 + exit 1 +fi diff --git a/runners/runtime_settings.sh b/runners/runtime_settings.sh index 09de8f25e5..949f34fdbf 100644 --- a/runners/runtime_settings.sh +++ b/runners/runtime_settings.sh @@ -41,6 +41,12 @@ case "${RUNNER_NAME%%_*}" in mi355x-amds) check_env_vars GITHUB_WORKSPACE export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" + export SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git + export SRT_SLURM_COMMIT=c3a5cecd5d4513aa69444323cd6bce668dcbdc88 + export SRT_SLURM_SHARED_BASE=/it-share/gharunners2/srt-slurm + export SRT_SLURM_CLUSTER_CONFIG="$GITHUB_WORKSPACE/runners/srt-slurm/mi355x-amds.yaml" + export SRT_SLURM_COMPUTE_ARCH=x86_64 + export AIPERF_MMAP_CACHE_HOST_PATH=/it-share/aiperf-cache ;; rtx6000pro-lat) export HF_HUB_CACHE_MOUNT=/var/lib/inferencex/hf-hub-cache diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 67a6eac780..349fa2e524 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -57,8 +57,14 @@ PYENV # Custom benchmarks inherit exported workflow settings through sbatch/srun; # native recipe environment and benchmark.env retain their override priority. local source="$INFERENCEX_SLURM_UTILS_DIR/../utils/srt-slurm" - if [[ "$framework" == "tilert" ]]; then - # Sole fork exception until NVIDIA supports the TileRT backend and router. + if [[ -n "${SRT_SLURM_REPOSITORY:-}" ]]; then + check_env_vars SRT_SLURM_COMMIT + git init "$destination" || return 1 + git -C "$destination" remote add origin "$SRT_SLURM_REPOSITORY" || return 1 + git -C "$destination" fetch --depth=1 origin "$SRT_SLURM_COMMIT" || return 1 + git -C "$destination" checkout --detach "$SRT_SLURM_COMMIT" || return 1 + elif [[ "$framework" == "tilert" ]]; then + # Fork exception until NVIDIA supports the TileRT backend and router. SRT_SLURM_COMMIT=6bc3f306bdafa1edfb5dded2fcda8f1ccede1bde git init "$destination" || return 1 git -C "$destination" remote add origin https://github.com/SemiAnalysisAI/srt-slurm.git || return 1 diff --git a/runners/srt-slurm/mi355x-amds.yaml b/runners/srt-slurm/mi355x-amds.yaml new file mode 100644 index 0000000000..4d695b99a2 --- /dev/null +++ b/runners/srt-slurm/mi355x-amds.yaml @@ -0,0 +1,46 @@ +# srt-slurm cluster profile for the MI355X AMD Slurm cluster. The login and +# compute nodes share /it-share, so source, output, image, and result paths do +# not need node-local transport. + +cluster: mi355x-amds +default_partition: compute +default_time_limit: "01:00:00" +output_dir: /it-share/gharunners2/srt-slurm/outputs + +gpus_per_node: 8 +visible_devices_env: ROCR_VISIBLE_DEVICES +default_gpu_exporter: null +network_interface: eno0 + +use_gpus_per_node_directive: false +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: false + +default_sbatch_directives: + gres: gpu:8 + cpus-per-task: "32" + mem: "128G" + +model_paths: + DeepSeek-V4-Pro-0813: /it-share/data/DeepSeek-V4-Pro-0813 + +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + # This host directory is already the Hub cache, not HF_HOME. + /it-share/hf-hub-cache: /hf_hub_cache/hub + /it-share/hf_home: /it-share/hf_home + # The launcher appends the exact shared workspace and result roots. + + +nginx_raise_ulimit: false + +# Preserve the existing per-allocation fabric/memory prerequisites through the +# upstream host-setup lifecycle, not a separate staging Slurm job. +default_host_setup: + # The existing GPU-drain check can take 15 minutes; leave room for fabric + # checks and host-memory preparation instead of cutting it off at 5 minutes. + timeout_seconds: 1200 + commands: + - IBDEVICES=rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7 bash "${GITHUB_WORKSPACE}/runners/prepare_mi355x_srt_host.sh" + nodes: all diff --git a/utils/test_amd_sglang_cli.py b/utils/test_amd_sglang_cli.py deleted file mode 100644 index 178be9a925..0000000000 --- a/utils/test_amd_sglang_cli.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import argparse - -import pytest - -from benchmarks.multi_node.amd_utils.sglang_cli import cuda_graph_flags - - -@pytest.mark.parametrize("split_available", [False, True]) -def test_legacy_option_preserves_batch_sizes(split_available: bool) -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--cuda-graph-bs", nargs="+", type=int) - if split_available: - parser.add_argument("--cuda-graph-bs-prefill", nargs="+", type=int) - parser.add_argument("--cuda-graph-bs-decode", nargs="+", type=int) - - prefill, decode = cuda_graph_flags(parser) - - for flag in (prefill, decode): - args = parser.parse_args([flag, "1", "4", "8"]) - assert args.cuda_graph_bs == [1, 4, 8] - - -def test_split_options_select_the_correct_phase() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--cuda-graph-bs-prefill", nargs="+", type=int) - parser.add_argument("--cuda-graph-bs-decode", nargs="+", type=int) - - prefill, decode = cuda_graph_flags(parser) - args = parser.parse_args([prefill, "2", "4", decode, "1", "8"]) - - assert args.cuda_graph_bs_prefill == [2, 4] - assert args.cuda_graph_bs_decode == [1, 8] - - -@pytest.mark.parametrize("available", [None, "--cuda-graph-bs-decode"]) -def test_missing_supported_options_fail_closed(available: str | None) -> None: - parser = argparse.ArgumentParser() - if available: - parser.add_argument(available, nargs="+", type=int) - - with pytest.raises(ValueError, match="no supported CUDA graph"): - cuda_graph_flags(parser) diff --git a/utils/test_srt_slurm.py b/utils/test_srt_slurm.py new file mode 100644 index 0000000000..b62820eb2f --- /dev/null +++ b/utils/test_srt_slurm.py @@ -0,0 +1,161 @@ +"""Behavioral checks for the CI-to-srt-slurm adapter; no GPU or Slurm required.""" + +import json +from pathlib import Path + +import pytest + +from infx.workflows.srt_slurm import collect_results, prepare_recipe + + +def prepare(tmp_path: Path, *, dp: bool, concurrency: str, eval_only: bool = False): + role = {"tp-size": 8, "enable-dp-attention": dp} + recipe = { + "model": {"container": "engine:image"}, + "frontend": {"args": {}}, + "schema": 2, + "engine": "sglang", + "roles": { + "prefill": {"args": {**role, "enable-hierarchical-cache": True}}, + "decode": { + "args": {**role, "cuda-graph-bs-decode": []}, + "env": { + "SGLANG_SIMULATE_ACC_LEN": "2.3", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + "ENGINE_OTHER_SETTING": "keep", + }, + }, + }, + "benchmark": {"type": "custom", "command": "benchmark", "env": {}}, + } + env = { + "IMAGE": "engine:image", + "MODEL": "example/model", + "CLIENT_IMAGE": "client:image", + "CONC_LIST": concurrency, + "PREFILL_TP": "8", + "DECODE_TP": "8", + "PREFILL_DP_ATTN": str(dp).lower(), + "DECODE_DP_ATTN": str(dp).lower(), + "DISABLE_CUSTOM_ALL_REDUCE": "0", + "HICACHE_RATIO": "1.25", + "PREFILL_ROUTER_POLICY": "consistent_hashing", + "EVAL_ONLY": str(eval_only).lower(), + "INFERENCEX_RUNTIME_ENV_VARS": "CUSTOM_CLIENT_SETTING", + "CUSTOM_CLIENT_SETTING": "from-caller", + "AIPERF_EXPERIMENTAL_FAST": "0", + "REQUIRE_POWER": "0", + } + return prepare_recipe( + recipe, + {}, + env, + workspace=tmp_path, + results_root=tmp_path / "results", + aiperf_cache=tmp_path / "cache", + image_cache=tmp_path / "images", + )[0] + + +@pytest.mark.parametrize( + ("dp", "concurrency", "requests", "graphs"), + [(False, "2 4", 8, [1, 2, 3, 4, 5, 6, 7, 8]), (True, "8 12", 24, [1, 2, 3])], +) +def test_concurrency_sizes_admission_and_phase_graphs( + tmp_path, dp, concurrency, requests, graphs +): + result = prepare(tmp_path, dp=dp, concurrency=concurrency) + prefill = result["roles"]["prefill"]["args"] + decode = result["roles"]["decode"]["args"] + assert prefill["max-running-requests"] == requests + assert decode["max-running-requests"] == requests + assert decode["cuda-graph-bs-decode"] == graphs + assert prefill["hicache-ratio"] == 1.25 + assert result["benchmark"]["container_image"] == "client:image" + assert result["benchmark"]["env"]["CUSTOM_CLIENT_SETTING"] == "from-caller" + assert result["benchmark"]["env"]["AIPERF_EXPERIMENTAL_FAST"] == "0" + assert result["benchmark"]["env"]["REQUIRE_POWER"] == "0" + + +def test_eval_removes_synthetic_acceptance_but_keeps_other_engine_environment(tmp_path): + result = prepare(tmp_path, dp=False, concurrency="4", eval_only=True) + assert result["roles"]["decode"]["env"] == {"ENGINE_OTHER_SETTING": "keep"} + assert result["benchmark"]["env"]["DECODE_TP"] == "8" + + +def test_rejects_dp_concurrency_that_cannot_capture_one_decode_batch(tmp_path): + with pytest.raises(ValueError, match="too small"): + prepare(tmp_path, dp=True, concurrency="2") + + +def test_fixed_sequence_preserves_dispatch_pin_and_sizes_admission(tmp_path): + recipe = { + "schema": 2, + "engine": "sglang", + "model": {"container": "engine:image"}, + "roles": { + "prefill": {"args": {"enable-dp-attention": True}}, + "decode": { + "args": {"cuda-graph-bs": [1, 2, 4]}, + "env": { + "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", + "MORI_MAX_DISPATCH_TOKENS_DECODE": "1", + }, + }, + }, + } + result, profile = prepare_recipe( + recipe, + {"output_dir": "${RESULTS}"}, + { + "IMAGE": "engine:image", + "CONC_LIST": "16 48", + "PREFILL_EP": "8", + "DECODE_TP": "8", + "DECODE_MTP_SIZE": "1", + "RESULTS": "/shared/results", + }, + workspace=tmp_path, + results_root=tmp_path / "results", + aiperf_cache=tmp_path / "cache", + image_cache=tmp_path / "images", + ) + assert result["roles"]["prefill"]["args"]["max-running-requests"] == 48 + assert result["roles"]["decode"]["args"] == { + "cuda-graph-bs": [1, 2, 4], + "max-running-requests": 48, + } + assert result["roles"]["decode"]["env"] == { + "SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", + "MORI_MAX_DISPATCH_TOKENS_DECODE": "12", + "SGLANG_MORI_DISPATCH_INTER_KERNEL_SWITCH_THRESHOLD": "12", + } + assert profile["output_dir"] == "/shared/results" + + +@pytest.mark.parametrize("has_metadata", [True, False]) +def test_collect_eval_from_native_job_logs(tmp_path, has_metadata): + workspace = tmp_path / "workspace" + workspace.mkdir() + output = tmp_path / "job-42" + eval_dir = output / "logs" / "eval_results" + eval_dir.mkdir(parents=True) + (eval_dir / "results.json").write_text('{"score": 0.9}') + if has_metadata: + (eval_dir / "meta_env.json").write_text('{"eval_concurrency": 48}') + submission = {"slurm_job_id": "42", "output_dir": str(output)} + env = {"RESULT_FILENAME": "example", "EVAL_ONLY": "true"} + if not has_metadata: + with pytest.raises(ValueError, match="No eval metadata"): + collect_results( + submission, env, workspace=workspace, results_root=tmp_path / "results" + ) + return + collect_results( + submission, env, workspace=workspace, results_root=tmp_path / "results" + ) + assert json.loads((workspace / "results.json").read_text()) == {"score": 0.9} + assert json.loads((workspace / "meta_env.json").read_text()) == { + "eval_concurrency": 48 + } diff --git a/utils/test_synthetic_acceptance.py b/utils/test_synthetic_acceptance.py index 0c26b9fbc6..ffa8b01aa0 100644 --- a/utils/test_synthetic_acceptance.py +++ b/utils/test_synthetic_acceptance.py @@ -163,7 +163,7 @@ def test_kimi_curve_requires_an_explicit_supported_sampler( ("framework", "args", "environment", "expected"), [ ( - "dynamo-sglang", + "sglang-disagg", { "speculative-algorithm": "DSpark", "speculative-dspark-block-size": 3, @@ -229,7 +229,7 @@ def test_engine_token_selection_and_environment( "environment", [{"EVAL_ONLY": "true"}, {"IS_AGENTIC": "0"}, {"SPEC_DECODING": "none"}], ) -@pytest.mark.parametrize("framework", ["vllm", "dynamo-sglang", "trt"]) +@pytest.mark.parametrize("framework", ["vllm", "vllm-disagg", "sglang", "dynamo-sglang", "trt"]) def test_real_runs_clear_synthetic_without_a_curve( tmp_path: Path, framework: str, environment: dict[str, str] ) -> None: @@ -251,7 +251,7 @@ def test_real_runs_clear_synthetic_without_a_curve( build_overrides(recipe, framework, env, golden_dir=tmp_path / "absent"), ) role = result["roles"]["agg"] - if framework == "vllm": + if framework in {"vllm", "vllm-disagg"}: assert json.loads(role["args"]["speculative-config"]) == { "method": "dspark", "num_speculative_tokens": 3, From 4bc0acd5bc32640744724ce8b1329987da63745e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 22 Sep 2026 15:29:40 -0500 Subject: [PATCH 2/6] test: validate MI300X aggregate through shared srt runtime --- .../qwen3-0.6b/mi300x/agg-fixed-seq.yaml | 94 +++++++++++++++++++ configs/smoke/amd-srt-mi300x.yaml | 24 +++++ perf-changelog.yaml | 8 ++ runners/launch_mi300x-amd.sh | 6 ++ runners/launch_mi355x-amds.sh | 65 +------------ runners/runtime_settings.sh | 12 ++- runners/srt-slurm/mi300x-amd.yaml | 19 ++++ runners/srt_runtime.sh | 70 ++++++++++++++ 8 files changed, 235 insertions(+), 63 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml create mode 100644 configs/smoke/amd-srt-mi300x.yaml create mode 100644 runners/srt-slurm/mi300x-amd.yaml create mode 100644 runners/srt_runtime.sh diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml new file mode 100644 index 0000000000..d4c5bfba0c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml @@ -0,0 +1,94 @@ +# Minimal MI300X aggregate validation. It exercises native SGLang Router and +# one aggregate SGLang worker; this is orchestration validation, not tuning. + +schema: 2 +name: "mi300x-sglang-qwen3-0.6b-agg-fixed-seq" + +model: + path: "hf:Qwen/Qwen3-0.6B" + container: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910" + precision: "fp16" + +identity: + model: + repo: "Qwen/Qwen3-0.6B" + container: + image: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910" + +slurm: + time_limit: "00:30:00" + +resources: + gpu_type: "mi300x" + gpus_per_node: 8 +frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 1 + + env: + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + args: + served-model-name: "Qwen/Qwen3-0.6B" + tensor-parallel-size: 1 + mem-fraction-static: 0.50 + context-length: 2048 + max-running-requests: 8 + disable-cuda-graph: true + enable-metrics: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 180 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in ${CONC_LIST}; do + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3-0.6B \ + --tokenizer Qwen/Qwen3-0.6B \ + --dataset-name random \ + --random-input-len 128 \ + --random-output-len 32 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "${concurrency}" \ + --num-prompts "$((concurrency * 4))" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3-0.6b-agg-isl128-osl32-c${concurrency}.json"; + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/smoke/amd-srt-mi300x.yaml b/configs/smoke/amd-srt-mi300x.yaml new file mode 100644 index 0000000000..5d1aeaaa78 --- /dev/null +++ b/configs/smoke/amd-srt-mi300x.yaml @@ -0,0 +1,24 @@ +qwen3-0.6b-fp16-mi300x-sglang-srt-agg-smoke: + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:mi300x-amd + precision: fp16 + framework: sglang + multinode: true + disagg: false + scenarios: + fixed-seq-len: + - isl: 128 + osl: 32 + search-space: + - conc-list: + - 4 + num-nodes: 1 + worker: + num-worker: 1 + tp: 1 + ep: 1 + dp-attn: false + additional-settings: + - CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml diff --git a/perf-changelog.yaml b/perf-changelog.yaml index eb5584fd0b..02ec449f55 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8604,3 +8604,11 @@ description: - 'Update the AMD srt-slurm runtime to c3a5cecd5d4513aa69444323cd6bce668dcbdc88: native connector resolution, allocator-owned MoRI-IO listener ports, and discovery-aware vLLM Router readiness. Preserve workload options and engine images.' pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - '*-mi355x-*-disagg*' + - qwen3-0.6b-fp16-mi355x-sglang-srt-agg + - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg + description: + - 'Pin AMD runtime 7a7ca652fad37026bbf476b3f4368cd34c73c1c6 and share the native srt-slurm invocation between MI355X and MI300X. Add an explicit MI300X Qwen3-0.6B aggregate smoke matrix without changing production serving settings.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 diff --git a/runners/launch_mi300x-amd.sh b/runners/launch_mi300x-amd.sh index 6d88f743cd..b7390b0dbd 100644 --- a/runners/launch_mi300x-amd.sh +++ b/runners/launch_mi300x-amd.sh @@ -4,6 +4,12 @@ source "$(dirname "${BASH_SOURCE[0]}")/../benchmarks/benchmark_lib.sh" --validat check_env_vars IS_MULTINODE set -eo pipefail +if [[ -n "${CONFIG_FILE:-}" ]]; then + source "$(dirname "${BASH_SOURCE[0]}")/srt_runtime.sh" + run_srt_recipe_job + exit $? +fi + export HF_HUB_CACHE_MOUNT="/raid/inferencex/models/hub" export AIPERF_MMAP_CACHE_MOUNT="/raid/inferencex/aiperf-mmap-cache" export AIPERF_DATASET_MMAP_CACHE_DIR="/aiperf_mmap_cache" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 6bd157c483..0f8f27f562 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -7,68 +7,9 @@ check_env_vars EVAL_ONLY IS_AGENTIC IS_MULTINODE KEEP_LOGS RUN_EVAL # One cluster entry point. Migrated recipes delegate the entire allocation and # serving lifecycle to srt-slurm; the adapter only maps CI inputs and artifacts. if [[ -n "${CONFIG_FILE:-}" ]]; then - set -eo pipefail - check_env_vars GITHUB_WORKSPACE RESULT_FILENAME IMAGE MODEL FRAMEWORK \ - SRT_SLURM_REPOSITORY SRT_SLURM_COMMIT SRT_SLURM_SHARED_BASE \ - SRT_SLURM_CLUSTER_CONFIG SRT_SLURM_COMPUTE_ARCH AIPERF_MMAP_CACHE_HOST_PATH \ - GITHUB_RUN_ID GITHUB_RUN_ATTEMPT RUNNER_NAME HOME - - SHARED_BASE="$SRT_SLURM_SHARED_BASE" - SHARED_AIPERF_CACHE="$AIPERF_MMAP_CACHE_HOST_PATH" - SHARED_RESULTS="${SHARED_BASE}/results" - RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RUNNER_NAME}" - WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" - SRT_REPO_DIR="${WORK_DIR}/srt-slurm" - CONFIG_PATH="${CONFIG_FILE%%:*}" - LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" - CLUSTER_PROFILE="$SRT_SLURM_CLUSTER_CONFIG" - ADAPTER=infx.workflows.srt_slurm - - # Only shared, user-owned directories are prepared here. Pyxis manages - # container imports/lifetime; engines use the cache mounted by the profile. - mkdir -p "$WORK_DIR" "$SHARED_RESULTS" "$SHARED_AIPERF_CACHE" - setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" 0 - - export PATH="$HOME/.local/bin:$PATH" - cd "$SRT_REPO_DIR" - uv venv --python 3.12 - uv pip install -e . - source .venv/bin/activate - - PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" python -m "$ADAPTER" prepare \ - --recipe "$LOCAL_RECIPE" --profile "$CLUSTER_PROFILE" \ - --work-dir "$WORK_DIR" --workspace "$GITHUB_WORKSPACE" \ - --results-root "$SHARED_RESULTS" --aiperf-cache "$SHARED_AIPERF_CACHE" \ - --image-cache "${SHARED_BASE}/containers" - export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" - # Use upstream's normal setup with the prepared cluster configuration; - # providing it first also keeps setup non-interactive in Actions. - cp "$SRTSLURM_CONFIG" "$SRT_REPO_DIR/srtslurm.yaml" - make setup ARCH="$SRT_SLURM_COMPUTE_ARCH" - PREPARED_RECIPE="${WORK_DIR}/recipe.yaml" - - # Do not leak the CI host's bytecode-cache location into the containers. - # Recipes may still explicitly set their own container cache environment. - unset PYTHONPYCACHEPREFIX - apply_srt_recipe "$PREPARED_RECIPE" "$FRAMEWORK" -f "$PREPARED_RECIPE" \ - --tags "mi355x,inferencex,github-actions,${RUN_KEY}" --json \ - > "${WORK_DIR}/submission.json" - JOB_ID=$(jq -er '.slurm_job_id' "${WORK_DIR}/submission.json") - OUTPUT_DIR=$(jq -er '.output_dir' "${WORK_DIR}/submission.json") - echo "SRT_SLURM_JOB_ID=$JOB_ID" - - # srt-slurm handles queue/accounting transitions and propagates failures. - # Always collect available diagnostics, including on an unsuccessful job. - job_rc=0 - srtctl wait "$JOB_ID" --log-file "${OUTPUT_DIR}/logs/sweep_${JOB_ID}.log" || job_rc=$? - collect_rc=0 - PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" python -m "$ADAPTER" collect --submission "${WORK_DIR}/submission.json" \ - --workspace "$GITHUB_WORKSPACE" --results-root "$SHARED_RESULTS" || collect_rc=$? - [[ "$job_rc" -eq 0 ]] || exit "$job_rc" - [[ "$collect_rc" -eq 0 ]] || exit "$collect_rc" - printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" - echo "MI355X srt-slurm validation completed successfully" - exit 0 + source "$(dirname "${BASH_SOURCE[0]}")/srt_runtime.sh" || exit 1 + run_srt_recipe_job + exit $? fi if [[ "$IS_MULTINODE" == "true" ]]; then diff --git a/runners/runtime_settings.sh b/runners/runtime_settings.sh index 949f34fdbf..0641e23cf7 100644 --- a/runners/runtime_settings.sh +++ b/runners/runtime_settings.sh @@ -38,11 +38,21 @@ case "${RUNNER_NAME%%_*}" in export B300_HF_CACHE_HOST_DIR="$HOME/.cache/huggingface" export B300_HF_CACHE_CONTAINER_DIR=/hf_hub_cache ;; + mi300x-amd) + check_env_vars GITHUB_WORKSPACE HOME + export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" + export SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git + export SRT_SLURM_COMMIT=7a7ca652fad37026bbf476b3f4368cd34c73c1c6 + export SRT_SLURM_SHARED_BASE="$HOME/srt-slurm" + export SRT_SLURM_CLUSTER_CONFIG="$GITHUB_WORKSPACE/runners/srt-slurm/mi300x-amd.yaml" + export SRT_SLURM_COMPUTE_ARCH=x86_64 + export AIPERF_MMAP_CACHE_HOST_PATH="$HOME/srt-slurm/aiperf-cache" + ;; mi355x-amds) check_env_vars GITHUB_WORKSPACE export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" export SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git - export SRT_SLURM_COMMIT=c3a5cecd5d4513aa69444323cd6bce668dcbdc88 + export SRT_SLURM_COMMIT=7a7ca652fad37026bbf476b3f4368cd34c73c1c6 export SRT_SLURM_SHARED_BASE=/it-share/gharunners2/srt-slurm export SRT_SLURM_CLUSTER_CONFIG="$GITHUB_WORKSPACE/runners/srt-slurm/mi355x-amds.yaml" export SRT_SLURM_COMPUTE_ARCH=x86_64 diff --git a/runners/srt-slurm/mi300x-amd.yaml b/runners/srt-slurm/mi300x-amd.yaml new file mode 100644 index 0000000000..c42e9089ee --- /dev/null +++ b/runners/srt-slurm/mi300x-amd.yaml @@ -0,0 +1,19 @@ +cluster: mi300x-amd +default_partition: compute-0 +default_time_limit: "00:30:00" +output_dir: ${SRT_SLURM_SHARED_BASE}/outputs +gpus_per_node: 8 +visible_devices_env: ROCR_VISIBLE_DEVICES +default_gpu_exporter: null +network_interface: '' +use_gpus_per_node_directive: true +use_segment_sbatch_directive: false +use_exclusive_sbatch_directive: true +default_sbatch_directives: + cpus-per-task: "128" + mem: "0" +default_mounts: + /dev/kfd: /dev/kfd + /dev/dri: /dev/dri + /raid/inferencex/models/hub: /hf_hub_cache/hub +nginx_raise_ulimit: false diff --git a/runners/srt_runtime.sh b/runners/srt_runtime.sh new file mode 100644 index 0000000000..6398b76015 --- /dev/null +++ b/runners/srt_runtime.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Shared native srt-slurm execution; cluster paths and defaults come from callers. +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" || return 1 + +run_srt_recipe_job() ( + set -eo pipefail + check_env_vars CONFIG_FILE EVAL_ONLY IS_AGENTIC IS_MULTINODE KEEP_LOGS RUN_EVAL \ + GITHUB_WORKSPACE RESULT_FILENAME IMAGE MODEL FRAMEWORK \ + SRT_SLURM_REPOSITORY SRT_SLURM_COMMIT SRT_SLURM_SHARED_BASE \ + SRT_SLURM_CLUSTER_CONFIG SRT_SLURM_COMPUTE_ARCH AIPERF_MMAP_CACHE_HOST_PATH \ + GITHUB_RUN_ID GITHUB_RUN_ATTEMPT RUNNER_NAME HOME + + SHARED_BASE="$SRT_SLURM_SHARED_BASE" + SHARED_AIPERF_CACHE="$AIPERF_MMAP_CACHE_HOST_PATH" + SHARED_RESULTS="${SHARED_BASE}/results" + RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RUNNER_NAME}" + WORK_DIR="${GITHUB_WORKSPACE}/.srt-slurm-${RUN_KEY}" + SRT_REPO_DIR="${WORK_DIR}/srt-slurm" + CONFIG_PATH="${CONFIG_FILE%%:*}" + LOCAL_RECIPE="${GITHUB_WORKSPACE}/benchmarks/multi_node/srt-slurm-recipes/${CONFIG_PATH#recipes/}" + CLUSTER_PROFILE="$SRT_SLURM_CLUSTER_CONFIG" + ADAPTER=infx.workflows.srt_slurm + + # Only shared, user-owned directories are prepared here. Pyxis manages + # container imports/lifetime; engines use the cache mounted by the profile. + mkdir -p "$WORK_DIR" "$SHARED_RESULTS" "$SHARED_AIPERF_CACHE" + setup_srt_slurm "$SRT_REPO_DIR" "$FRAMEWORK" 0 + + export PATH="$HOME/.local/bin:$PATH" + cd "$SRT_REPO_DIR" + uv venv --python 3.12 + uv pip install -e . + source .venv/bin/activate + + PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" python -m "$ADAPTER" prepare \ + --recipe "$LOCAL_RECIPE" --profile "$CLUSTER_PROFILE" \ + --work-dir "$WORK_DIR" --workspace "$GITHUB_WORKSPACE" \ + --results-root "$SHARED_RESULTS" --aiperf-cache "$SHARED_AIPERF_CACHE" \ + --image-cache "${SHARED_BASE}/containers" + export SRTSLURM_CONFIG="${WORK_DIR}/srtslurm.yaml" + # Use upstream's normal setup with the prepared cluster configuration; + # providing it first also keeps setup non-interactive in Actions. + cp "$SRTSLURM_CONFIG" "$SRT_REPO_DIR/srtslurm.yaml" + make setup ARCH="$SRT_SLURM_COMPUTE_ARCH" + PREPARED_RECIPE="${WORK_DIR}/recipe.yaml" + + # Do not leak the CI host's bytecode-cache location into the containers. + # Recipes may still explicitly set their own container cache environment. + unset PYTHONPYCACHEPREFIX + apply_srt_recipe "$PREPARED_RECIPE" "$FRAMEWORK" -f "$PREPARED_RECIPE" \ + --tags "inferencex,github-actions,${RUN_KEY}" --json \ + > "${WORK_DIR}/submission.json" + JOB_ID=$(jq -er '.slurm_job_id' "${WORK_DIR}/submission.json") + OUTPUT_DIR=$(jq -er '.output_dir' "${WORK_DIR}/submission.json") + echo "SRT_SLURM_JOB_ID=$JOB_ID" + + # srt-slurm handles queue/accounting transitions and propagates failures. + # Always collect available diagnostics, including on an unsuccessful job. + job_rc=0 + srtctl wait "$JOB_ID" --log-file "${OUTPUT_DIR}/logs/sweep_${JOB_ID}.log" || job_rc=$? + collect_rc=0 + PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" python -m "$ADAPTER" collect --submission "${WORK_DIR}/submission.json" \ + --workspace "$GITHUB_WORKSPACE" --results-root "$SHARED_RESULTS" || collect_rc=$? + [[ "$job_rc" -eq 0 ]] || exit "$job_rc" + [[ "$collect_rc" -eq 0 ]] || exit "$collect_rc" + printf '%s\n' "$SRT_SLURM_COMMIT" > "$GITHUB_WORKSPACE/srt-slurm-producer-sha.txt" + echo "srt-slurm validation completed successfully" + exit 0 +) From 74e5c50889f56a7ca007aba579b571db680d8831 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 12:29:58 -0500 Subject: [PATCH 3/6] refactor: reuse shared Slurm monitoring for AMD ports --- runners/srt_runtime.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runners/srt_runtime.sh b/runners/srt_runtime.sh index 6398b76015..c5d8b5cb67 100644 --- a/runners/srt_runtime.sh +++ b/runners/srt_runtime.sh @@ -55,10 +55,10 @@ run_srt_recipe_job() ( OUTPUT_DIR=$(jq -er '.output_dir' "${WORK_DIR}/submission.json") echo "SRT_SLURM_JOB_ID=$JOB_ID" - # srt-slurm handles queue/accounting transitions and propagates failures. # Always collect available diagnostics, including on an unsuccessful job. job_rc=0 - srtctl wait "$JOB_ID" --log-file "${OUTPUT_DIR}/logs/sweep_${JOB_ID}.log" || job_rc=$? + stream_slurm_job_log "$JOB_ID" "${OUTPUT_DIR}/logs/sweep_${JOB_ID}.log" || job_rc=$? + verify_slurm_job_status "$JOB_ID" || job_rc=$? collect_rc=0 PYTHONPATH="$GITHUB_WORKSPACE${PYTHONPATH:+:$PYTHONPATH}" python -m "$ADAPTER" collect --submission "${WORK_DIR}/submission.json" \ --workspace "$GITHUB_WORKSPACE" --results-root "$SHARED_RESULTS" || collect_rc=$? From 68c37f979c554da506f0a5b0099ffabe54c45331 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 12:41:53 -0500 Subject: [PATCH 4/6] build: use upstream srt-slurm v2.22.1 for AMD ports --- perf-changelog.yaml | 6 ++++++ runners/runtime_settings.sh | 4 ---- runners/slurm_utils.sh | 10 ++-------- runners/srt_runtime.sh | 2 +- utils/srt-slurm | 2 +- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 02ec449f55..5485a7e7f5 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8612,3 +8612,9 @@ description: - 'Pin AMD runtime 7a7ca652fad37026bbf476b3f4368cd34c73c1c6 and share the native srt-slurm invocation between MI355X and MI300X. Add an explicit MI300X Qwen3-0.6B aggregate smoke matrix without changing production serving settings.' pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - '*' + description: + - 'Bump the NVIDIA srt-slurm submodule to v2.22.1 (3cbc5dd256af2bfd2fed09b724628c3f5456c85f) and use it for AMD ports instead of a separate fork pin. Reuse the shared Slurm log and completion helpers.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 diff --git a/runners/runtime_settings.sh b/runners/runtime_settings.sh index 0641e23cf7..e12901473b 100644 --- a/runners/runtime_settings.sh +++ b/runners/runtime_settings.sh @@ -41,8 +41,6 @@ case "${RUNNER_NAME%%_*}" in mi300x-amd) check_env_vars GITHUB_WORKSPACE HOME export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" - export SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git - export SRT_SLURM_COMMIT=7a7ca652fad37026bbf476b3f4368cd34c73c1c6 export SRT_SLURM_SHARED_BASE="$HOME/srt-slurm" export SRT_SLURM_CLUSTER_CONFIG="$GITHUB_WORKSPACE/runners/srt-slurm/mi300x-amd.yaml" export SRT_SLURM_COMPUTE_ARCH=x86_64 @@ -51,8 +49,6 @@ case "${RUNNER_NAME%%_*}" in mi355x-amds) check_env_vars GITHUB_WORKSPACE export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" - export SRT_SLURM_REPOSITORY=https://github.com/SemiAnalysisAI/srt-slurm.git - export SRT_SLURM_COMMIT=7a7ca652fad37026bbf476b3f4368cd34c73c1c6 export SRT_SLURM_SHARED_BASE=/it-share/gharunners2/srt-slurm export SRT_SLURM_CLUSTER_CONFIG="$GITHUB_WORKSPACE/runners/srt-slurm/mi355x-amds.yaml" export SRT_SLURM_COMPUTE_ARCH=x86_64 diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 349fa2e524..67a6eac780 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -57,14 +57,8 @@ PYENV # Custom benchmarks inherit exported workflow settings through sbatch/srun; # native recipe environment and benchmark.env retain their override priority. local source="$INFERENCEX_SLURM_UTILS_DIR/../utils/srt-slurm" - if [[ -n "${SRT_SLURM_REPOSITORY:-}" ]]; then - check_env_vars SRT_SLURM_COMMIT - git init "$destination" || return 1 - git -C "$destination" remote add origin "$SRT_SLURM_REPOSITORY" || return 1 - git -C "$destination" fetch --depth=1 origin "$SRT_SLURM_COMMIT" || return 1 - git -C "$destination" checkout --detach "$SRT_SLURM_COMMIT" || return 1 - elif [[ "$framework" == "tilert" ]]; then - # Fork exception until NVIDIA supports the TileRT backend and router. + if [[ "$framework" == "tilert" ]]; then + # Sole fork exception until NVIDIA supports the TileRT backend and router. SRT_SLURM_COMMIT=6bc3f306bdafa1edfb5dded2fcda8f1ccede1bde git init "$destination" || return 1 git -C "$destination" remote add origin https://github.com/SemiAnalysisAI/srt-slurm.git || return 1 diff --git a/runners/srt_runtime.sh b/runners/srt_runtime.sh index c5d8b5cb67..21120b4bd6 100644 --- a/runners/srt_runtime.sh +++ b/runners/srt_runtime.sh @@ -7,7 +7,7 @@ run_srt_recipe_job() ( set -eo pipefail check_env_vars CONFIG_FILE EVAL_ONLY IS_AGENTIC IS_MULTINODE KEEP_LOGS RUN_EVAL \ GITHUB_WORKSPACE RESULT_FILENAME IMAGE MODEL FRAMEWORK \ - SRT_SLURM_REPOSITORY SRT_SLURM_COMMIT SRT_SLURM_SHARED_BASE \ + SRT_SLURM_SHARED_BASE \ SRT_SLURM_CLUSTER_CONFIG SRT_SLURM_COMPUTE_ARCH AIPERF_MMAP_CACHE_HOST_PATH \ GITHUB_RUN_ID GITHUB_RUN_ATTEMPT RUNNER_NAME HOME diff --git a/utils/srt-slurm b/utils/srt-slurm index 984180e5b8..3cbc5dd256 160000 --- a/utils/srt-slurm +++ b/utils/srt-slurm @@ -1 +1 @@ -Subproject commit 984180e5b8755aef85e9995048b5a16cb5336bce +Subproject commit 3cbc5dd256af2bfd2fed09b724628c3f5456c85f From e8fae41a6d709328bb9d8185085c79465e50fd2b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 13:15:18 -0500 Subject: [PATCH 5/6] cleanup: remove AMD port smoke scaffolding and changelog entries --- .../qwen3-0.6b/mi300x/agg-fixed-seq.yaml | 94 --------------- .../qwen3-0.6b/mi355x/agg-fixed-seq.yaml | 97 --------------- .../mi355x/disagg-1p1d-fixed-seq.yaml | 111 ------------------ configs/amd-master.yaml | 55 --------- configs/smoke/amd-srt-mi300x.yaml | 24 ---- docs/configuration-procedures.md | 2 +- perf-changelog.yaml | 31 ----- runners/launch_mi300x-amd.sh | 6 - runners/runtime_settings.sh | 8 -- runners/srt-slurm/mi300x-amd.yaml | 19 --- 10 files changed, 1 insertion(+), 446 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml delete mode 100644 configs/smoke/amd-srt-mi300x.yaml delete mode 100644 runners/srt-slurm/mi300x-amd.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml deleted file mode 100644 index d4c5bfba0c..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Minimal MI300X aggregate validation. It exercises native SGLang Router and -# one aggregate SGLang worker; this is orchestration validation, not tuning. - -schema: 2 -name: "mi300x-sglang-qwen3-0.6b-agg-fixed-seq" - -model: - path: "hf:Qwen/Qwen3-0.6B" - container: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910" - precision: "fp16" - -identity: - model: - repo: "Qwen/Qwen3-0.6B" - container: - image: "lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910" - -slurm: - time_limit: "00:30:00" - -resources: - gpu_type: "mi300x" - gpus_per_node: 8 -frontend: - type: sglang-router - enable_multiple_frontends: false - args: - policy: round_robin - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 1 - - env: - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache/hub" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - PYTHONUNBUFFERED: "1" - args: - served-model-name: "Qwen/Qwen3-0.6B" - tensor-parallel-size: 1 - mem-fraction-static: 0.50 - context-length: 2048 - max-running-requests: 8 - disable-cuda-graph: true - enable-metrics: true - -srun_options: - container-writable: "" - container-remap-root: "" - mem: "0" - -health_check: - max_attempts: 180 - interval_seconds: 5 - -benchmark: - type: custom - command: | - set -eo pipefail - source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only - check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST - result_root="/results/${SLURM_JOB_ID}" - mkdir -p "${result_root}/fixed-seq" - trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT - for concurrency in ${CONC_LIST}; do - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai-chat \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/chat/completions \ - --model Qwen/Qwen3-0.6B \ - --tokenizer Qwen/Qwen3-0.6B \ - --dataset-name random \ - --random-input-len 128 \ - --random-output-len 32 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - --num-warmups "${concurrency}" \ - --num-prompts "$((concurrency * 4))" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_root}/fixed-seq" \ - --result-filename "qwen3-0.6b-agg-isl128-osl32-c${concurrency}.json"; - done - env: - HF_HOME: /hf_hub_cache - HF_HUB_CACHE: /hf_hub_cache/hub - HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml deleted file mode 100644 index 94cf4c3d11..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Minimal MI355X aggregate validation. It exercises native SGLang Router and -# one aggregate SGLang worker; this is orchestration validation, not tuning. - -schema: 2 -name: "mi355x-sglang-qwen3-0.6b-agg-fixed-seq" - -model: - path: "hf:Qwen/Qwen3-0.6B" - container: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" - precision: "fp16" - -identity: - model: - repo: "Qwen/Qwen3-0.6B" - container: - image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" - frameworks: - sglang: "0.5.17.dev20260809+g7120f3ee13" - sglang-router: "0.3.2" - -slurm: - time_limit: "00:30:00" - -resources: - gpu_type: "mi355x" - gpus_per_node: 1 -frontend: - type: sglang-router - enable_multiple_frontends: false - args: - policy: round_robin - -engine: sglang -roles: - agg: - nodes: 1 - workers: 1 - gpus: 1 - - env: - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache/hub" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - PYTHONUNBUFFERED: "1" - args: - served-model-name: "Qwen/Qwen3-0.6B" - tensor-parallel-size: 1 - mem-fraction-static: 0.50 - context-length: 2048 - max-running-requests: 8 - disable-cuda-graph: true - enable-metrics: true - -srun_options: - container-writable: "" - container-remap-root: "" - mem: "0" - -health_check: - max_attempts: 180 - interval_seconds: 5 - -benchmark: - type: custom - command: | - set -eo pipefail - source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only - check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST - result_root="/results/${SLURM_JOB_ID}" - mkdir -p "${result_root}/fixed-seq" - trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT - for concurrency in ${CONC_LIST}; do - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai-chat \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/chat/completions \ - --model Qwen/Qwen3-0.6B \ - --tokenizer Qwen/Qwen3-0.6B \ - --dataset-name random \ - --random-input-len 128 \ - --random-output-len 32 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - --num-warmups "${concurrency}" \ - --num-prompts "$((concurrency * 4))" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_root}/fixed-seq" \ - --result-filename "qwen3-0.6b-agg-isl128-osl32-c${concurrency}.json"; - done - env: - HF_HOME: /hf_hub_cache - HF_HUB_CACHE: /hf_hub_cache/hub - HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml deleted file mode 100644 index 524bc45fc3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# Minimal two-node MI355X P/D validation. SGLang Router provides the request -# plane and the bundled AMD MoRI transport moves KV between prefill and decode. - -schema: 2 -name: "mi355x-sglang-qwen3-0.6b-disagg-1p1d-fixed-seq" - -model: - path: "hf:Qwen/Qwen3-0.6B" - container: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" - precision: "fp16" - -identity: - model: - repo: "Qwen/Qwen3-0.6B" - container: - image: "lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809" - frameworks: - sglang: "0.5.17.dev20260809+g7120f3ee13" - sglang-router: "0.3.2" - amd-mori: "0.5.17.dev20260809+g7120f3ee13" - -slurm: - time_limit: "00:45:00" - -resources: - gpu_type: "mi355x" - gpus_per_node: 1 -frontend: - type: sglang-router - enable_multiple_frontends: false - args: - policy: round_robin - prefill-policy: round_robin - decode-policy: round_robin - -engine: sglang -roles: - prefill: - nodes: 1 - workers: 1 - gpus: 1 - env: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache/hub" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - PYTHONUNBUFFERED: "1" - IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" - MORI_RDMA_TC: "104" - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "1800" - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "1800" - args: &worker_config - served-model-name: "Qwen/Qwen3-0.6B" - tensor-parallel-size: 1 - disaggregation-transfer-backend: mori - mem-fraction-static: 0.50 - context-length: 2048 - max-running-requests: 8 - disable-cuda-graph: true - enable-metrics: true - decode: - nodes: 1 - workers: 1 - gpus: 1 - - env: *worker_environment - args: *worker_config - -srun_options: - container-writable: "" - container-remap-root: "" - mem: "0" - -health_check: - max_attempts: 240 - interval_seconds: 5 - -benchmark: - type: custom - command: | - set -eo pipefail - source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only - check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST - result_root="/results/${SLURM_JOB_ID}" - mkdir -p "${result_root}/fixed-seq" - trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT - for concurrency in ${CONC_LIST}; do - python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ - --backend openai-chat \ - --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ - --endpoint /v1/chat/completions \ - --model Qwen/Qwen3-0.6B \ - --tokenizer Qwen/Qwen3-0.6B \ - --dataset-name random \ - --random-input-len 128 \ - --random-output-len 32 \ - --random-range-ratio 1.0 \ - --random-num-workers 1 \ - --num-warmups "${concurrency}" \ - --num-prompts "$((concurrency * 4))" \ - --max-concurrency "${concurrency}" \ - --request-rate inf \ - --ignore-eos \ - --disable-tqdm \ - --save-result \ - --result-dir "${result_root}/fixed-seq" \ - --result-filename "qwen3-0.6b-disagg-isl128-osl32-c${concurrency}.json"; - done - env: - HF_HOME: /hf_hub_cache - HF_HUB_CACHE: /hf_hub_cache/hub - HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 28484ef8b6..d4f29da950 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -714,58 +714,3 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: search-space: # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } - -qwen3-0.6b-fp16-mi355x-sglang-srt-agg: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 - model: Qwen/Qwen3-0.6B - model-prefix: qwen3-0.6b - runner: cluster:mi355x-amds - precision: fp16 - framework: sglang - multinode: true - disagg: false - router: { name: sglang-router, version: "0.3.2" } - scenarios: - fixed-seq-len: - - isl: 128 - osl: 32 - search-space: - - conc-list: [1, 4] - num-nodes: 1 - worker: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/agg-fixed-seq.yaml" - -qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 - model: Qwen/Qwen3-0.6B - model-prefix: qwen3-0.6b - runner: cluster:mi355x-amds - precision: fp16 - framework: sglang-disagg - multinode: true - disagg: true - router: { name: sglang-router, version: "0.3.2" } - kv-p2p-transfer: mori - scenarios: - fixed-seq-len: - - isl: 128 - osl: 32 - search-space: - - conc-list: [1, 4] - prefill: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi355x/disagg-1p1d-fixed-seq.yaml" - decode: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false diff --git a/configs/smoke/amd-srt-mi300x.yaml b/configs/smoke/amd-srt-mi300x.yaml deleted file mode 100644 index 5d1aeaaa78..0000000000 --- a/configs/smoke/amd-srt-mi300x.yaml +++ /dev/null @@ -1,24 +0,0 @@ -qwen3-0.6b-fp16-mi300x-sglang-srt-agg-smoke: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi30x-20260910 - model: Qwen/Qwen3-0.6B - model-prefix: qwen3-0.6b - runner: cluster:mi300x-amd - precision: fp16 - framework: sglang - multinode: true - disagg: false - scenarios: - fixed-seq-len: - - isl: 128 - osl: 32 - search-space: - - conc-list: - - 4 - num-nodes: 1 - worker: - num-worker: 1 - tp: 1 - ep: 1 - dp-attn: false - additional-settings: - - CONFIG_FILE=recipes/sglang/qwen3-0.6b/mi300x/agg-fixed-seq.yaml diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 6477619e8e..a7667db823 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -25,7 +25,7 @@ Archive deprecated entries in [`configs/deprecated/amd-master.yaml`](../configs/ ## Dependency submodules -Git records the exact dependency commits. [`.gitmodules`](../.gitmodules) defines the repositories: AIPerf at `utils/aiperf`, NVIDIA srt-slurm at `utils/srt-slurm`. TileRT and the MI355X AMD port use pinned fork checkouts in `setup_srt_slurm()`, not separate submodules. The MI355X fork and revision are selected in `runners/runtime_settings.sh`; other runners keep their existing runtime selection. +Git records the exact dependency commits. [`.gitmodules`](../.gitmodules) defines the repositories: AIPerf at `utils/aiperf`, NVIDIA srt-slurm at `utils/srt-slurm`. TileRT is a documented manual fork checkout in `setup_srt_slurm()`, not a separate submodule. Initialize them before running benchmarks locally: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5485a7e7f5..86eb8dbb63 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8587,34 +8587,3 @@ - "将镜像从已失效的 nightly-eed1f3d0 重新固定到 vllm/vllm-openai-rocm:nightly-rocm100-3df4ae153eb385e27b52f26c81f8edb9e20b9984（与 MI355X 臂在 SemiAnalysisAI/InferenceX#3326 中所用相同 commit 的 ROCm 10.0 nightly 渠道构建）；该镜像包含 vllm-project/vllm#57491，将两处 is_cuda() 判定放宽为 is_cuda_alike()，使 engram_config 在 gfx942 上可解析；由于 cpu_offload 现通过 VLLM_PLE_CPU_OFFLOAD 默认开启，配方按 TP 显式设置该值" - "新增 --no-swa-bounded-replay：vllm-project/vllm#56227 在两个 pin 之间将 SWA bounded replay 默认开启，其依赖的 window clamp 在 ROCm sparse SWA 路径中缺失，曾使所有 gfx950 数据点以 HSA_STATUS_ERROR_MEMORY_FAULT 崩溃；gfx942 使用同一路径。prefix caching 保持开启" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3337 - -- config-keys: - - qwen3-0.6b-fp16-mi355x-sglang-srt-agg - - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg - - '*-mi355x-*-disagg*' - description: - - 'Refresh the shared AMD srt-slurm integration onto current InferenceX: v2 roles, native cluster profiles, golden - AgentX acceptance, and the synchronized AMD/MoRI/ATOM/runtime stack; retire the replaced manual launcher.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 - -- config-keys: - - '*-mi355x-*-disagg*' - - qwen3-0.6b-fp16-mi355x-sglang-srt-agg - - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg - description: - - 'Update the AMD srt-slurm runtime to c3a5cecd5d4513aa69444323cd6bce668dcbdc88: native connector resolution, allocator-owned MoRI-IO listener ports, and discovery-aware vLLM Router readiness. Preserve workload options and engine images.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 - -- config-keys: - - '*-mi355x-*-disagg*' - - qwen3-0.6b-fp16-mi355x-sglang-srt-agg - - qwen3-0.6b-fp16-mi355x-sglang-srt-disagg - description: - - 'Pin AMD runtime 7a7ca652fad37026bbf476b3f4368cd34c73c1c6 and share the native srt-slurm invocation between MI355X and MI300X. Add an explicit MI300X Qwen3-0.6B aggregate smoke matrix without changing production serving settings.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 - -- config-keys: - - '*' - description: - - 'Bump the NVIDIA srt-slurm submodule to v2.22.1 (3cbc5dd256af2bfd2fed09b724628c3f5456c85f) and use it for AMD ports instead of a separate fork pin. Reuse the shared Slurm log and completion helpers.' - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 diff --git a/runners/launch_mi300x-amd.sh b/runners/launch_mi300x-amd.sh index b7390b0dbd..6d88f743cd 100644 --- a/runners/launch_mi300x-amd.sh +++ b/runners/launch_mi300x-amd.sh @@ -4,12 +4,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/../benchmarks/benchmark_lib.sh" --validat check_env_vars IS_MULTINODE set -eo pipefail -if [[ -n "${CONFIG_FILE:-}" ]]; then - source "$(dirname "${BASH_SOURCE[0]}")/srt_runtime.sh" - run_srt_recipe_job - exit $? -fi - export HF_HUB_CACHE_MOUNT="/raid/inferencex/models/hub" export AIPERF_MMAP_CACHE_MOUNT="/raid/inferencex/aiperf-mmap-cache" export AIPERF_DATASET_MMAP_CACHE_DIR="/aiperf_mmap_cache" diff --git a/runners/runtime_settings.sh b/runners/runtime_settings.sh index e12901473b..c3a15e5bca 100644 --- a/runners/runtime_settings.sh +++ b/runners/runtime_settings.sh @@ -38,14 +38,6 @@ case "${RUNNER_NAME%%_*}" in export B300_HF_CACHE_HOST_DIR="$HOME/.cache/huggingface" export B300_HF_CACHE_CONTAINER_DIR=/hf_hub_cache ;; - mi300x-amd) - check_env_vars GITHUB_WORKSPACE HOME - export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" - export SRT_SLURM_SHARED_BASE="$HOME/srt-slurm" - export SRT_SLURM_CLUSTER_CONFIG="$GITHUB_WORKSPACE/runners/srt-slurm/mi300x-amd.yaml" - export SRT_SLURM_COMPUTE_ARCH=x86_64 - export AIPERF_MMAP_CACHE_HOST_PATH="$HOME/srt-slurm/aiperf-cache" - ;; mi355x-amds) check_env_vars GITHUB_WORKSPACE export BENCHMARK_LOGS_DIR="$GITHUB_WORKSPACE/benchmark_logs" diff --git a/runners/srt-slurm/mi300x-amd.yaml b/runners/srt-slurm/mi300x-amd.yaml deleted file mode 100644 index c42e9089ee..0000000000 --- a/runners/srt-slurm/mi300x-amd.yaml +++ /dev/null @@ -1,19 +0,0 @@ -cluster: mi300x-amd -default_partition: compute-0 -default_time_limit: "00:30:00" -output_dir: ${SRT_SLURM_SHARED_BASE}/outputs -gpus_per_node: 8 -visible_devices_env: ROCR_VISIBLE_DEVICES -default_gpu_exporter: null -network_interface: '' -use_gpus_per_node_directive: true -use_segment_sbatch_directive: false -use_exclusive_sbatch_directive: true -default_sbatch_directives: - cpus-per-task: "128" - mem: "0" -default_mounts: - /dev/kfd: /dev/kfd - /dev/dri: /dev/dri - /raid/inferencex/models/hub: /hf_hub_cache/hub -nginx_raise_ulimit: false From 91d8ca2bf28037e3dc11af97882b4787fa673896 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 23 Sep 2026 13:17:00 -0500 Subject: [PATCH 6/6] feat(amd): port Qwen3.5 FP8 disaggregation to srt-slurm --- .../disagg-1p1d-tp4p-tp8d-fixed-seq.yaml | 153 ++++++++++++++++++ configs/amd-master.yaml | 44 +++++ 2 files changed, 197 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml new file mode 100644 index 0000000000..f4759dbc9b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml @@ -0,0 +1,153 @@ +# Production-scale MI355X port of the existing Qwen3.5 FP8 1P1D recipe. +# Each role owns one full 8-GPU node; the native SGLang Router provides the +# request plane and AMD MoRI moves KV directly between the P/D workers. + +schema: 2 +name: "mi355x-sglang-qwen3.5-fp8-disagg-1p1d-tp4p-tp8d-fixed-seq" + +model: + path: "hf:Qwen/Qwen3.5-397B-A17B-FP8" + container: "lmsysorg/sglang:v0.5.16-rocm720-mi35x" + precision: "fp8" + +identity: + model: + repo: "Qwen/Qwen3.5-397B-A17B-FP8" + container: + image: "lmsysorg/sglang:v0.5.16-rocm720-mi35x" + frameworks: + sglang: "0.5.16" + sglang-router: "0.3.2" + +slurm: + time_limit: "08:00:00" + +# This topology owns every GPU on each MI355X node. Give SGLang the complete +# 128-core cpuset as well: its ROCm image enables per-GPU CPU affinity, which +# maps the eight TP ranks across the full dual-socket CPU topology. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 +frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: round_robin + prefill-policy: round_robin + decode-policy: round_robin + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 4 + env: &common_environment + HF_HOME: "/hf_hub_cache" + # Hugging Face stores hub snapshots under $HF_HOME/hub. Keep the explicit + # cache variables on that same path so srt-slurm's prefetch and every + # backend process resolve the identical, current snapshot. + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + IBDEVICES: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + GLOO_SOCKET_IFNAME: "eno0" + NCCL_SOCKET_IFNAME: "eno0" + MORI_RDMA_TC: "104" + MORI_IO_TC: "104" + MORI_IO_SQ_BACKOFF_TIMEOUT_US: "50000" + MORI_IO_QP_MAX_SEND_WR: "16384" + MORI_IO_QP_MAX_CQE: "32768" + MORI_IO_QP_MAX_SGE: "2" + MORI_SHMEM_MODE: "ISOLATION" + SGLANG_USE_AITER: "1" + AITER_LOG_LEVEL: "ERROR" + SGLANG_MORI_DISPATCH_DTYPE: "auto" + SGLANG_MORI_QP_PER_TRANSFER: "4" + SGLANG_MORI_NUM_WORKERS: "4" + SGLANG_DISAGGREGATION_NUM_PRE_ALLOCATE_REQS: "32" + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: "3600" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "3600" + SGLANG_HEALTH_CHECK_TIMEOUT: "600" + args: &common_config + served-model-name: "Qwen/Qwen3.5-397B-A17B-FP8" + tensor-parallel-size: 4 + disaggregation-transfer-backend: mori + disaggregation-ib-device: "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7" + kv-cache-dtype: "fp8_e4m3" + attention-backend: aiter + moe-dense-tp-size: 1 + load-balance-method: round_robin + watchdog-timeout: 3600 + decode-log-interval: 1000 + log-level: warning + mem-fraction-static: 0.80 + max-running-requests: 128 + chunked-prefill-size: 262144 + context-length: 16384 + disable-radix-cache: true + cuda-graph-bs: [1, 2, 3, 4, 5, 6, 7, 8] + enable-metrics: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: *common_environment + args: + <<: *common_config + mem-fraction-static: 0.85 + prefill-round-robin-balance: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + trap 'tar -C /logs -czf "'"${result_root}"'/runtime-logs.tar.gz" . 2>/dev/null || true' EXIT + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if ((num_prompts < 16)); then + num_prompts=16 + fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai-chat \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/chat/completions \ + --model Qwen/Qwen3.5-397B-A17B-FP8 \ + --tokenizer Qwen/Qwen3.5-397B-A17B-FP8 \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --random-num-workers 1 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --disable-tqdm \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "qwen3.5-fp8-disagg-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index d4f29da950..b5881c9a9d 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -714,3 +714,47 @@ dsv41flash-fp4-mi355x-vllm-agentic-dspark: search-space: # Follow upstream AMD Engram defaults; omit the CUDA-only config flag. - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32] } + +qwen3.5-fp8-mi355x-sglang-disagg: + image: lmsysorg/sglang:v0.5.16-rocm720-mi35x + model: Qwen/Qwen3.5-397B-A17B-FP8 + model-prefix: qwen3.5 + runner: cluster:mi355x-amds + precision: fp8 + framework: sglang-disagg + router: { name: sglang-router, version: "0.3.2" } + kv-p2p-transfer: mori + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # The srt-slurm recipe owns the complete c8-c128 sweep in one Slurm + # allocation so model initialization is paid once instead of per point. + # 1P+1D TP4P+TP8D/EP1 remains the current baseline: TP4 prefill saves + # four GPUs without changing the TP8 decode service. + # dp-attn intentionally false: with --enable-dp-attention + + # --moe-a2a-backend mori, sglang auto-promotes moe_ep_size=tp_size, + # but is_deepep_class_backend() excludes MoRI, so + # num_shared_slots stays at the global value (1) and the + # (num_experts - num_shared_slots) % moe_ep_size assertion in + # fused_moe_triton/layer.py fires for Qwen3.5 (512 routed + 1 shared). + - spec-decoding: "none" + conc-list: [8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/mi355x/disagg-1p1d-tp4p-tp8d-fixed-seq.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0"