diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml new file mode 100644 index 0000000000..5152b438d4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml @@ -0,0 +1,141 @@ +# DeepSeek-V4-Pro 1P1D ATOM P/D serving on MI355X. Native AToMesh +# routes prefill/decode traffic while ATOM transfers KV state through +# Mooncake's default RDMA transport. The custom benchmark preserves the +# legacy 8k/1k search and request-length distribution. + +schema: 2 +name: "mi355x-atom-dsv4-disagg-1p1d-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "rocm/atom-dev:nightly_202609031453" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/atom-dev:nightly_202609031453" + frameworks: + atomesh: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 +frontend: + type: atomesh + enable_multiple_frontends: false + args: + policy: random + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + +environment: &runtime_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + # Use the official image's ROCm/Mooncake library paths (Python 3.12). + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + SAFETENSORS_FAST_GPU: "1" + VLLM_LOG_LEVEL: "WARNING" + ATOM_LOG_LEVEL: "WARNING" + AITER_LOG_LEVEL: "WARNING" + LOG_LEVEL: "WARNING" + LOGLEVEL: "WARNING" + +engine: + type: atom + connector: mooncake +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: &worker_environment + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + args: + # Upstream workflow MAX_MODEL_LEN = ISL + OSL + 256. + max-model-len: 9472 + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: *worker_environment + args: + max-model-len: 9472 + cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 128 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + +# The legacy exclusive allocation exposed the full 128-core, unlimited-memory +# node; do not inherit the shared small-model profile's 32-core / 128G limits. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + exclusive: "" + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST RANDOM_RANGE_RATIO + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio "${RANDOM_RANGE_RATIO}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-1p1d-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + <<: *runtime_environment diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml new file mode 100644 index 0000000000..2dc2fe1055 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml @@ -0,0 +1,145 @@ +# DeepSeek-V4-Pro 2P1D ATOM P/D serving on MI355X. Each logical endpoint +# occupies one TP8 node; the two prefill endpoints and decode endpoint use +# data-parallel attention/TBO as in the legacy production configuration. + +schema: 2 +name: "mi355x-atom-dsv4-disagg-2p1d-dpa-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "rocm/atom-dev:nightly_202609031453" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/atom-dev:nightly_202609031453" + frameworks: + atomesh: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 +frontend: + type: atomesh + enable_multiple_frontends: false + args: + policy: random + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + +environment: &runtime_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + # Use the official image's ROCm/Mooncake library paths (Python 3.12). + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + SAFETENSORS_FAST_GPU: "1" + VLLM_LOG_LEVEL: "WARNING" + ATOM_LOG_LEVEL: "WARNING" + AITER_LOG_LEVEL: "WARNING" + LOG_LEVEL: "WARNING" + LOGLEVEL: "WARNING" + +engine: + type: atom + connector: mooncake +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + env: &worker_environment + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + GPU_MAX_HW_QUEUES: "5" + ATOM_CPU_AFFINITY: "1" + args: + # Upstream workflow MAX_MODEL_LEN = ISL + OSL + 256. + max-model-len: 9472 + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: *worker_environment + args: + max-model-len: 9472 + cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 2048 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + +# The legacy exclusive allocation exposed the full 128-core, unlimited-memory +# node; do not inherit the shared small-model profile's 32-core / 128G limits. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + exclusive: "" + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -eo pipefail + source /infmax-workspace/benchmarks/benchmark_lib.sh --validation-only + check_env_vars SLURM_JOB_ID SRT_FRONTEND_HOST SRT_FRONTEND_PORT CONC_LIST RANDOM_RANGE_RATIO + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in ${CONC_LIST}; do + num_prompts=$((concurrency * 10)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio "${RANDOM_RANGE_RATIO}" \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-2p1d-dpa-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + <<: *runtime_environment diff --git a/configs/deprecated/amd-master.yaml b/configs/deprecated/amd-master.yaml index faa8e91e3d..1e1f601e1b 100644 --- a/configs/deprecated/amd-master.yaml +++ b/configs/deprecated/amd-master.yaml @@ -38,6 +38,58 @@ dsr1-fp4-mi355x-sglang-mtp: search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } +dsv4-fp4-mi355x-atom-disagg: + image: rocm/atom-dev:nightly_202609031453 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # 2P1D DPA+TP8 + - conc-list: [256, 512, 768, 1024, 2048] + prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + # 1P1D TP8 + - conc-list: [4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + # 1P1D TP8 + dsr1-fp4-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: amd/DeepSeek-R1-0528-MXFP4-Preview diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5485a7e7f5..e3c99a4121 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8618,3 +8618,16 @@ description: - 'Bump the NVIDIA srt-slurm submodule to v2.22.1 (3cbc5dd256af2bfd2fed09b724628c3f5456c85f) and use it for AMD ports instead of a separate fork pin. Reuse the shared Slurm log and completion helpers.' pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - 'Restack this port directly on #2542 and migrate its recipes to srt-slurm v2, preserving the existing port workload + settings. Keep this upstream-deprecated workload in the archived master config, outside active sweeps.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - 'Validate the archived ATOM/AToMesh port with synchronized AMD srt-slurm stack 7a7ca652fad37026bbf476b3f4368cd34c73c1c6; preserve serving images and workload settings.' + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627