diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh index 89cb78146..b487ce23b 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh @@ -2,20 +2,24 @@ set -eo pipefail set -x -# DeepSeek-V4-Pro FP4 on B300 with vLLM MTP (num_speculative_tokens=3). -# Throughput fixes synthetic acceptance to AL 2.49; EVAL_ONLY keeps real +# DeepSeek-V4-Pro-0813 FP4 on B300 with vLLM DSpark (num_speculative_tokens=6). +# Throughput fixes synthetic acceptance to AL 3.77; EVAL_ONLY keeps real # verification. Cudagraph capture sizes are in tokens (see the capture block). # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# TP8 and TP4 c8 are GPU-resident. TP4 c16, DEP4, and DEP8 use DRAM offload -# with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. +# TP points are GPU-resident. DEP8 uses DRAM offload with +# KV_OFFLOAD_BACKEND=vllm-simple or mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars DCP_SIZE EVAL_ONLY PCP_SIZE +check_env_vars DCP_SIZE EVAL_ONLY PCP_SIZE HF_HOME + +# Reuse the launcher's persistent HF mount for AIPerf's dataset mmap cache. +export AIPERF_DATASET_MMAP_CACHE_DIR="$HF_HOME/aiperf_mmap_cache" +mkdir -p "$AIPERF_DATASET_MMAP_CACHE_DIR" VLLM_CP_ARGS=() if [ "$DCP_SIZE" -gt 1 ]; then @@ -39,8 +43,8 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then exit 1 fi -# DEP8 (TP8 + DP-attention) gets a larger prefill token budget and lower -# GPU-memory headroom than DEP4. +# DEP8 (TP8 + DP-attention) pins KV cache bytes per concurrency tier and +# keeps extra GPU-memory headroom. IS_DEP8=false if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then IS_DEP8=true @@ -85,7 +89,7 @@ export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" # Match the environment used by v4pro-b300.yaml. export VLLM_USE_V2_MODEL_RUNNER=1 -export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_ENGINE_READY_TIMEOUT_S=7200 export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 export VLLM_DSV4_MEGA_FP8_COMBINE=1 export NCCL_NVLS_ENABLE=1 @@ -110,20 +114,14 @@ case "$KV_OFFLOAD_BACKEND" in CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) # Identical prefixes must hash to identical block keys across DP ranks. export PYTHONHASHSEED=42 - # DEP keeps eager offload for cross-rank block-hash stability; plain TP - # uses lazy offload. - SIMPLE_LAZY_OFFLOAD=false - if [ "$DP_ATTENTION" != "true" ]; then - SIMPLE_LAZY_OFFLOAD=true - fi OFFLOAD_CONFIG=$(cat < "$BATCH_SCRIPT" BATCH_ARGS=(--parsable --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" - --nodes=1 --ntasks=1 --gres="gpu:$GPU_COUNT" --exclusive --mem=0 + --nodes=1 --ntasks=1 --cpus-per-task=192 --gres="gpu:$GPU_COUNT" --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --job-name="$RUNNER_NAME" --export=ALL --chdir="$GITHUB_WORKSPACE" --output="$BATCH_LOG") if [[ -n "${SALLOC_EXCLUDE:-}" ]]; then @@ -340,7 +340,7 @@ else # weights. Only the root holding MODEL_PATH is mounted -- mounting both roots # makes pyxis fail whenever the unused one is absent on the node. MODEL_BASENAME="${MODEL##*/}" - if [[ "$MODEL_BASENAME" == "DeepSeek-V4-Pro-0813" ]]; then + if [[ "$MODEL_BASENAME" == "DeepSeek-V4-Pro-0813" && "$FRAMEWORK" != "vllm" ]]; then MODEL_MOUNT_DIR="$SHARED_MODEL_ROOT" elif [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then MODEL_MOUNT_DIR="$MODEL_ROOT" @@ -389,10 +389,12 @@ else check_env_vars GPU_COUNT + export SLURM_CPUS_PER_TASK=192 SALLOC_ARGS=( --partition="$SLURM_PARTITION" --account="$SLURM_ACCOUNT" -N 1 + --cpus-per-task="$SLURM_CPUS_PER_TASK" --gres="gpu:$GPU_COUNT" --exclusive --mem=0