diff --git a/benchmarks/single_node/agentic/README.md b/benchmarks/single_node/agentic/README.md deleted file mode 100644 index fa35ea2b3b..0000000000 --- a/benchmarks/single_node/agentic/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Agentic single-node benchmarks - -> **Legacy.** Single-node AgentX configs now run as declarative srt-slurm -> recipes (`benchmarks/single_node/srt-slurm-recipes////agentic.yaml`, -> see #3428). The only bash scripts left here are the ones a master config still -> reaches through a search-space entry without an `srt-recipe:`; delete each one -> when its config is ported. `install_h1{0,2}0_block32_configs.py` and -> `kernel_configs/` also serve the multi-node DeepSeek-V4.1-Flash H100/H200 recipes. - -**MVP / experimental.** Nothing in this directory is an official InferenceX -benchmark. Results are not published on https://inferencex.com and are not -intended to be cited. - -These launchers exist to develop and validate the agentic-coding scenario -type before it is promoted to first-class status. The scripts themselves -are best-effort and mainly serve as a reference implementation of how the -plumbing (env vars, scenario routing, result paths) should work. Specific -models and configs may be broken at any given time. Multi-node support in -particular is not yet first-class. - -## DRAM KV offload memory policy - -Agentic scenarios use `kv-offloading` for the resource tier and -`kv-offload-backend` for the backend implementation. `kv-offloading` is -currently either `none` or `dram`. When it is `dram`, the backend must be set: - -```yaml -- dram-utilization: 0.80 - search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 32] } - - { tp: 8, kv-offloading: none, conc-list: [16, 32] } -``` - -`kv-offload-backend.name` is required when offloading is enabled. Its -`version` is optional: omit it for framework-native implementations without an -independent release, and include it for separately versioned packages such as -LMCache or Mooncake. - -Agentic matrix generation uses a 3600-second default duration. Reusable -workflow callers can still override the `duration` input explicitly. - -Agentic master configs must use an exact `cluster:` runner label so every -search-space point lands on the same hardware fleet. Machine-level host memory -is declared once in the `configs/runners.yaml` `hardware` entry matching -that runner label: - -```yaml -hardware: - cluster:b300-dsxe: - available-cpu-dram-mib: 3977095 - gpus-per-node: 8 -``` - -The matrix generator combines the master config utilization with runner -hardware metadata and emits the aggregate budget as -`floor(min(available MiB, 2,861,022) * 1,048,576 * utilization * tp / gpus-per-node / 1,000,000,000)`. -The `2,861,022 MiB` cap is the 3 TB decimal DRAM limit. For example, TP4 in -an eight-GPU B300 search at 80% utilization receives 1,199 GB while TP8 -receives 2,399 GB. - -Benchmark scripts must consume `TOTAL_CPU_DRAM_GB` rather than replace it -with model-specific constants. Backends with per-rank or per-pool settings must -divide this aggregate budget accordingly. DSv4 SGLang is the exception because -it exposes only `--hicache-ratio`. Its empirically measured ratios are capped -in the model launchers to remain below the generated byte budget. diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_atom_mtp.sh deleted file mode 100755 index 57cd56bafa..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# DeepSeek-V4.1-Flash AgentX on MI355X: TP2 / TP4 with five-token DSpark. -# https://github.com/ROCm/ATOM/blob/53b11c9a665e786798785acbedfdfd4da3fb87c4/recipes/DeepSeek-V4.1-Flash-Agentic.md -# The launcher routes both mtp and draft_model through the _mtp suffix. -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EP_SIZE DP_ATTENTION EVAL_ONLY PORT - -if [[ "$TP" != 2 && "$TP" != 4 ]] || [[ "$EP_SIZE" != 1 || "$DP_ATTENTION" != false ]]; then - echo "ERROR: this recipe requires TP=2 or TP=4, EP_SIZE=1, and DP_ATTENTION=false" >&2 - exit 1 -fi -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -# Resume incomplete downloads and keep the tokenizer aligned with the server. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -export AGENTIC_TOKENIZER_PATH="$MODEL_PATH" - -# Respect the GPU allocation chosen by Slurm, including nonzero GPU pairs. -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi -export OMP_NUM_THREADS=4 -export ATOM_NUMA_BIND=0 -export ATOM_DISABLE_MMAP=true -export AITER_LOG_LEVEL=WARNING - -resolve_trace_source -install_agentic_deps - -# Upstream's one-hour AgentX profile uses five warmup requests per lane. -# build_replay_cmd retains the workflow's explicit duration / fast-mode inputs. -export AIPERF_WARMUP_REQUESTS_PER_LANE=5 -export AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT=300 -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="atom:" - -wait_for_amd_gpu_clean -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -SERVER_PID="" -cleanup_atom_server() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "ATOM server" 60 - exit "$exit_code" -} -trap cleanup_atom_server EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -# At c32, capture every size through 32; all other points use the sparse list. -CAPTURE_SIZES='[1,2,3,4,5,6,7,8,16,32,48,64,128]' -if [[ "$CONC" -eq 32 ]]; then - CAPTURE_SIZES='[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,48,64,128]' -fi - -# golden_al_distribution/dsv41flash_dspark.yaml: thinking_on, K5 -> AL 3.51. -# Throughput follows the upstream recipe; accuracy eval verifies real drafts. -SPEC_ARGS=(--method dspark --num-speculative-tokens 5) -if [[ "$EVAL_ONLY" != true ]]; then - SPEC_ARGS+=(--spec-decode-acceptance-length 3.51) -fi - -ATOM_CMD=( - python3 -u -m atom.entrypoints.openai_server - --model "$MODEL_PATH" --served-model-name "$MODEL" --trust-remote-code - --host 0.0.0.0 --server-port "$PORT" - --tensor-parallel-size "$TP" - --kv_cache_dtype bf16 --index-cache-dtype fp8 - --gpu-memory-utilization 0.9 --max-num-seqs 128 - --max-num-batched-tokens 16384 --attn-prefill-chunk-size 16384 - --enable_prefix_caching --block-size 16 - --state-checkpoint-interval-tokens 8192 - --level 3 --cudagraph-mode FULL --cudagraph-capture-sizes "$CAPTURE_SIZES" - "${SPEC_ARGS[@]}" - --tool-call-parser dsml_v41 -) -write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" -"${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [[ "$EVAL_ONLY" == true ]]; then - run_eval --port "$PORT" -else - # AgentX traces contain fully formed chat payloads; preserve native handling. - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_atom_mtp.sh deleted file mode 100644 index cea659c059..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Agentic trace replay benchmark for GLM5.2 FP4 on MI355X using ATOM MTP -# -# Required env vars: -# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, -# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" - -if [[ -v SLURM_JOB_ID ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# ROCR/HIP visibility under slurm cgroups. -if [[ -v ROCR_VISIBLE_DEVICES ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -# DCP is enabled for large-concurrency points (C16+) via dcp-size in -# configs/amd-master.yaml; default 1 keeps the small-concurrency TP-only path. -DCP_SIZE="${DCP_SIZE:-1}" - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps - -# Require the ATOM Prometheus stream in every official result. -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="atom:" - -wait_for_amd_gpu_clean - -SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -LMCACHE_PIDS=() -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "ATOM server" 60 - local i - for i in "${!LMCACHE_PIDS[@]}"; do - stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" - done - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -OFFLOAD_ARGS=() - -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - lmcache) - require_agentic_kv_offload_backend lmcache - - export PYTHONHASHSEED=0 - export LMCACHE_LOCAL_CPU=True - export LMCACHE_MAX_LOCAL_CPU_SIZE="$TOTAL_CPU_DRAM_GB" - export LMCACHE_CHUNK_SIZE=256 - export OFFLOAD_MIN_LOAD_TOKENS=8192 - export LMCACHE_NUMA_MODE=auto - - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"lmcache_offload\",\"kv_role\":\"offload\"}" - ) - ;; - *) - echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2 - exit 1 - ;; -esac - -echo "Starting atom server..." -export PYTHONNOUSERSITE=1 - -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export AITER_USE_FLYDSL_MOE_SORTING=1 - -if (( DCP_SIZE > 1 )); then - # TP+DCP large-concurrency path: [1,2,4,8] then 12..(2*CONC) step 4. - CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8' - for ((size = 12; size <= CONC * 2; size += 4)); do - CUDAGRAPH_CAPTURE_SIZES+=",${size}" - done - CUDAGRAPH_CAPTURE_SIZES+=']' -else - case "$CONC" in - 1) CUDAGRAPH_CAPTURE_SIZES='[1,2]' ;; - 2) CUDAGRAPH_CAPTURE_SIZES='[1,2,4]' ;; - 4) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8]' ;; - 8) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8,12,16]' ;; - 10) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8,12,16,20]' ;; - 12) CUDAGRAPH_CAPTURE_SIZES='[1,2,4,8,12,16,20,24]' ;; - *) - echo "Unsupported CONC=$CONC for TP-only path" >&2 - exit 2 - ;; - esac -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") #TP -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -gt 1 ]; then #DP+EP - PARALLEL_ARGS=(--tensor-parallel-size "$TP" --enable-dp-attention --enable-expert-parallel) - else - PARALLEL_ARGS=(--tensor-parallel-size "$TP" --enable-dp-attention ) - fi -fi -if (( DCP_SIZE > 1 )); then - PARALLEL_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") -fi - -# Draft depth per concurrency; forced acceptance length is the golden value for -# that depth from -# https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/glm5.2_mtp.yaml -# (glm-5.2-fp8, thinking_on): K5 -> 3.61, K4 -> 3.33, K3 -> 2.99. -if (( DCP_SIZE > 1 )); then - if (( CONC >= 48 )); then - NUM_SPEC_TOKENS=3; SIMULATE_ACC_LEN=2.99 - else - NUM_SPEC_TOKENS=4; SIMULATE_ACC_LEN=3.33 - fi -else - case "$CONC" in - 1|2|4|8) NUM_SPEC_TOKENS=5; SIMULATE_ACC_LEN=3.61 ;; - 10|12) NUM_SPEC_TOKENS=4; SIMULATE_ACC_LEN=3.33 ;; - *) - echo "Unsupported CONC=$CONC for TP-only MTP path" >&2 - exit 2 - ;; - esac -fi -SPEC_ARGS=( - --method mtp - --num-speculative-tokens "$NUM_SPEC_TOKENS" -) -if [ "${EVAL_ONLY}" != "true" ]; then - SPEC_ARGS+=(--spec-decode-acceptance-length "$SIMULATE_ACC_LEN") -fi -echo "DCP_SIZE=$DCP_SIZE NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS SIMULATE_ACC_LEN=$SIMULATE_ACC_LEN" - -ATOM_CMD=( - python -m atom.entrypoints.openai_server - --model "$MODEL_PATH" - --host 0.0.0.0 - --server-port "$PORT" - "${PARALLEL_ARGS[@]}" - --gpu-memory-utilization 0.95 - --enable_prefix_caching - # "model.layers.78.*" excludes the MTP head from online quantization, - # keeping layer 78 in native BF16: it ships unquantized in amd/GLM-5.2-MXFP4, - # and the expert excludes only reach layers 0-77, so without this the whole - # MTP block would be online-quantized to ptpc_fp8 while the target's experts - # stay MXFP4. - --online_quant_config '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*.mlp.gate","model.layers.[0-9].mlp.*expert*","model.layers.[1-6][0-9].mlp.*expert*","model.layers.7[0-7].mlp.*expert*","model.layers.78.*"]}' - --max-num-seqs "$((2 * CONC))" - --cudagraph-capture-sizes "$CUDAGRAPH_CAPTURE_SIZES" - --max-num-batched-tokens 16384 - --kv_cache_dtype fp8 - --index_cache_dtype fp4 - --block-size 64 - "${SPEC_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" -"${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/install_h100_block32_configs.py b/benchmarks/single_node/agentic/install_h100_block32_configs.py deleted file mode 100644 index 88c3fc97f6..0000000000 --- a/benchmarks/single_node/agentic/install_h100_block32_configs.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Install the H100 V4.1 tiling configs into the imported SGLang kernel package.""" # noqa: INP001 -- Standalone recipe executable. - -import hashlib -import json -import shutil -import sys -from pathlib import Path - -from sglang.kernels.ops.quantization import fp8_kernel - - -def main() -> None: - source = Path(sys.argv[1]) - artifacts = Path(sys.argv[2]) / "fp8_kernel_configs" - device = fp8_kernel.get_device_name().replace(" ", "_") - if device != "NVIDIA_H100_80GB_HBM3": - raise RuntimeError(f"H100 tiling configs cannot be installed on {device}") - destination = Path(fp8_kernel.__file__).resolve().parent / "configs" - destination.mkdir(exist_ok=True) - artifacts.mkdir(parents=True, exist_ok=True) - for n, k in [ - (5120, 1024), - (5120, 288), - (576, 5120), - (1792, 5120), - (512, 5120), - (1280, 5120), - ]: - name = ( - f"N={n},K={k},device_name={device},dtype=fp8_w8a8,block_shape=[32, 32].json" - ) - path = source / name - expected = { - int(m): config for m, config in json.loads(path.read_text()).items() - } - shutil.copyfile(path, destination / name) - shutil.copyfile(path, artifacts / name) - fp8_kernel.get_w8a8_block_fp8_configs.cache_clear() - actual = fp8_kernel.get_w8a8_block_fp8_configs(n, k, 32, 32) - if actual != expected: - raise RuntimeError(f"SGLang did not resolve the installed config: {name}") - print( - f"Installed {destination / name}: sha256={hashlib.sha256(path.read_bytes()).hexdigest()}" - ) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/single_node/agentic/install_h200_block32_configs.py b/benchmarks/single_node/agentic/install_h200_block32_configs.py deleted file mode 100644 index 3d34a865e2..0000000000 --- a/benchmarks/single_node/agentic/install_h200_block32_configs.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Install the H200 V4.1 tiling configs into the imported SGLang kernel package.""" # noqa: INP001 -- Standalone recipe executable. - -import hashlib -import json -import shutil -import sys -from pathlib import Path - -from sglang.kernels.ops.quantization import fp8_kernel - - -def main() -> None: - source = Path(sys.argv[1]) - artifacts = Path(sys.argv[2]) / "fp8_kernel_configs" - tp = int(sys.argv[3]) - if tp not in (4, 8): - raise ValueError(f"H200 tiling configs support TP4 or TP8, got TP{tp}") - device = fp8_kernel.get_device_name().replace(" ", "_") - if device != "NVIDIA_H200": - raise RuntimeError(f"H200 tiling configs cannot be installed on {device}") - destination = Path(fp8_kernel.__file__).resolve().parent / "configs" - destination.mkdir(exist_ok=True) - artifacts.mkdir(parents=True, exist_ok=True) - shapes = [(1792, 5120)] - if tp == 8: - shapes += [(5120, 1024), (5120, 288), (576, 5120)] - for n, k in shapes: - name = f"N={n},K={k},device_name={device},dtype=fp8_w8a8,block_shape=[32, 32].json" - path = source / name - expected = {int(m): config for m, config in json.loads(path.read_text()).items()} - shutil.copyfile(path, destination / name) - shutil.copyfile(path, artifacts / name) - fp8_kernel.get_w8a8_block_fp8_configs.cache_clear() - actual = fp8_kernel.get_w8a8_block_fp8_configs(n, k, 32, 32) - if actual != expected: - raise RuntimeError(f"SGLang did not resolve the installed config: {name}") - print( - f"Installed {destination / name}: sha256={hashlib.sha256(path.read_bytes()).hexdigest()}" - ) - - -if __name__ == "__main__": - main() diff --git a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=1280,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=1280,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 971110754f..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=1280,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "0": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - }, - "1": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": false - }, - "5": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": false - }, - "10": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": true - }, - "20": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 8, - "SWAP_AB": true - }, - "40": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 8, - "SWAP_AB": true - }, - "80": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": true - }, - "160": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 8, - "SWAP_AB": true - }, - "190": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 8, - "SWAP_AB": true - }, - "191": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - }, - "320": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - }, - "321": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=1792,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=1792,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 986ff1953b..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=1792,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": false, - "num_stages": 3, - "num_warps": 4 - }, - "6": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "24": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 4, - "num_warps": 4 - }, - "96": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 4, - "num_warps": 4 - }, - "128": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "num_stages": 3, - "num_warps": 4 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=512,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=512,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 3c4f7326b0..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=512,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "0": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - }, - "1": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": false - }, - "5": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": false - }, - "10": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": false - }, - "20": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": true - }, - "40": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 16, - "SWAP_AB": true - }, - "80": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 8, - "SWAP_AB": true - }, - "160": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 4, - "SWAP_AB": true - }, - "320": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - "SPLIT_K": 4, - "SWAP_AB": true - }, - "321": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=5120,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=5120,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index fece264886..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=5120,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 5 - }, - "4": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 4 - }, - "16": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 4 - }, - "32": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 5 - }, - "64": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 5 - }, - "256": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 5 - }, - "1024": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 5 - }, - "4096": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 4 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=5120,K=288,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=5120,K=288,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index ccf51c6e0a..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=5120,K=288,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 2 - }, - "4": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 2 - }, - "16": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 8, - "num_stages": 2 - }, - "32": { - "BLOCK_SIZE_M": 32, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 5 - }, - "64": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - }, - "256": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 5 - }, - "1024": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 2 - }, - "4096": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 3 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=576,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=576,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 4e45c66e6d..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/N=576,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": false, - "num_stages": 3, - "num_warps": 4 - }, - "4": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "6": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "16": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "24": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "32": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "64": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 4, - "num_warps": 4 - }, - "96": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 4, - "num_warps": 4 - }, - "128": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 16, - "num_stages": 5, - "num_warps": 4 - }, - "193": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 32, - "num_stages": 4, - "num_warps": 4 - }, - "256": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 32, - "num_stages": 4, - "num_warps": 4 - }, - "1024": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "GROUP_SIZE_M": 1, - "num_stages": 4, - "num_warps": 4 - }, - "4096": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "GROUP_SIZE_M": 1, - "num_stages": 5, - "num_warps": 4 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/provenance.json b/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/provenance.json deleted file mode 100644 index 9c4fff096c..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h100_dsv41_block32/provenance.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "sglang_commit": "0f6761b54facebb47f2068f87ecccd8f14da3a0e", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5", - "hardware": "NVIDIA H100 80GB HBM3", - "tuner": "benchmark/kernels/quantization/tuning_block_wise_kernel.py", - "search_change": "Use BLOCK_SIZE_K=[32] for checkpoint block32 scales; upstream CUDA candidates [64,128] otherwise filter to an empty search.", - "tuning_slurm_job": 18452, - "selected_audit_slurm_job": 18457, - "resolver_install_slurm_job": 18458, - "validation": "39 cases: FP32 dequantized reference rtol=0.02/atol=0.05; exact CUDA graph/eager equality; alternating timing order. BF16 rounding may differ from default tiling.", - "selection": "N5120/K288 batch64 restored to upstream default after timing regression.", - "precision": "Stock FP8 inputs and scales, BF16 output; no quantization or model changes.", - "split_k_audit": { - "slurm_job": 18479, - "cases": 80, - "mechanism": "Pinned upstream Hopper SPLIT_K and SWAP_AB launch configuration, FP32 partial reduction, unchanged FP8 inputs/scales and BF16 output", - "shapes": [ - "N1792/K5120", - "N576/K5120" - ], - "boundary_validation": "Slurm18480:56/56 FP32-reference and exact CUDA-graph replay checks passed, batch boundariesM1..1024; changed cases all faster in alternating-order microtimings", - "large_batch_policy": "Retain original fallback selection for M129..8192 using explicit boundary anchors" - }, - "draft_projection_audit": { - "device": "NVIDIA H100 80GB HBM3", - "image": "nightly-dev-cu13-20260922-582389ce", - "search": { - "job": 18492, - "cases": 144, - "status": "COMPLETED0:0" - }, - "boundary": { - "job": 18493, - "cases": 206, - "status": "COMPLETED0:0" - }, - "trimmed_boundary": { - "job": 18494, - "cases": 20, - "status": "COMPLETED0:0" - }, - "reference": "FP32 dequant reference rtol.02/atol.05; exact eager/graph replay; no quantization or kernel code changes", - "ranges": { - "N512": "M1..320 tuned; M0 and M>=321 stock", - "N1280": "M1..190 tuned; M0 and M>=191 stock" - }, - "retained_speedup_range": [ - 1.1861693666195017, - 6.770688638312714 - ], - "baseline_l2_max": 0.0016977186314761639, - "candidate_l2_max": 0.0016970767173916101, - "hashes": { - "N=1280,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json": "635f684b6ed8b538709249baa5c474bf8583f952bcee330e01fd0f9e0ce714a3", - "N=512,K=5120,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[32, 32].json": "193fc012550dce39bb4de0a4bde28eed5cee79f2900ed95841fe18ccb6d8c187" - }, - "serving_qualification": { - "candidate_run": 35683668032, - "baseline_run": 35683669703, - "protocol": "Matched canonical C1, 3600 seconds, normal warmup; zero profiled errors and valid power in both runs", - "candidate_total_tps_per_gpu": 1239.57314, - "baseline_total_tps_per_gpu": 958.51774, - "candidate_p90_interactivity": 90.89516, - "baseline_p90_interactivity": 60.93631, - "full_gsm8k_run": 35683671358, - "full_gsm8k_examples": 1319, - "full_gsm8k_strict_accuracy": 0.9696739954510993, - "final_sweep": "Required after integration" - } - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 83620a6b10..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 4, - "num_warps": 4 - }, - "2": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "num_stages": 3, - "num_warps": 4 - }, - "6": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "24": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "96": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "128": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "num_stages": 3, - "num_warps": 4 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=5120,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=5120,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 850f61947e..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=5120,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 3 - }, - "4": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 4 - }, - "16": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 5 - }, - "32": { - "BLOCK_SIZE_M": 32, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 4 - }, - "64": { - "BLOCK_SIZE_M": 32, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 3 - }, - "256": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 5 - }, - "1024": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - }, - "4096": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 4 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=5120,K=288,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=5120,K=288,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 94b8a3fccf..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=5120,K=288,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_M": 32, - "BLOCK_SIZE_N": 64, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 8, - "num_stages": 2 - }, - "4": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 1, - "num_warps": 4, - "num_stages": 4 - }, - "16": { - "BLOCK_SIZE_M": 32, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 2 - }, - "32": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 32, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3 - }, - "64": { - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 16, - "num_warps": 4, - "num_stages": 4 - }, - "256": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 3 - }, - "1024": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 4 - }, - "4096": { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "BLOCK_SIZE_K": 32, - "GROUP_SIZE_M": 64, - "num_warps": 4, - "num_stages": 4 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=576,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=576,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json deleted file mode 100644 index 4359ec7f93..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/N=576,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "1": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": false, - "num_stages": 3, - "num_warps": 4 - }, - "4": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": false, - "num_stages": 3, - "num_warps": 4 - }, - "6": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": false, - "num_stages": 3, - "num_warps": 4 - }, - "16": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "24": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "32": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 16, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "64": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "96": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 32, - "GROUP_SIZE_M": 32, - "SPLIT_K": 8, - "SWAP_AB": true, - "num_stages": 3, - "num_warps": 4 - }, - "128": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 16, - "num_stages": 5, - "num_warps": 4 - }, - "193": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 64, - "num_stages": 3, - "num_warps": 4 - }, - "256": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 16, - "BLOCK_SIZE_N": 64, - "GROUP_SIZE_M": 64, - "num_stages": 3, - "num_warps": 4 - }, - "1024": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "GROUP_SIZE_M": 1, - "num_stages": 5, - "num_warps": 4 - }, - "4096": { - "BLOCK_SIZE_K": 32, - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": 128, - "GROUP_SIZE_M": 1, - "num_stages": 5, - "num_warps": 4 - } -} diff --git a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/provenance.json b/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/provenance.json deleted file mode 100644 index 074329f645..0000000000 --- a/benchmarks/single_node/agentic/kernel_configs/h200_dsv41_block32/provenance.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "sglang_commit": "0f6761b54facebb47f2068f87ecccd8f14da3a0e", - "image": "lmsysorg/sglang:nightly-dev-cu13-20260921-0f6761b5", - "hardware": "NVIDIA H200", - "tuner": "benchmark/kernels/quantization/tuning_block_wise_kernel.py", - "search_change": "Use BLOCK_SIZE_K=[32] for checkpoint block32 scales; upstream CUDA candidates [64,128] otherwise filter to an empty search.", - "tuning_slurm_job": 88009, - "selected_audit_and_resolver_slurm_job": 88030, - "validation": "39 cases against FP32 dequantized reference at rtol=0.02/atol=0.05; exact CUDA graph/eager equality; alternating timing order. BF16 rounding may differ from default tiling.", - "selection": "N5120/K288 batch32 restored to upstream default after timing regression.", - "precision": "Stock FP8 inputs and scales, BF16 output; no quantization or model changes.", - "splitk_tuning_slurm_job": 88079, - "splitk_selected_boundary_validation_slurm_job": 88084, - "splitk_scope": "Supported upstream SPLIT_K/SWAP_AB launch settings for N1792K5120 and N576K5120 at small verification batches; unchanged prior config selections for all M129..8192.", - "splitk_validation": "72/72 exploratory cases and56/56 selected boundary cases passed FP32reference rtol=.02/atol=.05 plus exact eager/graph equality; alternating timing order, selected changed cases improved1.37\u20136.22x." -} diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh deleted file mode 100644 index 764e40e461..0000000000 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Kimi-K3 MXFP4 on MI355X / MI350X (gfx950) with ATOM DSpark. Companion to -# kimik3_fp4_mi355x_mtp.sh (same checkpoint under vLLM). -# TP8 only: the 1.561 TB checkpoint is ~195 GB/GPU on 288 GB parts. -# apply_k3_container_patches.sh is not sourced: it targets a patched vLLM container. -# -# Serving bands, one fresh server per concurrency point: -# interactive (1, 4) DCP1, DSpark 7, no LMCache -# mid (14, 16) DCP8, DSpark 3, LMCache 128 GB/rank -# throughput (48, 56, 72) DCP8, no draft, LMCache 128 or 192 GB/rank -# -# Required env vars: -# MODEL, MODEL_PATH, TP, DCP_SIZE, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, -# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION - -DRAFT_MODEL="Inferact/Kimi-K3-DSpark" - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars DCP_SIZE - -echo "MODEL=$MODEL TP=$TP DCP_SIZE=${DCP_SIZE} CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" - -if [[ -v SLURM_JOB_ID ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# The 0907 image bakes in HTTP(S)_PROXY=http://127.0.0.1:3128 from its build -# host; nothing listens there on a benchmark node, so every outbound request -# (uv bootstrap, aiperf install, trace fetch, hf download) would hit a dead proxy. -unset HTTP_PROXY HTTPS_PROXY http_proxy https_proxy -export NO_PROXY="${NO_PROXY:-localhost,127.0.0.1,::1}" -export no_proxy="$NO_PROXY" - -if [ "$TP" -ne 8 ]; then - echo "Error: Kimi-K3 MXFP4 is a 1.56 TB checkpoint and only fits at TP=8 on" >&2 - echo " 288 GB gfx950 parts (~195 GB/GPU). Got TP=$TP." >&2 - exit 1 -fi - -if [[ -v ROCR_VISIBLE_DEVICES ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -wait_for_amd_gpu_clean - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps - -# Require the ATOM Prometheus stream in every official result. -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="atom:" - -# Long agentic turns against a 1M context are prefill-bound on the server. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 - -# Gate at <=1% (~2.9 GB) rather than the default 10%: ATOM sizes the KV pool -# from mem_get_info() at startup, so prior-job VRAM residue is folded into -# non_torch and moves the KV pool between otherwise identical reruns. -wait_for_amd_gpu_clean 1 - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "ATOM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -# Per-concurrency knobs. Interactive band: GPU-resident, no DCP, deepest -# published draft. Mid band: decode is KV-bandwidth-bound over 100k+ contexts, -# so DCP8 (from dcp-size in configs/amd-master.yaml), LMCache DRAM tier, and -# ReplaySSM rebuilding KDA state from the checkpoint ring. Throughput band: -# past the knee the draft forward no longer pays for itself, so no draft. -MAX_NUM_BATCHED_TOKENS=8192 -GPU_MEM_UTIL=0.90 -AITER_REUSE_IDENTICAL_COMM_GROUPS=0 -case "$CONC" in - 1|4) - MAX_NUM_SEQS=32 - ATOM_ENABLE_REPLAYSSM=0 - NUM_SPEC_TOKENS=7 - SPEC_DECODE_AL=3.84 - ;; - 14|16) - MAX_NUM_SEQS=32 - ATOM_ENABLE_REPLAYSSM=1 - NUM_SPEC_TOKENS=3 - SPEC_DECODE_AL=3.00 - ;; - 48|56|72) - MAX_NUM_SEQS=$((2 * CONC)) - ATOM_ENABLE_REPLAYSSM=0 - NUM_SPEC_TOKENS=0 - SPEC_DECODE_AL=0 - # Only the two widest points reuse identical AITER communicator groups. - if [ "$CONC" -ge 56 ]; then - AITER_REUSE_IDENTICAL_COMM_GROUPS=1 - fi - ;; - *) - echo "Unsupported CONC=$CONC" >&2 - exit 2 - ;; -esac -export ATOM_ENABLE_REPLAYSSM -export AITER_REUSE_IDENTICAL_COMM_GROUPS - -# Full CUDA graphs over [1 .. window * (1 + draft tokens)]: the verify step -# submits one row per draft token on top of the accepted token, so capturing -# only up to the window would send every speculative decode down the eager path. -CUDAGRAPH_MAX_NUM_SEQS="$((2 * CONC))" -GRAPH_MAX=$((CUDAGRAPH_MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS))) -CUDAGRAPH_CAPTURE_SIZES="[$(seq -s, 1 "$GRAPH_MAX")]" -echo "CUDAGRAPH_MAX_NUM_SEQS=$CUDAGRAPH_MAX_NUM_SEQS GRAPH_MAX=$GRAPH_MAX" - -# The paged KV rides the LMCache CPU tier from concurrency 8 up; KDA recurrent -# state is rebuilt by ReplaySSM from the in-GPU checkpoint ring, so the whole -# per-rank budget goes to the paged KV. -OFFLOAD_ARGS=() - -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - lmcache) - require_agentic_kv_offload_backend lmcache - - # TOTAL_CPU_DRAM_GB is the aggregate budget; LMCACHE_MAX_LOCAL_CPU_SIZE - # is per rank, so divide by TP (agentic README). Handing a rank the - # whole aggregate never finishes pinning and hangs the launch. - export PYTHONHASHSEED=0 - export LMCACHE_LOCAL_CPU=True - export LMCACHE_MAX_LOCAL_CPU_SIZE="$((TOTAL_CPU_DRAM_GB / TP))" - # DCP-locked: the offload hash block is block-size(128) x dcp(8) = 1024, - # so the KV grid and the state-checkpoint grid coincide and the joint - # load aims both legs at one boundary. 512 or 2048 misaligns it. - export LMCACHE_CHUNK_SIZE=1024 - # K3 is a hybrid; without this gate the paged KV never reaches the CPU - # tier, which reads as a tier that is on and idle. - export OFFLOAD_KV_FOR_HYBRID=1 - # Pin the eight ranks to the two sockets explicitly; a pinned pool on the - # wrong socket pays a cross-socket hop on every offload read. - export LMCACHE_NUMA_MODE=auto - export ATOM_NUMA_BIND=1 - export ATOM_NUMA_NODE=0,0,0,0,1,1,1,1 - export ATOM_AUTO_NUMA_BIND=0 - # Statistics only; the submitted numbers were measured with it on. - export OFFLOAD_PROFILE=1 - # The GPU staging buffer defaults to 2 chunks (8 MiB) and one K3 state - # entry is 54.78 MiB; a buffer too small for one entry makes the transfer - # path decline to build, which reads like a tier that is on and idle. - export OFFLOAD_GPU_STAGING_CHUNKS=32 - - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"lmcache_offload\",\"kv_role\":\"offload\"}" - ) - ;; - *) - echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2 - exit 1 - ;; -esac - -echo "Starting atom server..." -export PYTHONNOUSERSITE=1 - -# Without it the aiter kernel logs flood the server log for the whole replay. -export AITER_LOG_LEVEL="WARNING" -export AITER_SITUV2_A4W4=1 -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export AITER_FLYDSL_STAGE2_FP8=1 -# Anchor-only state checkpointing: the demand rung is 47% of checkpoint writes -# but reads back 2.8% of the time, against 85.2% for a prompt-end anchor, so it -# costs more in evictions than its reuse is worth on these traces. -export ATOM_STATE_CHECKPOINT_DEMAND=0 -export ATOM_GDN_SSM_DTYPE=fp16 - -# https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml -# 7 draft tokens -> AL 3.84 -# 3 draft tokens -> AL 3.00 -# Concurrency 32 and up serve without a draft: past the throughput knee it no -# longer pays for itself against the resident batch. -SPEC_ARGS=() -if [ "$NUM_SPEC_TOKENS" -gt 0 ]; then - # Stage the draft into the HF cache first: an uncached repo id makes every - # rank pull the same 7 GB at once, silently (log stops after "Loading - # drafter model...") and at ~0.7 MB/s on one cluster. Shared-cache - # downloads can hit transient stale handles, hence the retries. - for attempt in 1 2 3 4 5; do - hf download "$DRAFT_MODEL" && break - if [ "$attempt" = 5 ]; then - echo "hf download of $DRAFT_MODEL failed after $attempt attempts" >&2 - exit 1 - fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 - done - - SPEC_ARGS=( - --method dspark - --draft-model "$DRAFT_MODEL" - --num-speculative-tokens "$NUM_SPEC_TOKENS" - ) - if [ "${EVAL_ONLY}" != "true" ]; then - SPEC_ARGS+=(--spec-decode-acceptance-length "$SPEC_DECODE_AL") - fi -fi -echo "SPEC_DECODE_AL=$SPEC_DECODE_AL NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS" - -ATOM_CMD=( - python -m atom.entrypoints.openai_server - --model "$MODEL_PATH" - # The AgentX client asks for $MODEL on the wire (benchmark_lib.sh passes - # --model ${SERVED_MODEL_NAME:-$MODEL}); a mismatch 404s at warmup. - --served-model-name "$MODEL" - --host 0.0.0.0 - --server-port "$PORT" - --trust-remote-code - # ATOM reads the tool-call wire format off the chat template at startup - # ("auto", the default), and this image already resolves it correctly -- - # the server log says `Tool-call format: kimi_k3 (from the chat template)`. - # Pinning it is for the day the template stops rendering a tools payload: - # auto then falls back to delivering tool calls as plain text, which the - # vendor verifier reads as a model that cannot call tools. An explicit name - # fails closed instead, and the log line changes to `(from - # --tool-call-parser)` so which path ran is visible. - # - # This is ATOM's own flag. It has no --enable-auto-tool-choice, and - # --reasoning-parser belongs to the mesh router, not this server; argparse - # here is strict, so either one would fail the launch. - --tool-call-parser kimi_k3 - --tensor-parallel-size "$TP" - --decode-context-parallel-size "${DCP_SIZE}" - --kv_cache_dtype fp8 - --block-size 128 - --max-num-seqs "$MAX_NUM_SEQS" - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" - --gpu-memory-utilization "$GPU_MEM_UTIL" - --enable_prefix_caching - # -1 is ladder off, checkpointing on: the prompt-end anchor still places a - # checkpoint, the fixed-interval grid does not. ATOM's default 8192 would - # change STATE pool occupancy and prefix reuse in the linear-attention layers. - --state-checkpoint-interval-tokens -1 - --level 3 - --cudagraph-mode FULL - --cudagraph-capture-sizes "$CUDAGRAPH_CAPTURE_SIZES" - # ptpc_fp8 is a GLOBAL online-quant target, and model_loader/loader.py runs - # the streamer for speculative draft loads too, so without the trailing - # entries it re-quantises the BF16 draft checkpoint. The draft tree carries - # no root prefix -- its decoder layers are `layers.{0..4}` where the - # target's are `model.layers.{i}` -- and _matches_exclude globs match the - # whole name, so a pattern anchored on `layers.` cannot reach the target. - # `context_proj` sits outside `layers.` and needs its own entry: it is the - # aux-fusion linear, 35840x7168, larger than a whole draft decoder layer. - --online_quant_config '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","*self_attn.[qkv]_conv1d*","*block_sparse_moe.experts*","*block_sparse_moe.routed_expert_*","*vision_tower*","*mm_projector*","layers.*.self_attn.fused_qkv_a_proj","layers.*.self_attn.q_b_proj","layers.*.self_attn.kv_b_proj","layers.*.self_attn.o_proj","layers.*.mlp.gate_up_proj","layers.*.mlp.down_proj","context_proj"]}' - "${SPEC_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" -"${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh deleted file mode 100644 index 8b7d287945..0000000000 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_mtp.sh +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Kimi-K3 MXFP4 on MI355X / MI350X (gfx950) with vLLM DSpark. Server flags -# follow the upstream AMD recipe (https://recipes.vllm.ai/moonshotai/Kimi-K3). -# TP8 only: the 1.561 TB checkpoint is ~195 GB/GPU on 288 GB parts, and -# upstream strategy_min_gpus agrees, so there is no DP-attention arm. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE - -source "$(dirname "$0")/../../benchmark_lib.sh" - -wait_for_amd_gpu_clean - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE -check_env_vars DCP_SIZE EVAL_ONLY - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [ "$TP" -ne 8 ]; then - echo "Error: Kimi-K3 MXFP4 is a 1.56 TB checkpoint and only fits at TP=8 on" >&2 - echo " 288 GB gfx950 parts (~195 GB/GPU). Got TP=$TP." >&2 - exit 1 -fi - -# ROCR/HIP visibility for vLLM 0.14+ -if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps - -# Env from the upstream AMD recipe. -export VLLM_ROCM_AITER_MLA_ASM_PADDING=asm -export VLLM_ROCM_USE_AITER=1 -export SAFETENSORS_FAST_GPU=1 -export VLLM_ROCM_USE_AITER_MOE_SITUV2_A8W4=1 -export AITER_SITUV2_A8W4=1 -export AITER_BF16_FP8_MOE_BOUND=0 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 - -# MEC FW <177 has an RCCL memory reclaim issue. -mec_version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') -if [[ "$mec_version" == "" || ${mec_version:-0} -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -# 2.8T of weights off a shared/NFS mount takes far longer than the default. -export VLLM_ENGINE_READY_TIMEOUT_S="7200" - -# Long agentic turns against a 1M context are prefill-bound on the server. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -LMCACHE_PID="" - -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - stop_background_process_tree "$LMCACHE_PID" "LMCache server" - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -# TOTAL_CPU_DRAM_GB is the aggregate host-DRAM budget from the matrix generator -# (capped at the 3 TB agentic limit) and must be consumed as given, never -# replaced with a model-specific constant (benchmarks/single_node/agentic/README.md). -OFFLOAD_ARGS=() - -if agentic_kv_offload_enabled; then -case "${KV_OFFLOAD_BACKEND:-}" in - vllm-simple) - require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TP )) - # Identical prefixes must hash to identical block keys across ranks. - export PYTHONHASHSEED=42 - SIMPLE_LAZY_OFFLOAD="false" - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":$CPU_BYTES_PER_RANK,\"lazy_offload\":$SIMPLE_LAZY_OFFLOAD}}" - ) - echo "SimpleCPUOffloadConnector: ${CPU_BYTES_PER_RANK} B/rank x ${TP} ranks, lazy_offload=$SIMPLE_LAZY_OFFLOAD" - ;; - lmcache) - require_agentic_kv_offload_backend "$KV_OFFLOAD_BACKEND" - - LMCACHE_VERSION=0.5.5.dev114+rocm7.2 - LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-rocm" - - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-rocm-7-0==14.1.1" \ - "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" - - # LMCache 0.5.5 eagerly imports the Mooncake backend, whose native .so - # needs libglog, libjsoncpp, libibverbs, librdmacm and libnuma; the vLLM - # ROCm image ships none of them. - LMCACHE_NATIVE_LIBS=(libglog.so.0 libjsoncpp.so.25 libibverbs.so.1 librdmacm.so.1 libnuma.so.1) - for lib in "${LMCACHE_NATIVE_LIBS[@]}"; do - if ! ldconfig -p | grep -q "$lib"; then - apt-get update - apt-get install -y \ - libgoogle-glog0v5 libjsoncpp25 libibverbs1 librdmacm1 libnuma1 - break - fi - done - python3 -c \ - "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ - >/dev/null - - # One MP server per node (docs.lmcache.ai/recipes/kimi_k3.html). The chunk - # must be a multiple of every KV group's tokens_per_block: the hybrid - # KDA/MLA layout registers attention groups at 1536 and a KDA state group - # at 3072, so 3072 is the minimum, and --separate-object-groups is - # required for the multi-group layout. - LMCACHE_PORT=6555 - LMCACHE_HTTP_PORT=8090 - LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" - - LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" - - # DCP shards decode KV across TP ranks: one GPU worker per rank and the - # larger 12288 chunk; a non-DCP arm uses one worker and the 3072 minimum. - if [ "${DCP_SIZE}" -gt 1 ]; then - LMCACHE_MAX_GPU_WORKERS=8 - LMCACHE_CHUNK_SIZE=12288 - else - LMCACHE_MAX_GPU_WORKERS=1 - LMCACHE_CHUNK_SIZE=3072 - fi - - LMCACHE_CMD=( - lmcache server - --host 127.0.0.1 - --port "$LMCACHE_PORT" - --http-host 127.0.0.1 - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb 10 - --chunk-size "$LMCACHE_CHUNK_SIZE" - --separate-object-groups - --enable-extra-logging - --extra-logging-interval 30 - --max-cpu-workers 8 - --max-gpu-workers "$LMCACHE_MAX_GPU_WORKERS" - --eviction-policy LRU - --supported-transfer-mode lmcache_driven - --shm-name "" - ) - append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - wait_for_ready \ - --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck" \ - --log "$LMCACHE_LOG" \ - --pid "$LMCACHE_PID" \ - --sleep-interval 1 \ - --timeout 600 - - # 100k-330k-token agentic prefixes make single retrieves large; use the - # same MQ timeout headroom as the MiniMax-M3 arm. - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected vllm-simple or lmcache)" >&2 - exit 1 - ;; -esac -fi - -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -case "$CONC" in - 1) - SYNTHETIC_ACCEPT_LEN=3.84 - SPEC_NUM_TOKENS=7 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=16384 - ;; - # k descends with concurrency; each k is paired with its own AL. - 4) - SYNTHETIC_ACCEPT_LEN=3.62 - SPEC_NUM_TOKENS=5 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 - ;; - 8|10) - SYNTHETIC_ACCEPT_LEN=3.36 - SPEC_NUM_TOKENS=4 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 - ;; - 12|14) - SYNTHETIC_ACCEPT_LEN=3.00 - SPEC_NUM_TOKENS=3 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 - ;; - 44|48|52) - SPEC_NUM_TOKENS=0 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 - ;; - *) - SPEC_NUM_TOKENS=0 - GPU_MEM_UTIL=0.9 - MAX_NUM_BATCHED_TOKENS=8192 - ;; -esac - -# ---- DSpark draft ------------------------------------------------------------ -# Published checkpoint, unmodified. vLLM #55966 makes ROCM_AITER_MLA accept its -# non-causal dflash_config, so the local causal rewrite is no longer needed. -DSPARK_DRAFT_PATH="${DSPARK_DRAFT_PATH:-Inferact/Kimi-K3-DSpark}" - -# ---- CUDA graph mode --------------------------------------------------------- -# FULL_AND_PIECEWISE needs breakable graphs (no @support_torch_compile on K3 AMD). -# Breakable costs KV pool, scaling with capture size 2*CONC*(1+k): -11.5% at conc 4 -# (capture 48), -1/3 at conc 48 (capture 96, deadlocks at 86% usage). Enabled only -# for conc 1 and 4, the two cells measured under it. -case "$CONC" in - 1|4) - CUDAGRAPH_MODE="${CUDAGRAPH_MODE:-FULL_AND_PIECEWISE}" - export VLLM_USE_BREAKABLE_CUDAGRAPH=1 - ;; - *) - CUDAGRAPH_MODE="${CUDAGRAPH_MODE:-FULL}" - export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - ;; -esac - -SPEC_ARGS=() -if [ "$SPEC_NUM_TOKENS" -gt 0 ]; then -if [ "${EVAL_ONLY}" = "true" ]; then - SPEC_ARGS=( - --speculative-config - "{\"model\":\"$DSPARK_DRAFT_PATH\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"ROCM_AITER_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\": \"block\"}" - ) -else - SPEC_ARGS=( - --speculative-config - "{\"model\":\"$DSPARK_DRAFT_PATH\",\"num_speculative_tokens\":$SPEC_NUM_TOKENS,\"method\":\"dspark\",\"attention_backend\":\"ROCM_AITER_MLA\",\"kv_cache_dtype\":\"fp8\",\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" - ) - fi -fi - -MAX_NUM_SEQS=$((2 * CONC)) -MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * (1 + SPEC_NUM_TOKENS))) -CUDAGRAPH_CAPTURE_SIZES="$(seq -s, 2 "$MAX_CUDAGRAPH_CAPTURE_SIZE")" -COMPILATION_CONFIG_ARGS=(--compilation-config "{\"mode\":3,\"cudagraph_mode\":\"$CUDAGRAPH_MODE\",\"max_cudagraph_capture_size\":$MAX_CUDAGRAPH_CAPTURE_SIZE,\"custom_ops\":[\"+fused_rms_norm_gated\"],\"cudagraph_capture_sizes\":[$CUDAGRAPH_CAPTURE_SIZES]}") - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 -export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS="1200" - - -# DCP shards decode KV across the TP ranks, so it must divide TP. -if [ $((TP % DCP_SIZE)) -ne 0 ]; then - echo "Error: TP='$TP' must be divisible by DCP_SIZE='$DCP_SIZE'" >&2 - exit 1 -fi -CP_ARGS=() -ATTN_BE_ARGS=() -if [ "$DCP_SIZE" -gt 1 ]; then - CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) - ATTN_BE_ARGS+=(--attention-backend ROCM_AITER_MLA) -fi -export VLLM_USE_DIRECT_DCP_A2A=0 -export VLLM_USE_DIRECT_DCP_Q_GATHER=0 -export VLLM_USE_DIRECT_DCP_KV_GATHER=0 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - --moe-backend auto - --tensor-parallel-size "$TP" - "${EP_ARGS[@]}" - --load-format fastsafetensors - --gpu-memory-utilization "$GPU_MEM_UTIL" - --language-model-only - --max-num-seqs "$MAX_NUM_SEQS" - --enable-auto-tool-choice - --tool-call-parser kimi_k3 - --reasoning-parser kimi_k3 - --max-model-len 1048576 - --enable-prefix-caching - --kv-cache-dtype "fp8" - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" - --attention-config '{"mla_prefill_backend":"ROCM_AITER_FA"}' - "${ATTN_BE_ARGS[@]}" - "${COMPILATION_CONFIG_ARGS[@]}" - "${SPEC_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" - "${CP_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh deleted file mode 100644 index 1d2fbb285f..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# MiniMax-M3 MXFP4 on MI355X / MI350X (gfx950) with ATOM EAGLE3. Companion to -# minimaxm3_fp4_mi355x_mtp.sh (same checkpoint under vLLM). TP2/TP4 follow -# the official ATOM MXFP4 recipe; TP8 is accepted for larger-memory variants. -# -# Required env vars: -# MODEL, MODEL_PATH, TP, DCP_SIZE, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, -# TOTAL_CPU_DRAM_GB, RESULT_DIR, RESULT_FILENAME, DURATION, EP_SIZE, DP_ATTENTION, -# EVAL_ONLY, ENABLE_PREFIX_CACHING, AITER_LOG_LEVEL -# Eval-only runs also require EVAL_FRAMEWORK and, for lm-eval, EVAL_TASKS_DIR. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL MODEL_PATH TP DCP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR RESULT_FILENAME DURATION EP_SIZE DP_ATTENTION EVAL_ONLY ENABLE_PREFIX_CACHING AITER_LOG_LEVEL -if [[ "$KV_OFFLOADING" != "none" ]]; then - check_env_vars KV_OFFLOAD_BACKEND -fi -if [[ "$EVAL_ONLY" == "true" ]]; then - check_env_vars EVAL_FRAMEWORK - if [[ "$EVAL_FRAMEWORK" == "lm-eval" || "$EVAL_FRAMEWORK" == "lm_eval" ]]; then - check_env_vars EVAL_TASKS_DIR - fi -fi - -echo "MODEL=$MODEL TP=$TP DCP_SIZE=$DCP_SIZE CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" - -if [[ -n "${SLURM_JOB_ID+x}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [ "$TP" -ne 2 ] && [ "$TP" -ne 4 ] && [ "$TP" -ne 8 ]; then - echo "Error: MiniMax-M3 MXFP4 supports TP2, TP4, or TP8 on 288 GB gfx950 parts." >&2 - exit 1 -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ "$MODEL_PATH" == "$MODEL" ]]; then - hf download "$MODEL" -elif [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -fi - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -hf download "$DRAFT_MODEL" - -wait_for_amd_gpu_clean - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps -# ATOM's server runs from the image's system venv, not the AIPerf venv from -# install_agentic_deps; MiniMax's tokenizer fallback needs these there. -ATOM_RUNTIME_DEPS=/tmp/inferencex-atom-runtime-deps -/opt/venv/bin/python -m pip install --quiet --target "$ATOM_RUNTIME_DEPS" --no-deps sentencepiece tiktoken - -# Require the ATOM Prometheus stream in every official result. -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="atom:" - -# Long agentic turns against a 1M context are prefill-bound on the server. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 - -wait_for_amd_gpu_clean - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "ATOM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -# Official MiniMax-M3 ATOM launch settings. -MAX_NUM_SEQS=$((2 * CONC)) -MAX_NUM_BATCHED_TOKENS=32768 -GPU_MEM_UTIL=0.95 - -# One place per concurrency. A band that is not listed exits; a band that is -# listed but leaves a feature off simply does not get it -- both failures are -# visible, unlike enabling a feature on a point nobody measured. -NUM_SPEC_TOKENS=3 -SPEC_DECODE_AL=2.78 -INDEXER_CP=0 -OFFLOAD_TIER="" -case "$CONC" in - 1|2|4|5|8|10|12|14|16) ;; - 15|24) INDEXER_CP=1 ;; - 20) INDEXER_CP=1; OFFLOAD_TIER=cpu256 ;; - 25|30) OFFLOAD_TIER=cpu256 ;; - 28|32) INDEXER_CP=1 ;; - 40|48) INDEXER_CP=1; OFFLOAD_TIER=cpu256 ;; - *) echo "Unsupported CONC=$CONC" >&2; exit 2 ;; -esac - -# Sized for the running batch, which is 22-41% of CONC on this workload, not for -# CONC itself. ModelRunner trims this to min(2*CONC, 8192). -CUDAGRAPH_CAPTURE_SIZES="[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,24,26,28,30,32,34,36,40,48,56,64]" - -OFFLOAD_ARGS=() - -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - lmcache) - require_agentic_kv_offload_backend lmcache - - export PYTHONHASHSEED=0 - export LMCACHE_LOCAL_CPU=True - - # GPUs 0-3 are on NUMA node 0, 4-7 on node 1. Ranks pinning host memory on - # one node starve each other: 256 GB/rank takes 45 min (TP2) / 27 min (TP4) - # all on node 0, and 21 s with TP2 split one per node. - if [[ -z "${ROCR_VISIBLE_DEVICES+x}" ]]; then - case "$TP" in - 2) NUMA_GPUS=0,4 ;; - 4) NUMA_GPUS=0,1,4,5 ;; - *) NUMA_GPUS="" ;; - esac - if [[ -n "$NUMA_GPUS" ]]; then - export ROCR_VISIBLE_DEVICES="$NUMA_GPUS" - export HIP_VISIBLE_DEVICES="$NUMA_GPUS" - echo "NUMA-spread GPUs for offload: $NUMA_GPUS" - fi - fi - - case "$OFFLOAD_TIER" in - cpu256) - export LMCACHE_MAX_LOCAL_CPU_SIZE="$((TOTAL_CPU_DRAM_GB / TP))" - export LMCACHE_CHUNK_SIZE=256 - # ATOM_SLRU needs rocm/atom-dev:nightly_202609140645-lirzhang-triton-build or later. - export ATOM_PREFIX_CACHE_POLICY=slru - export ATOM_PREFIX_CACHE_PROTECTED_RATIO=0.5 - export LMCACHE_CACHE_POLICY=ATOM_SLRU - # Must cover every rank; rank 0 alone halves the offload benefit. - export LMCACHE_LOOKUP_SERVER_WORKER_IDS="$(seq -s, 0 $((TP - 1)))" - ;; - *) - echo "CONC=$CONC has no measured offload tier" >&2 - exit 2 - ;; - esac - - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"lmcache_offload\",\"kv_role\":\"offload\"}" - ) - ;; - *) - echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2 - exit 1 - ;; -esac - -# TP4 only: ATOM requires tp_size == sparse_num_index_heads (4 for M3). CONC=20 is -# in both search-space rows, so this is what keeps it off the TP2 offload curve. -if [ "$INDEXER_CP" -eq 1 ] && [ "$TP" -eq 4 ]; then - export ATOM_M3_INDEXER_CP=1 - echo "ATOM_M3_INDEXER_CP=1 (TP4, CONC=$CONC)" -fi - -echo "Starting atom server..." -export PYTHONNOUSERSITE=1 - -# Without it the aiter kernel logs flood the server log for the whole replay. -export AITER_LOG_LEVEL -export AITER_SITUV2_A4W4=1 -export AITER_QUICK_REDUCE_QUANTIZATION=INT4 -export AITER_FLYDSL_STAGE2_FP8=1 -export ATOM_FORCE_ATTN_TRITON=1 - -# golden_al_distribution/minimaxm3_eagle3_gqa.yaml: minimax-m3.thinking_on[3] -> AL 2.78. -# Synthetic acceptance on throughput runs, real target verification on eval-only. -SPEC_ARGS=( - --method eagle3 - --draft-model "$DRAFT_MODEL" - --num-speculative-tokens "$NUM_SPEC_TOKENS" -) -if [ "${EVAL_ONLY}" != "true" ]; then - SPEC_ARGS+=(--spec-decode-acceptance-length "$SPEC_DECODE_AL") -fi -echo "SPEC_DECODE_AL=$SPEC_DECODE_AL NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS" - -ATOM_CMD=( - python -m atom.entrypoints.openai_server - --model "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --server-port "$PORT" - --trust-remote-code - --tensor-parallel-size "$TP" - --kv_cache_dtype fp8 - --block-size 128 - --max-num-seqs "$MAX_NUM_SEQS" - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" - --gpu-memory-utilization "$GPU_MEM_UTIL" - --cudagraph-capture-sizes "$CUDAGRAPH_CAPTURE_SIZES" - --index-cache-dtype fp8 - --online_quant_config '{"global_quant_config":"ptpc_fp8","exclude_layer":["lm_head","model.embed_tokens","vision_tower","multi_modal_projector","patch_merge_mlp","*block_sparse_moe"]}' - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - "${SPEC_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -if [[ "$ENABLE_PREFIX_CACHING" != "true" ]]; then - ATOM_CMD+=(--no-enable_prefix_caching) -fi -write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" -PYTHONPATH="$ATOM_RUNTIME_DEPS${PYTHONPATH:+:$PYTHONPATH}" \ - "${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh deleted file mode 100644 index def0ec12d6..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# MiniMax-M3 FP4 on MI355X with vLLM EAGLE3 speculative decoding. -# -# Required env vars: -# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, -# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION - -source "$(dirname "$0")/../../benchmark_lib.sh" - - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -NUM_SPEC_TOKENS=3 -# golden_al_distribution/minimaxm3_eagle3_gqa.yaml: minimax-m3.thinking_on[3] -SYNTHETIC_ACCEPT_LEN=2.78 - -if [[ -v SLURM_JOB_ID ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# ROCR/HIP visibility for vLLM 0.14+ -if [[ -v ROCR_VISIBLE_DEVICES ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -hf download "$DRAFT_MODEL" - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps - -# Require the vLLM Prometheus stream in every official result. -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -LMCACHE_PIDS=() -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - local i - for i in "${!LMCACHE_PIDS[@]}"; do - stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" - done - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -OFFLOAD_ARGS=() - -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - lmcache) - require_agentic_kv_offload_backend lmcache - - # Keep the image's torch/ROCm stack; install only LMCache's missing pure-Python deps. - LMCACHE_VERSION="0.5.3" - LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm" - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-rocm-7-0==14.1.1" \ - "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" - python3 -c \ - "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ - >/dev/null - - # Split the node-level DRAM limit evenly across one MP server per TP rank. - LMCACHE_N_SERVERS="$TP" - LMCACHE_L1_SIZE_GB="$TOTAL_CPU_DRAM_GB" - SHM_FREE_GB=$(df -BG --output=avail /dev/shm 2>/dev/null | tail -1 | tr -dc '0-9') - if [ -n "$SHM_FREE_GB" ] && [ "$SHM_FREE_GB" -gt 0 ]; then - SHM_CAP_GB=$((SHM_FREE_GB * 90 / 100)) - if [ "$LMCACHE_L1_SIZE_GB" -gt "$SHM_CAP_GB" ]; then - echo "Error: LMCache L1 ${LMCACHE_L1_SIZE_GB} GB exceeds 90% of free /dev/shm (${SHM_CAP_GB} GB)." >&2 - exit 1 - fi - fi - LMCACHE_L1_SHARD_GB=$((LMCACHE_L1_SIZE_GB / LMCACHE_N_SERVERS)) - if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then - echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2 - exit 1 - fi - - LMCACHE_SERVER_URLS=() - LMCACHE_HTTP_PORTS=() - LMCACHE_LOGS=() - : > "$RESULT_DIR/lmcache_command.txt" - for shard in $(seq 0 $((LMCACHE_N_SERVERS - 1))); do - shard_port=$((5555 + shard)) - shard_http_port=$((8080 + shard)) - shard_log="${LMCACHE_LOG%.log}_${shard}.log" - LMCACHE_CMD=( - lmcache server - --host 127.0.0.1 - --port "$shard_port" - --http-host 127.0.0.1 - --http-port "$shard_http_port" - --l1-size-gb "$LMCACHE_L1_SHARD_GB" - --l1-init-size-gb 10 - --l1-read-ttl-seconds 7200 - --chunk-size 256 - --max-workers 2 - --eviction-policy LRU - --supported-transfer-mode lmcache_driven - ) - append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" - "${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 & - LMCACHE_PIDS+=($!) - LMCACHE_HTTP_PORTS+=("$shard_http_port") - LMCACHE_LOGS+=("$shard_log") - LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}") - done - for shard in "${!LMCACHE_PIDS[@]}"; do - wait_for_ready \ - --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \ - --log "${LMCACHE_LOGS[$shard]}" \ - --pid "${LMCACHE_PIDS[$shard]}" \ - --sleep-interval 1 \ - --timeout 600 - done - LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}") - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}" - ) - ;; - *) - echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND (expected empty or lmcache)" >&2 - exit 1 - ;; -esac - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -# Synthetic acceptance standardizes throughput against the committed golden -# EAGLE3-GQA curve. Accuracy evals must use real target verification. -if [ "${EVAL_ONLY}" = "true" ]; then - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"ROCM_AITER_UNIFIED_ATTN\"}" -else - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"ROCM_AITER_UNIFIED_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1 - -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 - -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - "${PARALLEL_ARGS[@]}" - --trust-remote-code - --block-size 128 - --gpu-memory-utilization 0.90 - --enable-chunked-prefill - --max-num-batched-tokens 32768 - --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY", "cudagraph_capture_sizes": [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,20,22,24,26,28,30,32,34,36,48,64,72,80,88,96,104,112,120,128]}' - --language-model-only - --enable-prefix-caching - --attention-backend ROCM_AITER_UNIFIED_ATTN - --moe-backend aiter - --kv-cache-dtype fp8 - --attention-config '{"indexer_kv_dtype": "fp8"}' - --tool-call-parser minimax_m3 - --reasoning-parser minimax_m3 - --enable-auto-tool-choice - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --max-num-seqs "$((2 * CONC))" - --stream-interval 20 - --speculative-config "$SPEC_CONFIG" - "${OFFLOAD_ARGS[@]}" -) -write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh deleted file mode 100755 index fa0ae5239c..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# AgentX trace replay for Qwen3.5-397B-A17B NVFP4 on B300 with SGLang -# native NEXTN MTP. Throughput uses the committed golden synthetic AL; evals -# retain real target-model verification. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. - HICACHE_ALIGNMENT_RESERVE_GB=$TP - HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then - echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 - exit 1 - fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 - exit 1 - fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy write_through_selective - ) -fi - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) -STREAM_INTERVAL=50 - -# TP4 needs parallel tokenization to keep 256k AgentX warmups below the client -# request timeout. Keep TP2 on SGLang's single-worker default: multi-tokenizer -# startup races with the TP2 HiCache shared-memory initialization path. -TOKENIZER_ARGS=() -if [ "$TP" -ge 4 ]; then - TOKENIZER_ARGS=(--tokenizer-worker-num 6) -fi - -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -# Keep server-side connections alive beyond AIPerf's 300-second client pool -# timeout so bursty AgentX trajectories cannot reuse a closing idle socket. -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --enable-symm-mem - --quantization modelopt_fp4 - --fp4-gemm-backend flashinfer_cutlass - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend trtllm_mha - --moe-runner-backend flashinfer_trtllm - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 16384 - --chunked-prefill-size 16384 - --mem-fraction-static 0.80 - --stream-interval "$STREAM_INTERVAL" - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --enable-metrics - --enable-cache-report - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -capture_cache_metrics -trap capture_cache_metrics EXIT - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh deleted file mode 100755 index 3a41757657..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# AgentX trace replay for Qwen3.5-397B-A17B FP8 on B300 with SGLang -# native NEXTN MTP. Throughput uses the committed golden synthetic AL; evals -# retain real target-model verification. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. - HICACHE_ALIGNMENT_RESERVE_GB=$TP - HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then - echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 - exit 1 - fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 - exit 1 - fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy write_through_selective - ) -fi - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) - -# TP4 needs parallel tokenization to keep 256k AgentX warmups below the client -# request timeout. Keep TP2 on SGLang's single-worker default: multi-tokenizer -# startup races with the TP2 HiCache shared-memory initialization path. -TOKENIZER_ARGS=() -if [ "$TP" -ge 4 ]; then - TOKENIZER_ARGS=(--tokenizer-worker-num 6) -fi - -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -# Keep server-side connections alive beyond AIPerf's 300-second client pool -# timeout so bursty AgentX trajectories cannot reuse a closing idle socket. -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --enable-symm-mem - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend trtllm_mha - --moe-runner-backend flashinfer_trtllm - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 16384 - --chunked-prefill-size 16384 - --mem-fraction-static 0.80 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --enable-metrics - --enable-cache-report - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -capture_cache_metrics -trap capture_cache_metrics EXIT - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp8_h100_mtp.sh deleted file mode 100755 index 3f6726ee69..0000000000 --- a/benchmarks/single_node/agentic/qwen3.8next_fp8_h100_mtp.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Qwen3.8-Flash-Next FP8 on H100 with SGLang NEXTN MTP. Hopper has no NVFP4 -# tensor cores, so this arm serves Qwen/Qwen3.8-Flash-Next-FP8 (172.8 GiB). -# The cookbook has no H100 command; this is the H200 arm adjusted: TP8/EP8 -# (TP4 leaves ~43 GiB/rank of weights on an 80 GB card, too little for the -# 256k-capped traces) and --mem-fraction-static 0.75. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR \ - DURATION EP_SIZE -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# 256k-capped with-subagents corpus (470 traces): the unfiltered corpus has -# requests up to ~1M tokens the server would reject, and 80 GB is the tightest part here. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank. - HICACHE_HOST_POOL_COUNT="2" - HICACHE_WRITE_POLICY="write_through_selective" - HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SPEC_ARGS=( - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -# Acceptance is pinned to the committed golden AL (golden_al_distribution/README.md). -# EVAL_ONLY leaves it off: simulated acceptance commits drafted tokens -# regardless of target logits and the eval would score ~0. -if [ "${EVAL_ONLY}" != "true" ]; then - # golden_al_distribution/qwen3.8next_mtp.yaml: thinking_on[3] = 2.32 - # (3 speculative tokens per step; AgentX replays run with thinking on). - export SGLANG_SIMULATE_ACC_LEN=2.32 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -{ set +x; } 2>/dev/null -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -if [ "$CUDA_GRAPH_MAX_BS" -gt 64 ]; then - CUDA_GRAPH_MAX_BS=64 -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - # Cookbook H200 / FP8 flags adjusted for H100's 80 GB; NVFP4 is unavailable on Hopper. - --tp-size "$TP" - --ep-size "$EP_SIZE" - --dp-size 1 - --mem-fraction-static 0.75 - --chunked-prefill-size 8192 - --linear-attn-prefill-backend flashinfer - --linear-attn-decode-backend flashinfer - # float32, not the cookbook's bfloat16: with NEXTN the GDN backend verifies - # through flashinfer's gated_delta_rule_mtp, which asserts - # initial_state.dtype == torch.float32 and aborts CUDA graph capture - # (flashinfer/gdn_decode.py:761 via gdn_backend.py target_verify). - --mamba-ssm-dtype float32 - "${SPEC_ARGS[@]}" - --reasoning-parser auto - # NEXTN silently resets --max-running-requests to 48 when it is unset, so - # this must stay explicit and sized to the AgentX concurrency. - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi