From 08f319b699dc70814a8a13a3cf688efc0fb1c8cb Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 26 Sep 2026 01:54:53 -0400 Subject: [PATCH] [Klaud Cold] Delete 53 unused legacy single-node AgentX scripts from benchmarks/single_node/agentic #3428 moved single-node AgentX onto native srt-slurm recipes. Delete every script in benchmarks/single_node/agentic that no master config reaches any more. Keep the nine scripts still reached by search-space entries without an srt-recipe, plus the H100/H200 block32 installers and kernel_configs that the multi-node DeepSeek-V4.1-Flash recipes use. The README now marks the folder as legacy. Co-Authored-By: Claude Opus 5.5 (1M context) --- benchmarks/single_node/agentic/README.md | 7 + .../agentic/check_dsv4_dspark_checkpoint.py | 119 ------ .../agentic/dsv41flash_fp4_b200_sglang_mtp.sh | 165 -------- .../agentic/dsv41flash_fp4_b200_vllm_mtp.sh | 1 - .../agentic/dsv41flash_fp4_b300_sglang_mtp.sh | 174 -------- .../agentic/dsv41flash_fp4_b300_vllm_mtp.sh | 93 ----- .../dsv41flash_fp4_gb200_sglang_mtp.sh | 187 --------- .../agentic/dsv41flash_fp4_gb200_vllm_mtp.sh | 1 - .../agentic/dsv41flash_fp4_gb300_sglang.sh | 1 - .../dsv41flash_fp4_gb300_sglang_mtp.sh | 176 -------- .../agentic/dsv41flash_fp4_gb300_vllm_mtp.sh | 1 - .../agentic/dsv41flash_fp4_h100_sglang.sh | 1 - .../agentic/dsv41flash_fp4_h100_sglang_mtp.sh | 222 ---------- .../agentic/dsv41flash_fp4_h100_vllm_mtp.sh | 90 ---- .../agentic/dsv41flash_fp4_h200_sglang_mtp.sh | 207 --------- .../agentic/dsv41flash_fp4_h200_vllm_mtp.sh | 1 - .../agentic/dsv41flash_fp4_mi300x_mtp.sh | 148 ------- .../agentic/dsv41flash_fp4_mi325x_mtp.sh | 155 ------- .../dsv41flash_fp4_mi355x_sglang_mtp.sh | 167 -------- .../agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh | 174 -------- .../agentic/dsv41flash_fp4_vllm_mtp.sh | 120 ------ .../agentic/dsv4_fp4_b200_sglang_mtp.sh | 280 ------------- .../agentic/dsv4_fp4_b200_vllm_mtp.sh | 325 --------------- .../agentic/dsv4_fp4_b300_sglang_mtp.sh | 310 -------------- .../agentic/dsv4_fp4_b300_vllm_mtp.sh | 328 --------------- .../agentic/dsv4_fp4_mi355x_atom_mtp.sh | 301 ------------- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 277 ------------ .../agentic/dsv4_fp4_mi355x_vllm_mtp.sh | 394 ------------------ .../agentic/glm5.2_fp4_b200_sglang_mtp.sh | 307 -------------- .../agentic/glm5.2_fp4_b300_sglang_mtp.sh | 234 ----------- .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 244 ----------- .../agentic/glm5.2_fp8_b200_sglang_mtp.sh | 386 ----------------- .../agentic/glm5.2_fp8_b300_sglang_mtp.sh | 353 ---------------- .../agentic/glm5.2_fp8_mi325x_mtp.sh | 128 ------ .../agentic/glm5.2_fp8_mi355x_sglang_mtp.sh | 288 ------------- .../agentic/kimik3_fp4_b300_vllm_mtp.sh | 295 ------------- .../agentic/minimaxm3_fp4_b200_mtp.sh | 186 --------- .../agentic/minimaxm3_fp4_b200_trt_mtp.sh | 192 --------- .../agentic/minimaxm3_fp4_b300_mtp.sh | 119 ------ .../agentic/minimaxm3_fp4_b300_trt_mtp.sh | 181 -------- .../agentic/minimaxm3_fp8_h100_mtp.sh | 160 ------- .../agentic/minimaxm3_fp8_h200_mtp.sh | 215 ---------- .../agentic/minimaxm3_fp8_mi300x_mtp.sh | 196 --------- .../agentic/minimaxm3_fp8_mi325x_mtp.sh | 106 ----- .../agentic/qwen3.5_fp4_b200_sglang_mtp.sh | 171 -------- .../agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh | 149 ------- .../agentic/qwen3.5_fp8_b200_sglang_mtp.sh | 165 -------- .../agentic/qwen3.5_fp8_h100_mtp.sh | 130 ------ .../agentic/qwen3.5_fp8_h200_mtp.sh | 128 ------ .../agentic/qwen3.5_fp8_mi300x_mtp.sh | 109 ----- .../agentic/qwen3.5_fp8_mi325x_mtp.sh | 150 ------- .../qwen3.8next_fp4_b200_sglang_mtp.sh | 197 --------- .../qwen3.8next_fp4_b300_sglang_mtp.sh | 174 -------- .../qwen3.8next_fp8_h200_sglang_mtp.sh | 147 ------- 54 files changed, 7 insertions(+), 9528 deletions(-) delete mode 100644 benchmarks/single_node/agentic/check_dsv4_dspark_checkpoint.py delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_b200_sglang_mtp.sh delete mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_b200_vllm_mtp.sh delete mode 100644 benchmarks/single_node/agentic/dsv41flash_fp4_b300_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_b300_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh delete mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_gb200_vllm_mtp.sh delete mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang.sh delete mode 100644 benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh delete mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_gb300_vllm_mtp.sh delete mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_h100_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_h200_sglang_mtp.sh delete mode 120000 benchmarks/single_node/agentic/dsv41flash_fp4_h200_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_mi325x_mtp.sh delete mode 100644 benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh delete mode 100644 benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh delete mode 100644 benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh delete mode 100644 benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang_mtp.sh delete mode 100644 benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/glm5.2_fp8_b300_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh delete mode 100755 benchmarks/single_node/agentic/glm5.2_fp8_mi355x_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp8_h200_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh delete mode 100755 benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh delete mode 100644 benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh delete mode 100755 benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh diff --git a/benchmarks/single_node/agentic/README.md b/benchmarks/single_node/agentic/README.md index 3f886e605f..fa35ea2b3b 100644 --- a/benchmarks/single_node/agentic/README.md +++ b/benchmarks/single_node/agentic/README.md @@ -1,5 +1,12 @@ # Agentic single-node benchmarks +> **Legacy.** Single-node AgentX configs now run as declarative srt-slurm +> recipes (`benchmarks/single_node/srt-slurm-recipes////agentic.yaml`, +> see #3428). The only bash scripts left here are the ones a master config still +> reaches through a search-space entry without an `srt-recipe:`; delete each one +> when its config is ported. `install_h1{0,2}0_block32_configs.py` and +> `kernel_configs/` also serve the multi-node DeepSeek-V4.1-Flash H100/H200 recipes. + **MVP / experimental.** Nothing in this directory is an official InferenceX benchmark. Results are not published on https://inferencex.com and are not intended to be cited. diff --git a/benchmarks/single_node/agentic/check_dsv4_dspark_checkpoint.py b/benchmarks/single_node/agentic/check_dsv4_dspark_checkpoint.py deleted file mode 100644 index bb2a1b2a42..0000000000 --- a/benchmarks/single_node/agentic/check_dsv4_dspark_checkpoint.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Fail before GPU startup if the pinned DSpark checkpoint is incomplete.""" - -import argparse -import hashlib -import json -import struct -from pathlib import Path - - -REVISION = "72e1d3230f6c080a530b0a1d46f8eb4602340597" -CONFIG_BLOB = "7fc3b0c42c815a0a7481b5ca15f42139f94ac4ab" -INDEX_SHA256 = "2de2ac1e43134f8b03bf6156067715b7c3c73b1a507329e606023c601a56d30a" -HEAD_KEYS = ( - "mtp.2.markov_head.markov_w1.weight", - "mtp.2.markov_head.markov_w2.weight", - "mtp.2.confidence_head.proj.weight", -) - - -def check_checkpoint(model_path: Path, revision: str) -> dict: - if revision != REVISION: - raise ValueError(f"Unsupported checkpoint revision: {revision}") - model_path = model_path.resolve(strict=True) - if model_path.parent.name == "snapshots" and model_path.name != revision: - raise ValueError(f"Snapshot path does not match revision: {model_path}") - - config_bytes = (model_path / "config.json").read_bytes() - config_blob = hashlib.sha1( - f"blob {len(config_bytes)}\0".encode() + config_bytes - ).hexdigest() - if config_blob != CONFIG_BLOB: - raise ValueError(f"config.json does not match the pinned DSpark checkpoint: {config_blob}") - config = json.loads(config_bytes) - expected = { - "dspark_block_size": 5, "dspark_noise_token_id": 128799, - "dspark_target_layer_ids": [58, 59, 60], "dspark_markov_rank": 512, - } - for key, value in expected.items(): - if config.get(key) != value: - raise ValueError(f"Invalid {key}: {config.get(key)!r}, expected {value!r}") - - index_bytes = (model_path / "model.safetensors.index.json").read_bytes() - index_sha256 = hashlib.sha256(index_bytes).hexdigest() - if index_sha256 != INDEX_SHA256: - raise ValueError(f"Weight index does not match the pinned checkpoint: {index_sha256}") - weight_map = json.loads(index_bytes)["weight_map"] - shard_keys: dict[str, set[str]] = {} - for key, filename in weight_map.items(): - if Path(filename).name != filename: - raise ValueError(f"Invalid shard filename: {filename}") - shard_keys.setdefault(filename, set()).add(key) - if len(shard_keys) != 66 or any(key not in weight_map for key in HEAD_KEYS): - raise ValueError("Expected 66 shards and all DSpark Markov/confidence head weights") - - shards = [] - heads = {} - for filename, keys in sorted(shard_keys.items()): - path = model_path / filename - # Read actual safetensors headers and both payload boundaries, checking - # file permissions, missing tensors and truncated payloads without - # streaming the entire checkpoint twice before each benchmark. - with path.open("rb") as stream: - header_size = struct.unpack("= len(tokenizer): - raise ValueError("DSpark noise token is outside the tokenizer vocabulary") - tokenizer_files = {} - for name in ("tokenizer.json", "tokenizer_config.json"): - data = (model_path / name).read_bytes() - tokenizer_files[name] = {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} - return { - "model": "deepseek-ai/DeepSeek-V4-Pro-0813", "requested_revision": revision, - "model_path": str(model_path), "config_git_blob": config_blob, - "index_sha256": index_sha256, "dspark_config": expected, - "dspark_heads": heads, "shard_count": len(shards), "shards": shards, - "tokenizer_path": str(model_path), "tokenizer_files": tokenizer_files, - "tokenizer_class": type(tokenizer).__name__, "tokenizer_vocab_size": len(tokenizer), - "readability_check": "All headers, indexed tensor names, payload sizes and boundary reads; not full weight hashes", - } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--model-path", type=Path, required=True) - parser.add_argument("--revision", required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - manifest = check_checkpoint(args.model_path, args.revision) - args.output.write_text(json.dumps(manifest, indent=2) + "\n") - print(f"DSpark checkpoint preflight passed: {manifest['shard_count']} shards, tokenizer and heads readable") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_b200_sglang_mtp.sh deleted file mode 100755 index 5d404bcc19..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_b200_sglang_mtp.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash AgentX on B200 with shipped-default DSpark. -# TP4 covers the full concurrency curve; TP2 covers C1-C8. -# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY SPEC_DECODING -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -if (( TP == 2 )); then - # Reduce CUDA allocator fragmentation during TP2 loading. - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - echo "TP2 CUDA allocator: $PYTORCH_CUDA_ALLOC_CONF" -fi - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export PYTHONNOUSERSITE=1 -export PYTHONUNBUFFERED=1 - -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL -# curve. SGLang ships thinking off by default for this model. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV41_REASONING_EFFORT=high - -# Keep Engram in per-rank host shards to reserve HBM for reusable KV. -case "$TP" in - 2|4) export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 ;; - *) echo "Unsupported DSpark TP=$TP; expected 2 or 4" >&2; exit 1 ;; -esac -export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts, but -# never let the pool exceed the decode graph batch: a DSpark verify step for a -# batch above the captured 64 runs eagerly and allocates its attention -# workspace on the fly, which OOMed the H200 eval at 128 running requests -# (6.4 GiB allocation with 2 GiB free, run 35306704553). Batches within the -# graph tier reuse the capture-time workspace instead. -CUDA_GRAPH_MAX_BS=64 -MAX_RUNNING_REQUESTS=$((2 * CONC)) -if (( MAX_RUNNING_REQUESTS > CUDA_GRAPH_MAX_BS )); then - MAX_RUNNING_REQUESTS=$CUDA_GRAPH_MAX_BS -fi -# Chunked requests leave reusable SWA tails in the radix tree. Size retained -# tails by session concurrency, rather than the capped running-request count. -# The tails and full KV share a fixed pool; the cap preserves full-prefix space. -SWA_PREFIX_TAILS=$((64 * CONC)) -MEM_FRACTION_STATIC=0.80 -CHUNKED_PREFILL_SIZE=4096 -if (( TP == 2 )); then - if (( CONC > 8 )); then - echo "TP2 supports CONC <= 8 within its smaller KV budget" >&2 - exit 1 - fi - # TP2 has twice as many chunk boundaries and approximately 147.76 GiB of - # target plus draft weights. Smaller chunks bound indexer workspace. - SWA_PREFIX_TAILS=$((128 * CONC)) - MEM_FRACTION_STATIC=0.92 - CHUNKED_PREFILL_SIZE=2048 -fi -if (( SWA_PREFIX_TAILS < 128 )); then - SWA_PREFIX_TAILS=128 -elif (( SWA_PREFIX_TAILS > 4096 )); then - SWA_PREFIX_TAILS=4096 -fi - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if (( CONC >= 32 )); then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" - -# Throughput uses the committed golden acceptance curve; evals verify real -# draft tokens. Leave the pinned nightly's draft computation/precision defaults. -if [[ "$SPEC_DECODING" != mtp ]]; then - echo "Unsupported SPEC_DECODING=$SPEC_DECODING; expected mtp" >&2 - exit 1 -fi -unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE -DSPARK_BLOCK_SIZE=5 -DSV41_GOLDEN_AL=3.51 -if [[ "$EVAL_ONLY" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi -echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" - --trust-remote-code - --tp "$TP" --ep-size "$EP_SIZE" - # Backends resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl - # on Blackwell); the cookbook warns that overriding them costs decode speed. - # The sparse-attention indexer and DSpark prefill buffers scale with the - # chunk times the 1M context; bound chunks to retain transient workspace. - --mem-fraction-static "$MEM_FRACTION_STATIC" - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - # Long AgentX prefills otherwise starve active draft/verify decode rounds. - --prefill-decode-interval 16 - --swa-prefix-tails "$SWA_PREFIX_TAILS" - --speculative-algorithm DSPARK - --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --reasoning-parser auto - --tool-call-parser auto - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800 s default watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics ${AIPERF_SERVER_METRICS_URLS}" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_b200_vllm_mtp.sh deleted file mode 120000 index 157d224098..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_b200_vllm_mtp.sh +++ /dev/null @@ -1 +0,0 @@ -dsv41flash_fp4_vllm_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_b300_sglang_mtp.sh deleted file mode 100644 index dfb7056385..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_b300_sglang_mtp.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash AgentX on B300 with SGLang, supporting STP and DSpark. -# The KV cache is GPU-resident; caller SPEC_DECODING selects the serving mode. -# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY SPEC_DECODING -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - MODEL_PATH=$(python3 -c 'from huggingface_hub import snapshot_download; import sys; print(snapshot_download(repo_id=sys.argv[1], local_files_only=True))' "$MODEL") - export MODEL_PATH -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export PYTHONNOUSERSITE=1 -export PYTHONUNBUFFERED=1 - -# Use the default DSpark precision shipped by the pinned SGLang nightly. - -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL -# curve. SGLang ships thinking off by default for this model. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV41_REASONING_EFFORT=high - -# Keep Engram tables in host RAM to make room for long-context AgentX KV. -# Per-rank anonymous mappings can use THP without requiring shared-memory THP -# or host sysctl changes. The table payload remains native FP8. -export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 -export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts, but -# never let the pool exceed the decode graph batch: a DSpark verify step for a -# batch above the captured 64 runs eagerly and allocates its attention -# workspace on the fly, which OOMed the H200 eval at 128 running requests -# (6.4 GiB allocation with 2 GiB free, run 35306704553). Batches within the -# graph tier reuse the capture-time workspace instead. -CUDA_GRAPH_MAX_BS=64 -MAX_RUNNING_REQUESTS=$((2 * CONC)) -if (( MAX_RUNNING_REQUESTS > CUDA_GRAPH_MAX_BS )); then - MAX_RUNNING_REQUESTS=$CUDA_GRAPH_MAX_BS -fi - -# AgentX reuses long prefixes across turns even at low session concurrency. -# At memory fraction 0.70, TP4 has 110.55 GiB of KV budget but TP2 has only -# 38.83 GiB. Give TP2's larger working sets more KV space before retaining -# additional SWA tails; keep the conservative low-concurrency allocation. -MEM_FRACTION_STATIC=0.70 -if (( TP == 2 && CONC >= 32 )); then - # At C64, 0.80 left 43.59 GiB after graphs but only 18.29M full tokens. - # Retain more long prefixes while leaving room for transient prefills. - MEM_FRACTION_STATIC=0.85 -elif (( TP == 2 && CONC >= 16 )); then - MEM_FRACTION_STATIC=0.80 -fi -SWA_PREFIX_TAILS=$((64 * CONC)) -if (( SWA_PREFIX_TAILS > 4096 )); then - SWA_PREFIX_TAILS=4096 -fi -if (( SWA_PREFIX_TAILS < 128 )); then - SWA_PREFIX_TAILS=128 -fi - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if (( CONC >= 32 )); then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" - -# STP and accuracy evaluations must never inherit synthetic acceptance. -unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE -SPECULATIVE_ARGS=() -case "$SPEC_DECODING" in - none) - echo "Non-speculative decoding; synthetic acceptance disabled" - ;; - mtp) - # Existing measured curve: dsv41flash_dspark.yaml, thinking_on, K5. - SPECULATIVE_ARGS=(--speculative-algorithm DSPARK --speculative-dspark-block-size 5) - if [[ "$EVAL_ONLY" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN=3.51 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token - fi - ;; - *) - echo "Unsupported SPEC_DECODING: $SPEC_DECODING" >&2 - exit 1 - ;; -esac - -# At high TP2 concurrency, test more prefill duty against the matched C64 -# baseline. Keep the latency-oriented cadence on low-C and TP4 points. -PREFILL_DECODE_INTERVAL=16 -if (( TP == 2 && CONC >= 32 )); then - PREFILL_DECODE_INTERVAL=4 -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" - --trust-remote-code - # Feed mmap weight copies sequentially from shared Lustre storage. - --weight-loader-prefetch-checkpoints - --tp "$TP" --ep-size "$EP_SIZE" - # Backends resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl - # on Blackwell); the cookbook warns that overriding them costs decode speed. - # Bound transient prefill allocations: the sparse-attention indexer and - # DSpark buffers scale with the chunk times the 1M context. Static KV - # memory is selected above from the measured TP2/TP4 weight footprints. - --mem-fraction-static "$MEM_FRACTION_STATIC" - --chunked-prefill-size 4096 - # Long AgentX prefills otherwise starve ready decode requests. - --prefill-decode-interval "$PREFILL_DECODE_INTERVAL" - "${SPECULATIVE_ARGS[@]}" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --swa-prefix-tails "$SWA_PREFIX_TAILS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --reasoning-parser auto - --tool-call-parser auto - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800 s default watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics ${AIPERF_SERVER_METRICS_URLS}" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_b300_vllm_mtp.sh deleted file mode 100755 index 233c79beec..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_b300_vllm_mtp.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# Native DeepSeek-V4.1-Flash DSpark and Engram UVA weight offload for B300. -# https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-3600}" -export VLLM_USE_RUST_FRONTEND=1 -export PYTHONUNBUFFERED=1 - -NUM_SPEC_TOKENS=5 - -# Piecewise CUDA graph capture sizes are multiples of the six-token DSpark -# verification block, with a denser set for smaller batches. -GRAPH_SIZES_2046='6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046' -GRAPH_SIZES_8190="${GRAPH_SIZES_2046},3072,4092,6144,8190" - -# Match each batched-token limit to the largest graph captured by that tier. -GPU_MEMORY_UTILIZATION="" -if (( CONC <= 4 )); then - GRAPH_SIZES="$GRAPH_SIZES_2046"; MAX_BATCHED_TOKENS=2048 -elif (( CONC >= 128 && TP == 2 )); then - GRAPH_SIZES="$GRAPH_SIZES_2046"; MAX_BATCHED_TOKENS=2048 - GPU_MEMORY_UTILIZATION=0.97 -else - GRAPH_SIZES="$GRAPH_SIZES_8190"; MAX_BATCHED_TOKENS=8192 -fi -MAX_NUM_SEQS=256 -CAPTURE_SIZE="${GRAPH_SIZES##*,}" -COMPILATION_CONFIG="{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[${GRAPH_SIZES}]}" - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" -echo "Using vLLM endpoint ${AIPERF_SERVER_URL}" - -# Golden AL: golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Accuracy evals keep real block rejection; other runs use synthetic acceptance at AL 3.51. -if [[ "${EVAL_ONLY:-false}" == true ]]; then - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":true}' -else - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}' -fi -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" - --language-model-only - --tokenizer-mode deepseek_v41 - --tool-call-parser deepseek_v41 --enable-auto-tool-choice - --reasoning-parser deepseek_v41 - --engram-config '{"cpu_offload":true}' - --speculative-config "$SPEC_CONFIG" - --max-model-len 1048576 - --compilation-config "$COMPILATION_CONFIG" - --max-cudagraph-capture-size "$CAPTURE_SIZE" - --max-num-batched-tokens "$MAX_BATCHED_TOKENS" - --max-num-seqs "$MAX_NUM_SEQS" - --disable-uvicorn-access-log -) -if [[ -n "$GPU_MEMORY_UTILIZATION" ]]; then - VLLM_CMD+=(--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION") -fi -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY:-false}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh deleted file mode 100755 index a41ea220f1..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_sglang_mtp.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash AgentX on GB200 with shipped-default DSpark serving. -# Match vLLM's TP2/EP1 and TP4/EP1 layouts with GPU-resident KV cache. -# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY SPEC_DECODING -require_agentic_kv_offload_none -export GPU_COUNT="$TP" -if (( TP == 2 )); then - # Bound fragmentation during stock MXFP4 loading and long-context prefills. - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -fi - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - MODEL_PATH=$(hf download "$MODEL") - export MODEL_PATH -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export PYTHONNOUSERSITE=1 -export PYTHONUNBUFFERED=1 - -# Use the default DSpark precision shipped by the pinned SGLang nightly. - -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL -# curve. SGLang ships thinking off by default for this model. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV41_REASONING_EFFORT=high - -# Keep the Engram weights in row-sharded host DRAM. GB200's 64 KiB-page -# kernel enables anonymous THP with madvise but disables shmem THP, so the -# shared memfd layout cannot obtain huge-page backing. The upstream per-rank -# layout uses anonymous mappings, MADV_HUGEPAGE and MADV_COLLAPSE for 512 MiB -# pages; row ownership and the original FP8 table weights are preserved. -# This trades two TP all-reduces for fewer host-table translation misses. -export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 -export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts, but -# never let the pool exceed the decode graph batch: a DSpark verify step for a -# batch above the captured 64 runs eagerly and allocates its attention -# workspace on the fly, which OOMed the H200 eval at 128 running requests -# (6.4 GiB allocation with 2 GiB free, run 35306704553). Batches within the -# graph tier reuse the capture-time workspace instead. -CUDA_GRAPH_MAX_BS=64 -MEM_FRACTION_STATIC=0.70 -CHUNKED_PREFILL_SIZE=4096 -case "$TP" in - 2) - # EP1 keeps all 384 experts tensor-sharded per rank. The nearby B200 - # EP1 run passed full GSM8K with these supported memory limits; GB200 - # still requires its own pool, graph and full-curve validation. - MEM_FRACTION_STATIC=0.92 - CHUNKED_PREFILL_SIZE=2048 - CUDA_GRAPH_MAX_BS=16 - ;; - 4) ;; - *) echo "Unsupported TP=$TP; expected 2 or 4" >&2; exit 1 ;; -esac -MAX_RUNNING_REQUESTS=$((2 * CONC)) -if (( MAX_RUNNING_REQUESTS > CUDA_GRAPH_MAX_BS )); then - MAX_RUNNING_REQUESTS=$CUDA_GRAPH_MAX_BS -fi - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if (( CONC >= 32 )); then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" - -# The caller selects native non-speculative serving or the bundled DSpark -# draft. STP and accuracy evals must never inherit synthetic acceptance. -unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE -SPECULATIVE_ARGS=() -case "$SPEC_DECODING" in - mtp) - DSPARK_BLOCK_SIZE=5 - DSV41_GOLDEN_AL=3.51 - SPECULATIVE_ARGS=(--speculative-algorithm DSPARK --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE") - if [[ "$EVAL_ONLY" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token - fi - echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" - ;; - none) - echo "Native non-speculative serving; synthetic acceptance disabled" - ;; - *) - echo "Unsupported SPEC_DECODING=$SPEC_DECODING; expected mtp or none" >&2 - exit 1 - ;; -esac - -# Cached prefixes need their final SWA window as well as full-attention KV. -# C16 measured 27.0M full tokens with 1,024 retained tails. Cap the reserve: -# uncapped 64*CONC at C128 would exceed this node's measured KV budget. -SWA_PREFIX_TAILS=$((64 * CONC)) -if (( TP == 2 )); then - SWA_PREFIX_TAILS=$((128 * CONC)) -fi -if (( SWA_PREFIX_TAILS > 1024 )); then - SWA_PREFIX_TAILS=1024 -fi - -# Earlier TP4/EP4 C16 canonical comparison: +13.65% p90 interactivity, -0.30% throughput, -# with p90 TTFT increasing from 2.35 s to 3.51 s. Other TP4 points keep defaults. -# TP2 retains the supported interval used by its B200 EP1 memory qualification. -SCHEDULING_ARGS=() -if (( TP == 2 || CONC == 16 )); then - SCHEDULING_ARGS=(--prefill-decode-interval 16) -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" - --trust-remote-code - --tp "$TP" --ep-size "$EP_SIZE" - # Backends resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl - # on Blackwell); the cookbook warns that overriding them costs decode speed. - # 0.70 rather than the cookbook's 0.8, and a bounded prefill chunk: the - # sparse-attention indexer and DSpark prefill buffers scale with the chunk - # times the 1M context, and the default 16384 chunk exhausted HBM on the - # first 66k-99k-token AgentX prompts. - --mem-fraction-static "$MEM_FRACTION_STATIC" - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --swa-prefix-tails "$SWA_PREFIX_TAILS" - "${SCHEDULING_ARGS[@]}" - "${SPECULATIVE_ARGS[@]}" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --reasoning-parser auto - --tool-call-parser auto - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800 s default watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics ${AIPERF_SERVER_METRICS_URLS}" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_vllm_mtp.sh deleted file mode 120000 index 157d224098..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb200_vllm_mtp.sh +++ /dev/null @@ -1 +0,0 @@ -dsv41flash_fp4_vllm_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang.sh deleted file mode 120000 index 02ddd80c3f..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang.sh +++ /dev/null @@ -1 +0,0 @@ -dsv41flash_fp4_gb300_sglang_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh deleted file mode 100644 index 5dd3edb473..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_sglang_mtp.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash AgentX on GB300 with native STP or DSpark serving. -# Both use TP2/TP4 and GPU-resident KV cache. -# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY SPEC_DECODING -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - # Engram's model-local cache advice needs actual checkpoint file paths. - MODEL_PATH=$(python3 -c 'from huggingface_hub import snapshot_download; import sys; print(snapshot_download(repo_id=sys.argv[1], local_files_only=True))' "$MODEL") - export MODEL_PATH -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export PYTHONNOUSERSITE=1 -export PYTHONUNBUFFERED=1 - -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL -# curve. SGLang ships thinking off by default for this model. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV41_REASONING_EFFORT=high - -# TP4 has room for the original Engram tables in HBM: the STP baseline used -# 73.3 GiB for weights before the ~47.2 GiB tables. Host shards still had 0% -# huge-page backing on some ranks after model-local cache advice, so avoid -# that lookup bottleneck on TP4. TP2 retains the anonymous host-table layout. -if (( TP >= 4 )); then - export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=0 - unset SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT -else - export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 - export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank -fi - -# The bundled Markov embedding/head weights are natively BF16. Preserve the -# nightly default that keeps W2 BF16 instead of converting it to FP32. -export SGLANG_DSPARK_OPT_MARKOV_W2_BF16=True - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts, but -# never let the pool exceed the decode graph batch: a DSpark verify step for a -# batch above the captured 64 runs eagerly and allocates its attention -# workspace on the fly, which OOMed the H200 eval at 128 running requests -# (6.4 GiB allocation with 2 GiB free, run 35306704553). Batches within the -# graph tier reuse the capture-time workspace instead. -CUDA_GRAPH_MAX_BS=64 -MAX_RUNNING_REQUESTS=$((2 * CONC)) -if (( MAX_RUNNING_REQUESTS > CUDA_GRAPH_MAX_BS )); then - MAX_RUNNING_REQUESTS=$CUDA_GRAPH_MAX_BS -fi - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if (( CONC >= 32 )); then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" - -# The caller selects native non-speculative serving or the bundled DSpark -# draft. STP and accuracy evals must never inherit synthetic acceptance. -unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE -SPECULATIVE_ARGS=() -# Bound long-prefill decode stalls for DSpark as well as the STP comparison. -SCHEDULING_ARGS=(--prefill-decode-interval 16) -# Compressed full-KV capacity alone cannot preserve a prefix whose SWA tail -# has been evicted. Reserve more cached tails within the static pool for the -# DSpark comparisons above C1; checkpoint math stays unchanged. -CACHE_ARGS=() -if [[ "$SPEC_DECODING" == mtp ]] && (( CONC >= 2 )); then - SWA_PREFIX_TAILS=$((64 * CONC)) - if (( SWA_PREFIX_TAILS > 4096 )); then - SWA_PREFIX_TAILS=4096 - fi - CACHE_ARGS=(--swa-prefix-tails "$SWA_PREFIX_TAILS") -fi -case "$SPEC_DECODING" in - mtp) - DSPARK_BLOCK_SIZE=5 - DSV41_GOLDEN_AL=3.51 - SPECULATIVE_ARGS=(--speculative-algorithm DSPARK --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE") - if [[ "$EVAL_ONLY" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token - fi - echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" - ;; - none) - # Interleave decode with long chunked prefills to bound decode stalls. - echo "Native non-speculative serving; synthetic acceptance disabled" - ;; - *) - echo "Unsupported SPEC_DECODING=$SPEC_DECODING; expected mtp or none" >&2 - exit 1 - ;; -esac - -# Test larger prefill batches at high TP2 concurrency while preserving -# the validated static cache budget and native 1M context. -MEM_FRACTION_STATIC=0.80 -CHUNKED_PREFILL_SIZE=4096 -if (( TP == 2 && CONC >= 64 )); then - CHUNKED_PREFILL_SIZE=8192 -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" - --trust-remote-code - --tp "$TP" --ep-size "$EP_SIZE" - # Backends resolve automatically (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl - # on Blackwell); the cookbook warns that overriding them costs decode speed. - # Bound prefill workspace while retaining the native 1M context. - --mem-fraction-static "$MEM_FRACTION_STATIC" - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - "${SPECULATIVE_ARGS[@]}" - "${SCHEDULING_ARGS[@]}" - "${CACHE_ARGS[@]}" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --reasoning-parser auto - --tool-call-parser auto - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800 s default watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics ${AIPERF_SERVER_METRICS_URLS}" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_vllm_mtp.sh deleted file mode 120000 index 157d224098..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_gb300_vllm_mtp.sh +++ /dev/null @@ -1 +0,0 @@ -dsv41flash_fp4_vllm_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang.sh deleted file mode 120000 index 7c15051545..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang.sh +++ /dev/null @@ -1 +0,0 @@ -dsv41flash_fp4_h100_sglang_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang_mtp.sh deleted file mode 100755 index 3b56dfb43e..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_h100_sglang_mtp.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash AgentX on H100 with SGLang DSpark. A copy of -# dsv41flash_fp4_sglang_mtp.sh rather than a symlink: H100 is not in the -# cookbook's hardware table, and 80 GB cards cannot hold the resident weights -# plus the row-sharded Engram tables (~23.6 GiB per rank at TP8, measured on -# the vLLM arm) and still leave a KV pool. The Engram tables move to per-rank anonymous -# host shards, and the prefill chunk is bounded so the sparse-attention indexer's -# [chunk, context] scoring buffer fits next to the weights at 1M context. -# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY SPEC_DECODING DP_ATTENTION -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - # Resolve the downloaded snapshot so upstream Engram page-cache advice - # can find the checkpoint files instead of treating the HF ID as a path. - MODEL_PATH=$(python3 -c 'from huggingface_hub import snapshot_download; import sys; print(snapshot_download(repo_id=sys.argv[1], local_files_only=True))' "$MODEL") - export MODEL_PATH -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -# Hardware-specific tiling only; checkpoint data, scales and dtypes are unchanged. -# Resolve and verify the installed configs through the nightly's actual loader. -python3 "$(dirname "$0")/install_h100_block32_configs.py" \ - "$(dirname "$0")/kernel_configs/h100_dsv41_block32" "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export PYTHONNOUSERSITE=1 -export PYTHONUNBUFFERED=1 -# Long-context indexer masks change allocation sizes across requests. At C20, -# the stock allocator OOMed on a 2.39 GiB mask with 5.61 GiB reserved but unused. -export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL -# curve. SGLang ships thinking off by default for this model. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV41_REASONING_EFFORT=high - -# Row-sharded anonymous host Engram tables allow transparent huge pages; -# the shared memfd layout cannot use them when shmem THP is disabled. -# Checkpoint weights/scales are unchanged, and the KV cache stays on GPU. -export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 -export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts, and -# keep decode graphs covering that fan-out down to the cookbook's 64. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -if [[ "$DP_ATTENTION" == true ]] && (( MAX_RUNNING_REQUESTS < TP )); then - MAX_RUNNING_REQUESTS=$TP -fi -CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS -if (( CUDA_GRAPH_MAX_BS < 64 )); then - CUDA_GRAPH_MAX_BS=64 -elif (( CUDA_GRAPH_MAX_BS > 128 )); then - CUDA_GRAPH_MAX_BS=128 -fi - -# The indexer's scoring buffer and the hyper-connection activations scale with -# the prefill chunk times the 1M context. 4096 at mem-fraction 0.8 left 16 GB -# of headroom on the 80 GB card and c8 OOMed in eager extend once five -# requests were live with a 570k-token prompt pending (run 35304509605); 2048 -# with 0.7 leaves 24 GB and halves the per-chunk working set. -# Back to 4096 with the static fraction kept at 0.7: at 2048 prefill ran near -# 1,000 tok/s and c4 failed AIPerf's 95% latency-coverage check (TTFT 87.6%, -# ITL 88.4% over the 3600 s window, run 35307250127) while c1/c2 passed. -# 0.7 leaves 24 GB for the doubled per-chunk working set instead of the -# 16 GB that OOMed c8 at 0.8/4096. -CHUNKED_PREFILL_SIZE=4096 - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if (( CONC >= 32 )); then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -SGLANG_BACKEND_PORT="$PORT" -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -if [[ "$DP_ATTENTION" == true ]]; then - # The shipped MoE DSpark worker requires attn_tp=1 under DP attention. - # Keep the engine-wide 4096-token chunk budget for the DP sweep; - # SGLang divides it by DP, yielding 512 tokens/rank at TP8/DP8. - PARALLEL_ARGS+=(--enable-dp-attention --dp-size "$TP" --enable-dp-lm-head) - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true -elif [[ "$DP_ATTENTION" != false ]]; then - echo "Error: DP_ATTENTION must be true or false, got '$DP_ATTENTION'" >&2 - exit 1 -fi -export AIPERF_SERVER_METRICS_URLS="http://localhost:${SGLANG_BACKEND_PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" - -# The caller selects native non-speculative serving or the bundled DSpark -# draft. STP and accuracy evals must never inherit synthetic acceptance. -unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE -SPECULATIVE_ARGS=() -# Bound long-prefill decode stalls for DSpark as well as the STP comparison. -SCHEDULING_ARGS=(--prefill-decode-interval 16) -case "$SPEC_DECODING" in - mtp) - DSPARK_BLOCK_SIZE=5 - DSV41_GOLDEN_AL=3.51 - SPECULATIVE_ARGS=(--speculative-algorithm DSPARK --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE") - if [[ "$EVAL_ONLY" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token - fi - echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" - ;; - none) - # Long AgentX prompts otherwise keep prefill ahead of every ready decode. - # Interleave decode steps without changing requests or context lengths. - echo "Native non-speculative serving; synthetic acceptance disabled" - ;; - *) - echo "Unsupported SPEC_DECODING=$SPEC_DECODING; expected mtp or none" >&2 - exit 1 - ;; -esac - -# The DP pool is per rank. 64 tails/rank preserves the C16 TP baseline's -# aggregate 512-tail reserve while leaving an estimated 4.4M full tokens/rank. -SWA_PREFIX_TAILS=$(( CONC >= 4 ? 32 * CONC : 8 * CONC )) -if [[ "$DP_ATTENTION" == true ]]; then - SWA_PREFIX_TAILS=64 -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - # Native MXFP4 Marlin supports Hopper with BF16 activations; dense FP8 - # operators and shipped DSpark precision remain unchanged. - --moe-runner-backend marlin - --mem-fraction-static 0.7 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - # The 14.96 GiB H100 KV budget cannot afford the larger Blackwell tail - # reserve. At C20, 640 tails retain about 5.3M full tokens while reducing - # the measured eviction pressure on the default 160-tail SWA pool. - # Retain the measured TP low-concurrency reserve; DP uses its own rank-local pool. - --swa-prefix-tails "$SWA_PREFIX_TAILS" - "${SPECULATIVE_ARGS[@]}" - "${SCHEDULING_ARGS[@]}" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --reasoning-parser auto - --tool-call-parser auto - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800 s default watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == true ]]; then - # Stable session keys preserve prefix reuse across turns. The DP-aware - # router selects a rank through SGLang's supported routed_dp_rank path. - ROUTER_LOG="$RESULT_DIR/router.log" - ROUTER_CMD=( - python3 -m sglang_router.launch_router - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" - --policy consistent_hashing - --request-id-headers x-correlation-id - --dp-aware - --host 0.0.0.0 --port "$PORT" - --prometheus-host 127.0.0.1 - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" - --connect-timeout-secs 900 --request-timeout-secs 14400 - --disable-health-check --disable-retries - ) - write_command "$RESULT_DIR/router_command.txt" "${ROUTER_CMD[@]}" - "${ROUTER_CMD[@]}" > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics ${AIPERF_SERVER_METRICS_URLS}" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_h100_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_h100_vllm_mtp.sh deleted file mode 100755 index b5dce965a0..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_h100_vllm_mtp.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash on H100. A copy of dsv41flash_fp4_vllm_mtp.sh rather than -# a symlink: at 1M context the sparse-attention indexer allocates a -# [max-num-batched-tokens, max-model-len] fp8 logits buffer during startup -# profiling (8192 x 1048576 x 2 B = 16 GiB), which OOMs next to ~36 GiB/GPU of -# weights on 80 GB cards. Capping batched tokens shrinks it; capping -# --max-model-len would force the 256k-capped corpus onto a 1M-context model. -# https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_RUST_FRONTEND=1 -export VLLM_USE_V2_MODEL_RUNNER=1 -export PYTHONUNBUFFERED=1 -# The indexer buffer is large enough that allocator fragmentation costs a KV block. -export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True - -# The indexer buffer scales linearly with batched tokens: 4096 puts it at -# 8 GiB, and anything larger did not fit next to the weights on this SKU. -MAX_NUM_BATCHED_TOKENS=4096 -# vLLM's default 1024 sizes sampler and scheduler buffers for a batch this -# recipe never runs; 2*CONC leaves headroom for AgentX subagent fan-out. -MAX_NUM_SEQS=$((2 * CONC)) -NUM_SPEC_TOKENS=5 -CAPTURE_SIZE=1 -while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do - CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) -done - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" -echo "Using vLLM endpoint ${AIPERF_SERVER_URL}" - -# Golden AL: golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Accuracy evals keep real block rejection; throughput fixes acceptance to AL 3.51. -if [[ "${EVAL_ONLY}" == true ]]; then - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":true}' -else - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}' -fi -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" - --language-model-only - --tokenizer-mode deepseek_v41 - --tool-call-parser deepseek_v41 --enable-auto-tool-choice - --reasoning-parser deepseek_v41 - --engram-config '{"cpu_offload":true}' - --speculative-config "$SPEC_CONFIG" - --max-model-len 1048576 - --max-num-seqs "$MAX_NUM_SEQS" - --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" - --gpu-memory-utilization 0.92 - --max-cudagraph-capture-size "$CAPTURE_SIZE" - --disable-uvicorn-access-log -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_h200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_h200_sglang_mtp.sh deleted file mode 100755 index 21ca4d8202..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_h200_sglang_mtp.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash AgentX on H200 with SGLang native DSpark, following the -# published vLLM baseline topologies TP4/EP1 and TP8/EP1. -# The KV cache is GPU-resident. -# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY DP_ATTENTION -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export PYTHONNOUSERSITE=1 -export PYTHONUNBUFFERED=1 - -# Install measured H200 launch configurations: the replicated projection on -# TP4 and TP8, plus the qualified TP8 sharded shapes. Kernel code, precision -# and large-prefill tilings remain unchanged. -python3 "$(dirname "$0")/install_h200_block32_configs.py" \ - "$(dirname "$0")/kernel_configs/h200_dsv41_block32" "$RESULT_DIR" "$TP" - -# Matched C1 screens favored CUTLASS on TP8 p90 interactivity, while Marlin -# retained a small throughput/interactivity advantage on TP4. Both consume -# native MXFP4 weights with BF16 activations; dense GEMMs are unchanged. -# EP1 tensor-shards the 2304-wide routed experts: TP4=576, TP8=288. -# The shipped SM90 CUTLASS method requires multiples of 128 and rejects -# those widths; native Marlin supports padding without changing precision. -MOE_RUNNER_BACKEND=marlin -if (( TP == 8 && EP_SIZE != 1 )); then - MOE_RUNNER_BACKEND=flashinfer_mxfp4 -fi - -# Use the default DSpark precision shipped by the pinned SGLang nightly. - -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL -# curve. SGLang ships thinking off by default for this model. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV41_REASONING_EFFORT=high - -# Move the two fp8 Engram tables to host memory, freeing ~23 GiB of HBM per -# GPU for the 1M-context prefill working set and KV pool. Use row-sharded -# anonymous mappings: H200 compute nodes allow anonymous THP with madvise, -# but shmem_enabled=never prevents the shared memfd layout from using huge -# pages. This changes table placement, preserving checkpoint weights/scales. -# The first sweep ran -# with the tables on GPU and the server died on the first long AgentX prompts -# (run 35304458924: c128 died at startup with torch.OutOfMemoryError (12 GiB allocation, 5 GiB free of 139.8 GiB, 127.6 GiB already held by PyTorch)). -export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 -export SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT=per_rank - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts, but -# never let the pool exceed the decode graph batch: a DSpark verify step for a -# batch above the captured 64 runs eagerly and allocates its attention -# workspace on the fly, which OOMed the H200 eval at 128 running requests -# (6.4 GiB allocation with 2 GiB free, run 35306704553). Batches within the -# graph tier reuse the capture-time workspace instead. -CUDA_GRAPH_MAX_BS=64 -MAX_RUNNING_REQUESTS=$((2 * CONC)) -if (( MAX_RUNNING_REQUESTS > CUDA_GRAPH_MAX_BS )); then - MAX_RUNNING_REQUESTS=$CUDA_GRAPH_MAX_BS -fi -if [[ "$DP_ATTENTION" == true ]] && (( MAX_RUNNING_REQUESTS < TP )); then - # SGLang divides this global cap by attention DP size when sizing pools. - # Low-concurrency DP still needs at least one request slot per rank. - MAX_RUNNING_REQUESTS=$TP -fi - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if (( CONC >= 32 )); then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -SGLANG_BACKEND_PORT="$PORT" -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -if [[ "$DP_ATTENTION" == true ]]; then - # The shipped MoE DSpark worker requires attn_tp=1 under DP attention. - # Keep the engine-wide 4096-token chunk budget across the supported DP8 arm; - # SGLang divides it by DP, yielding 512 tokens/rank at TP8/DP8. - PARALLEL_ARGS+=(--enable-dp-attention --dp-size "$TP" --enable-dp-lm-head) - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true -elif [[ "$DP_ATTENTION" != false ]]; then - echo "Error: DP_ATTENTION must be true or false, got '$DP_ATTENTION'" >&2 - exit 1 -fi -export AIPERF_SERVER_METRICS_URLS="http://localhost:${SGLANG_BACKEND_PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" - -# DSpark is the checkpoint's own bundled draft: no EAGLE/MTP path and no -# --speculative-num-steps knob; the block size is the only tunable. Golden AL: -# golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Throughput fixes acceptance to AL 3.51; accuracy evals keep real verification. -DSPARK_BLOCK_SIZE=5 -DSV41_GOLDEN_AL=3.51 -if [[ "${EVAL_ONLY}" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi -echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --attention-backend dsv4 --moe-runner-backend "$MOE_RUNNER_BACKEND" - # 0.70 rather than the cookbook's 0.8, and a bounded prefill chunk: the - # sparse-attention indexer and DSpark prefill buffers scale with the chunk - # times the 1M context, and the default 16384 chunk exhausted HBM on the - # first 66k-99k-token AgentX prompts. - --mem-fraction-static 0.70 - # 4096, as on B200/GB200: at 8192 the indexer's prefill top-k allocated 5 GiB - # with 29 requests in flight and OOMed c32 (run 35308550355). - --chunked-prefill-size 4096 - # Keep active decode requests progressing while long prefixes are queued. - --prefill-decode-interval 16 - --enable-decoder-swa-bounded-replay - # The default 4*max-running-requests retains too few SWA prefix tails: - # C16 exhausted its 94,976-slot SWA pool while millions of full-pool - # slots remained free. Rebalance the existing KV budget toward reusable - # tails; weights, KV precision and the total static budget stay unchanged. - --swa-prefix-tails 1024 - --speculative-algorithm DSPARK - --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --reasoning-parser auto - --tool-call-parser auto - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800 s default watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == true ]]; then - # Stable session keys preserve prefix reuse across turns. The DP-aware - # router selects a rank through SGLang's supported routed_dp_rank path. - ROUTER_LOG="$RESULT_DIR/router.log" - ROUTER_CMD=( - python3 -m sglang_router.launch_router - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" - --policy consistent_hashing - --request-id-headers x-correlation-id - --dp-aware - --host 0.0.0.0 --port "$PORT" - --prometheus-host 127.0.0.1 - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" - --connect-timeout-secs 900 --request-timeout-secs 14400 - --disable-health-check --disable-retries - ) - write_command "$RESULT_DIR/router_command.txt" "${ROUTER_CMD[@]}" - "${ROUTER_CMD[@]}" > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics ${AIPERF_SERVER_METRICS_URLS}" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_h200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_h200_vllm_mtp.sh deleted file mode 120000 index 157d224098..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_h200_vllm_mtp.sh +++ /dev/null @@ -1 +0,0 @@ -dsv41flash_fp4_vllm_mtp.sh \ No newline at end of file diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh deleted file mode 100755 index 046a22fbea..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi300x_mtp.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash on MI300X (gfx942): native DSpark, GPU-resident KV. A copy -# of the validated MI355X arm; gfx942 has no FP4 MFMA, so the MXFP4 experts run -# through whichever AITER MoE kernel vLLM's selector supports on this arch, and -# the Engram tables are row-sharded on GPU at TP8 and offloaded to pinned -# host memory below TP8 (192 GB x TP8 holds the 511 GB checkpoint with -# room for KV; TP4 and TP2 do not). -# https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -# AITER's Triton MoE GEMM warns on every call that Gluon (gfx1250-only) is -# unavailable; on gfx950 that was 98% of the server log. Set back to WARNING -# when diagnosing new AITER startup or runtime failures on gfx942. -export AITER_TRITON_LOG_LEVEL=ERROR -# DeepseekV41ForCausalLM is not torch-compiled upstream. On gfx950 the MI355X -# arm runs breakable piecewise graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=1); on -# gfx942 every worker segfaulted during "Capturing CUDA graphs (PIECEWISE)" -# with both the Triton W4A16 and the unfused MoE kernels (runs 35305045778, -# 35306398350), so capture only full decode graphs, the uniform-batch shape -# the ROCm sparse SWA backend supports, and run prefill eagerly as the -# MiniMax-M3 gfx942 arm does. -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export OMP_NUM_THREADS=1 -# Pin the full-context corpus for this 1M-context recipe. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_RUST_FRONTEND=1 -export PYTHONUNBUFFERED=1 - -# Upstream picks 1024 on GPUs with >= 160 GiB, and 2*CONC starves AgentX -# subagent fan-out at low CONC. 128 also keeps CAPTURE_SIZE deterministic. -MAX_NUM_SEQS=128 -NUM_SPEC_TOKENS=5 -CAPTURE_SIZE=1 -while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do - CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) -done - -# vllm-project/vllm#57491 widened the two is_cuda() gates to is_cuda_alike(), so -# this image resolves an Engram config on gfx942 and, because cpu_offload now -# defaults on through VLLM_PLE_CPU_OFFLOAD, an explicit value is required rather -# than the default. Keep the 183 GiB of tables resident at TP8 so that curve -# stays comparable with the validated concurrency 1-32 run; offload below TP8, -# where half or more of the checkpoint already fills a 192 GB card. -if (( TP < 8 )); then - ENGRAM_CONFIG='{"cpu_offload":true}' -else - ENGRAM_CONFIG='{"cpu_offload":false}' -fi - -# The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits -# buffer at startup: 16 GiB at 8192, 8 GiB at 4096. 8192 leaves TP4 ~64 GiB of -# KV per rank after the ~81 GiB of offloaded weights. There is no TP2 arm here: -# run 35671005506 measured "Available KV cache memory: -13.51 GiB" at TP2 even -# with the buffer halved, so a 192 GB card cannot serve this checkpoint at TP2. -BATCHED_TOKENS=8192 - -# Use the runner-specific port assigned by launch_mi300x-amd.sh. -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" -echo "Using vLLM endpoint ${AIPERF_SERVER_URL}" - -# Golden AL: golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Accuracy evals keep real block rejection; throughput fixes acceptance to AL 3.51. -# Adaptive verification stays off in both modes on ROCm: it trims verification -# requests on device, which DeepseekV4IndexerBackend does not support, so the -# engine refuses to start with it enabled. -if [[ "${EVAL_ONLY}" == true ]]; then - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' -else - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}' -fi -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" - --language-model-only - --tokenizer-mode deepseek_v41 - --tool-call-parser deepseek_v41 --enable-auto-tool-choice - --reasoning-parser deepseek_v41 - --engram-config "$ENGRAM_CONFIG" - # aiter: auto selection picked the unfused Triton MoE (TRITON_UNFUSED) on - # gfx942 in run 35306398350 and still segfaulted at piecewise capture, so - # the MoE kernel was not the culprit; keep the upstream recipe's name, - # which resolves to the dedicated Triton MXFP4 W4A16 kernel here. - --moe-backend aiter - --gpu-memory-utilization 0.9 - --speculative-config "$SPEC_CONFIG" - --max-model-len 1048576 - --max-num-seqs "$MAX_NUM_SEQS" - --max-cudagraph-capture-size "$CAPTURE_SIZE" - --max-num-batched-tokens "$BATCHED_TOKENS" - # vllm-project/vllm#56227 turned SWA bounded replay on by default between the - # eed1f3d0 pin and this one. It relies on a window clamp that landed in the - # FlashInfer and FlashMLA kernels; the ROCm sparse SWA path only gained the - # replay_start kwarg, which crashed every gfx950 point with - # HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a replay - # start. gfx942 runs the same ROCm sparse path. Drop this once ROCm clamps - # too; prefix caching stays on. - --no-swa-bounded-replay - --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - --disable-uvicorn-access-log -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -SERVER_PID="" -cleanup_server() { - local rc=$? - trap - EXIT INT TERM - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$rc" -} -trap cleanup_server EXIT -trap 'exit 130' INT -trap 'exit 143' TERM -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi325x_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi325x_mtp.sh deleted file mode 100755 index 495b7e346d..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi325x_mtp.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash on MI325X (gfx942): native DSpark, GPU-resident KV. A copy -# of the validated MI355X arm; gfx942 has no FP4 MFMA, so the MXFP4 experts run -# through whichever AITER MoE kernel vLLM's selector supports on this arch, and -# the Engram tables are row-sharded on GPU at TP8 and offloaded to pinned -# host memory below TP8 (256 GB x TP8 holds the 511 GB checkpoint with -# room for KV; TP4 and TP2 do not). -# https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -# AITER's Triton MoE GEMM warns on every call that Gluon (gfx1250-only) is -# unavailable; on gfx950 that was 98% of the server log. Set back to WARNING -# when diagnosing new AITER startup or runtime failures on gfx942. -export AITER_TRITON_LOG_LEVEL=ERROR -# DeepseekV41ForCausalLM is not torch-compiled upstream. On gfx950 the MI355X -# arm runs breakable piecewise graphs (VLLM_USE_BREAKABLE_CUDAGRAPH=1); on -# gfx942 every worker segfaulted during "Capturing CUDA graphs (PIECEWISE)" -# with both the Triton W4A16 and the unfused MoE kernels (runs 35305045778, -# 35306398350), so capture only full decode graphs, the uniform-batch shape -# the ROCm sparse SWA backend supports, and run prefill eagerly as the -# MiniMax-M3 gfx942 arm does. -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export OMP_NUM_THREADS=1 -# Pin the full-context corpus for this 1M-context recipe. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_RUST_FRONTEND=1 -export PYTHONUNBUFFERED=1 - -# Upstream picks 1024 on GPUs with >= 160 GiB, and 2*CONC starves AgentX -# subagent fan-out at low CONC. 128 also keeps CAPTURE_SIZE deterministic. -MAX_NUM_SEQS=128 -NUM_SPEC_TOKENS=5 -CAPTURE_SIZE=1 -while (( CAPTURE_SIZE < MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do - CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) -done - -# vllm-project/vllm#57491 widened the two is_cuda() gates to is_cuda_alike(), so -# this image resolves an Engram config on gfx942 and, because cpu_offload now -# defaults on through VLLM_PLE_CPU_OFFLOAD, an explicit value is required rather -# than the default. Keep the 183 GiB of tables resident at TP8 so that curve -# stays comparable with the validated concurrency 1-32 run; offload below TP8, -# where half or more of the checkpoint already fills a 256 GB card. -if (( TP < 8 )); then - ENGRAM_CONFIG='{"cpu_offload":true}' -else - ENGRAM_CONFIG='{"cpu_offload":false}' -fi - -# The sparse-attention indexer allocates a [batched-tokens, 1M] fp8 logits -# buffer at startup: 32 GiB at 16384, 16 GiB at 8192, 8 GiB at 4096. TP2 leaves -# ~145 GiB of offloaded weights on every rank, so halve the buffer and cap the -# scheduler there to keep a usable KV pool. Same shape as the Blackwell TP2 -# arms, which validated this cap set on B200, GB200 and GB300. -BATCHED_TOKENS=16384 -if (( TP == 2 )); then - BATCHED_TOKENS=8192 - MAX_NUM_SEQS=$((2 * CONC)) - if (( MAX_NUM_SEQS > 256 )); then MAX_NUM_SEQS=256; fi - if (( MAX_NUM_SEQS < 16 )); then MAX_NUM_SEQS=16; fi - if (( CAPTURE_SIZE > 512 )); then CAPTURE_SIZE=512; fi -fi - -# Use the runner-specific port assigned by launch_mi325x-amds.sh. -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" -echo "Using vLLM endpoint ${AIPERF_SERVER_URL}" - -# Golden AL: golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Accuracy evals keep real block rejection; throughput fixes acceptance to AL 3.51. -# Adaptive verification stays off in both modes on ROCm: it trims verification -# requests on device, which DeepseekV4IndexerBackend does not support, so the -# engine refuses to start with it enabled. -if [[ "${EVAL_ONLY}" == true ]]; then - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' -else - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}' -fi -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" - --language-model-only - --tokenizer-mode deepseek_v41 - --tool-call-parser deepseek_v41 --enable-auto-tool-choice - --reasoning-parser deepseek_v41 - --engram-config "$ENGRAM_CONFIG" - # aiter: auto selection picked the unfused Triton MoE (TRITON_UNFUSED) on - # gfx942 in run 35306398350 and still segfaulted at piecewise capture, so - # the MoE kernel was not the culprit; keep the upstream recipe's name, - # which resolves to the dedicated Triton MXFP4 W4A16 kernel here. - --moe-backend aiter - --gpu-memory-utilization 0.9 - --speculative-config "$SPEC_CONFIG" - --max-model-len 1048576 - --max-num-seqs "$MAX_NUM_SEQS" - --max-cudagraph-capture-size "$CAPTURE_SIZE" - --max-num-batched-tokens "$BATCHED_TOKENS" - # vllm-project/vllm#56227 turned SWA bounded replay on by default between the - # eed1f3d0 pin and this one. It relies on a window clamp that landed in the - # FlashInfer and FlashMLA kernels; the ROCm sparse SWA path only gained the - # replay_start kwarg, which crashed every gfx950 point with - # HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a replay - # start. gfx942 runs the same ROCm sparse path. Drop this once ROCm clamps - # too; prefix caching stays on. - --no-swa-bounded-replay - --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - --disable-uvicorn-access-log -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -SERVER_PID="" -cleanup_server() { - local rc=$? - trap - EXIT INT TERM - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$rc" -} -trap cleanup_server EXIT -trap 'exit 130' INT -trap 'exit 143' TERM -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_sglang_mtp.sh deleted file mode 100644 index d428c1be49..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_sglang_mtp.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash AgentX on MI355X with SGLang DSpark, following the -# cookbook's MI350X cell, with radix caching enabled for AgentX prefix reuse. -# The KV cache and TP4 Engram tables are GPU-resident. -# https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1 -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP EP_SIZE CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY PORT -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# ROCR/HIP visibility under slurm cgroups. -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# A server killed minutes earlier can still be draining HBM (KFD reclaim takes -# minutes), and booting into a half-drained node fails RCCL init with HIP -# 'unhandled cuda error'. Idle GPUs sit at up to ~4% VRAM, draining ones at -# 50-90%, so require every GPU <= 10%. -GPU_CLEAN=false -for i in $(seq 1 90); do - VRAM_MAX=$(rocm-smi --showmemuse 2>/dev/null | grep -oE "GPU Memory Allocated \(VRAM%\): [0-9]+" | awk '{if ($NF > m) m = $NF} END {print m+0}') - if [[ "${VRAM_MAX:-0}" -le 10 ]]; then echo "GPUs clean (vram%max=$VRAM_MAX after $((i*10))s)"; GPU_CLEAN=true; break; fi - echo "waiting for prior-job GPU memory reclaim: vram%max=$VRAM_MAX"; sleep 10 -done -[[ "$GPU_CLEAN" == true ]] || { echo "Error: GPUs still draining prior job's memory after 15min" >&2; exit 1; } - -# Pin the full-context corpus for this 1M-context recipe. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export PYTHONNOUSERSITE=1 -export PYTHONUNBUFFERED=1 - -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL -# curve. SGLang ships thinking off by default for this model. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV41_REASONING_EFFORT=high - -# Use the official model-preview image's native kernels, allocator and tuning -# CSV. TP4 Engram remains GPU-resident, as in the official MI350X recipe. -export SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=0 -export SGLANG_USE_AITER=1 -export SGLANG_MOE_PADDING=1 -export AITER_FLYDSL_FORCE_REDUCE=1 -export ROCM_QUICK_REDUCE_QUANTIZATION=NONE - -# Long-prefill scratch can fill the native allocator cache and starve HIP/RCCL -# allocations outside PyTorch. The preview's allocator only activates GC when -# per_process_memory_fraction is below 1; reclaim unused blocks at 80% of 99%. -export PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,per_process_memory_fraction:0.99 - -CUDA_GRAPH_MAX_BS=64 - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if (( CONC >= 32 )); then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# Use the runner-specific port assigned by launch_mi355x-amds.sh. -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -echo "Using SGLang endpoint ${AIPERF_SERVER_URL}" - -# DSpark is the checkpoint's own bundled draft: no EAGLE/MTP path and no -# --speculative-num-steps knob; the block size is the only tunable. Golden AL: -# golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Throughput fixes acceptance to AL 3.51; accuracy evals keep real verification. -DSPARK_BLOCK_SIZE=5 -DSV41_GOLDEN_AL=3.51 -unset SGLANG_SIMULATE_ACC_LEN SGLANG_SIMULATE_ACC_METHOD SGLANG_SIMULATE_ACC_TOKEN_MODE -if [[ "${EVAL_ONLY}" != true ]]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV41_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi -echo "DSpark block size: $DSPARK_BLOCK_SIZE, golden AL=$DSV41_GOLDEN_AL" - -# Official MI350X TP4/EP4 recipe adapted for AgentX radix prefix reuse. -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" - --trust-remote-code - --tp "$TP" --ep-size "$EP_SIZE" - # C32 still exhausted HBM at 0.80 with 4096-token chunks; reserve - # another 10% of physical HBM for native long-prefill scratch. - --mem-fraction-static 0.70 - # Native FP4 prefill scratch scales with query tokens times context. - # The 16384-token default OOMed five canonical cells; match the existing - # 4096-token prefill bound without reducing model context. - --chunked-prefill-size 4096 - # The native preview's compressed-KV store computes byte offsets in signed - # int32. Keep ratio-1 pages below 2 GiB without reducing the 1M context limit. - --max-total-tokens 3145728 - # AgentX C32 fanout reached 63 running requests before a native HIP illegal - # access. Bound admission; additional client requests remain queued. - --max-running-requests 32 - --speculative-algorithm DSPARK - --speculative-dspark-block-size "$DSPARK_BLOCK_SIZE" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - # Native breakable prefill still hit HIP illegal access with admission32 - # and ample measured HBM headroom. Test eager prefill; retain decode graphs. - --cuda-graph-backend-prefill disabled - --reasoning-parser auto - --tool-call-parser auto - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800 s default watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_|^PYTORCH_(HIP|CUDA|ALLOC)' | sort - echo "===================================" -} | tee "$SERVER_LOG" -SERVER_PID="" -cleanup_server() { - local rc=$? - trap - EXIT INT TERM - stop_background_process_tree "$SERVER_PID" "SGLang server" 60 - exit "$rc" -} -trap cleanup_server EXIT -trap 'exit 130' INT -trap 'exit 143' TERM -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics ${AIPERF_SERVER_METRICS_URLS}" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh deleted file mode 100755 index e53578124c..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_mi355x_vllm_mtp.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# DeepSeek-V4.1-Flash on MI355X: native DSpark, GPU-resident KV. -# https://github.com/vllm-project/recipes/blob/main/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -# AITER's Triton MoE GEMM warns on every call that Gluon (gfx1250-only) is -# unavailable; on gfx950 that was 98% of the server log. Set back to WARNING -# when diagnosing new AITER startup or runtime failures. -export AITER_TRITON_LOG_LEVEL=ERROR -# DeepseekV41ForCausalLM is not torch-compiled upstream, so the default -# cudagraph_mode=FULL_AND_PIECEWISE aborts at engine init ("piecewise CUDA -# graphs unavailable"); amd/attention.py uses eager_break_during_capture. -export VLLM_USE_BREAKABLE_CUDAGRAPH=1 -export OMP_NUM_THREADS=1 -# Pin the full-context corpus for this 1M-context recipe. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_RUST_FRONTEND=1 -export PYTHONUNBUFFERED=1 - -# vllm-project/vllm#57491 widened the two is_cuda() gates to is_cuda_alike(), so -# on gfx950 this image resolves an Engram config and an explicit value is needed -# rather than the VLLM_PLE_CPU_OFFLOAD default. -# -# The tables cost 47.2 GiB per rank at TP=4, so 94.4 GiB at TP=2, which does not -# fit beside half of the 511 GB checkpoint on a 288 GiB card: TP=2 always -# offloads. TP=4 keeps them resident, as the validated concurrency 1-32 run -# measured, because offloaded lookups go to pinned host memory over UVA and -# nothing below c128 is short of KV. Resident leaves 14.83M KV tokens, which is -# 232K per request at c64 and healthy, but only 116K at c128, under the 122K at -# which TP=2 c64 collapsed. Offloading lifts it to 33.06M, so 258K at c128. -if (( TP == 2 || CONC >= 128 )); then - ENGRAM_CONFIG='{"cpu_offload":true}' -else - ENGRAM_CONFIG='{"cpu_offload":false}' -fi - -# Graph capture covers twice the outer concurrency, floored at the #3058 size of -# 128 sequences, across the 1+5 DSpark token shape. Twice leaves headroom for -# AgentX subagent fan-out above the outer concurrency. -NUM_SPEC_TOKENS=5 -GRAPH_NUM_SEQS=$((2 * CONC)) -if (( GRAPH_NUM_SEQS < 128 )); then - GRAPH_NUM_SEQS=128 -fi -CAPTURE_SIZE=1 -while (( CAPTURE_SIZE < GRAPH_NUM_SEQS * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do - CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) -done - -# The sparse-attention indexer and its companion per-rank buffers scale with -# --max-num-batched-tokens at roughly 4.4 MiB per token, measured on gfx950, so -# a smaller prefill chunk buys KV room. TP=2 starts from half the per-rank space -# and is the arm that runs short: at the upstream 16384 it holds 7.84M KV -# tokens, 122K per request at c64, where run 35574132719 fell to a 17.6% prefix -# cache hit rate, 187 s TTFT and 150 tok/s against 94.8%, 1.3 s and 957 tok/s at -# c32. Every point that held had 232K per request or more, so keep the upstream -# chunk through c32 (245K at TP=2) and trade it away only above that. B300 runs -# 8192 at TP=4 and the Blackwell TP=2 arms run 4096 (#3320, #3321). -if (( CONC >= 128 )); then - (( TP == 2 )) && BATCHED_TOKENS=4096 || BATCHED_TOKENS=8192 -elif (( TP == 2 && CONC >= 64 )); then - BATCHED_TOKENS=8192 -else - BATCHED_TOKENS=16384 -fi - -# DSpark verifies 1+5 tokens per sequence, so a decode batch of max_num_seqs -# needs six times that many token slots. The MI355X API-server default of 1024 -# sequences therefore wants 6144, and below that the engram projection faults -# during profiling: TP=2 c128 at 4096 leaves four slots per sequence and dies -# with HSA_STATUS_ERROR_EXCEPTION at M=1024, N=129280, K=256, reproduced on two -# separate GPU pairs. Where the chunk is that small, cap in-flight sequences at -# the shape graph capture already covers, which also keeps the largest decode -# batch on a captured graph. Leave the default alone everywhere else. -DEFAULT_MAX_NUM_SEQS=1024 -MAX_NUM_SEQS="" -if (( BATCHED_TOKENS < DEFAULT_MAX_NUM_SEQS * (1 + NUM_SPEC_TOKENS) )); then - MAX_NUM_SEQS="$GRAPH_NUM_SEQS" -fi - -# Use the runner-specific port assigned by launch_mi355x-amds.sh. -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" -echo "Using vLLM endpoint ${AIPERF_SERVER_URL}" - -# Golden AL: golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Accuracy evals keep real block rejection; throughput fixes acceptance to AL 3.51. -# Adaptive verification stays off in both modes on ROCm: it trims verification -# requests on device, which DeepseekV4IndexerBackend does not support, so the -# engine refuses to start with it enabled. -if [[ "${EVAL_ONLY}" == true ]]; then - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":false}' -else - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}' -fi -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" - --language-model-only - --tokenizer-mode deepseek_v41 - --tool-call-parser deepseek_v41 --enable-auto-tool-choice - --reasoning-parser deepseek_v41 - --engram-config "$ENGRAM_CONFIG" - # aiter, not aiter_triton_mxfp4_bf16: the plain name opens vLLM's full - # priority list and the CK kernel at its head wins. CK quantizes - # activations to FP8 internally and dispatches the a8w4 experts - # (mfma_moe1_silu_mul_afp8_wfp4_bf16 / mfma_moe2_afp8_wfp4_bf16); the - # Triton name forces the W4A16 _moe_gemm_a16w4 kernel instead. - --moe-backend aiter - --gpu-memory-utilization 0.9 - --speculative-config "$SPEC_CONFIG" - --max-model-len 1048576 - --max-cudagraph-capture-size "$CAPTURE_SIZE" - --max-num-batched-tokens "$BATCHED_TOKENS" - # vllm-project/vllm#56227 added SWA bounded replay (default on) after the - # eed1f3d0 pin and before this one. It pads the replayed tokens' slots in the - # prefix-cacheable groups, but the window clamp it relies on landed in the - # FlashInfer and FlashMLA kernels; the ROCm sparse SWA path only gained the - # replay_start kwarg. On gfx950 every TP=2 and TP=4 point of run 35567570539 - # died with HSA_STATUS_ERROR_MEMORY_FAULT at the first prefix hit carrying a - # replay start. Drop this once ROCm clamps too; prefix caching stays on. - --no-swa-bounded-replay - --disable-uvicorn-access-log -) -if [[ -n "$MAX_NUM_SEQS" ]]; then - VLLM_CMD+=(--max-num-seqs "$MAX_NUM_SEQS") -fi -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -SERVER_PID="" -cleanup_server() { - local rc=$? - trap - EXIT INT TERM - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$rc" -} -trap cleanup_server EXIT -trap 'exit 130' INT -trap 'exit 143' TERM -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh deleted file mode 100644 index 0a0a326295..0000000000 --- a/benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# Native DeepSeek-V4.1-Flash DSpark and Engram UVA weight offload. -# https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4.1-Flash -source "$(dirname "$0")/../../benchmark_lib.sh" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE EVAL_ONLY VLLM_ENGINE_READY_TIMEOUT_S -require_agentic_kv_offload_none -export GPU_COUNT="$TP" - -# Complete/resume partial downloads instead of trusting nonempty directories. -if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -mkdir -p "$RESULT_DIR" -SERVER_LOG="$RESULT_DIR/server.log" -export VLLM_USE_RUST_FRONTEND=1 -export PYTHONUNBUFFERED=1 - -# Safetensors load strategy. vLLM auto-prefetches checkpoints only on NFS or -# Lustre; on other filesystems it memory-maps lazily. On cluster:h200-dgxc the -# HF cache is a VIRTIOFS mount, and on nightly-cd10ed6f the lazy path read this -# 475 GiB checkpoint at ~170 s/shard (19/48 shards when the 3600 s readiness -# deadline fired, run 35012494184) against ~12.5 s/shard for the same files on -# the deepseekv41-flash-0909 build (run 34504985992). Launchers whose cache is -# not a recognized network FS export VLLM_SAFETENSORS_LOAD_STRATEGY=prefetch so -# the shards are streamed into page cache by parallel readers first; the -# checkpoint fits comfortably in the ~1 TiB of host RAM those nodes expose. -LOAD_ARGS=() -if [[ -n "${VLLM_SAFETENSORS_LOAD_STRATEGY:-}" ]]; then - LOAD_ARGS=(--safetensors-load-strategy "$VLLM_SAFETENSORS_LOAD_STRATEGY") -fi - -# Preserve the upstream scheduler defaults; size graph capture for the sweep. -NUM_SPEC_TOKENS=5 -CAPTURE_SIZE="${DSV41_MIN_CUDAGRAPH_CAPTURE_SIZE}" -while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do - CAPTURE_SIZE=$((CAPTURE_SIZE * 2)) -done - -# Low-TP arms need the scheduler capped. Two cases reach the same limits: -# H200 TP4 leaves ~15 GiB of KV per 141 GB GPU, and TP2 on Blackwell leaves -# ~145 GiB of weights per rank (B200 180 GB, GB200 256 GB, GB300 277 GB). In -# both, the sparse-attention indexer's [batched-tokens, 1M] fp8 logits buffer -# is 32 GiB at the upstream 16384 batched tokens, and graph capture above 512 -# pushed the B200 TP2 KV budget to -10.8 GiB (run 35180394796: c1-c16 served, -# c32/c64/c128 died in memory profiling) and GB200 c128 to -5.34 GiB -# (run 35528745995). Cap batched tokens at 4096 (8 GiB, as the H100 arm does), -# bound the scheduler batch to the AgentX fan-out, and stop capturing above -# 512 tokens. B200/GB200 TP4 (180+ GB) and every TP8 arm keep the defaults. -GPU_MEM_MIB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits | head -n1 | tr -d ' ') -LOW_TP_ARGS=() -if (( TP == 2 || (TP < 8 && GPU_MEM_MIB < 150000) )); then - MAX_NUM_SEQS=$((2 * CONC)) - if (( MAX_NUM_SEQS > 256 )); then - MAX_NUM_SEQS=256 - fi - # FlashInfer's autotune dummy run batches max-num-seqs requests through - # the DSpark draft head; with 2-8 requests on TP2 it selected an invalid - # MXFP8 split-K tactic ((128, 8), (1, 1), True, False, 4) and the engine - # never started (run 35320655804: c1/c2/c4 failed, c8 with 16 seqs and - # every larger point served). 16 is the smallest value that has passed. - if (( MAX_NUM_SEQS < 16 )); then - MAX_NUM_SEQS=16 - fi - if (( CAPTURE_SIZE > 512 )); then - CAPTURE_SIZE=512 - fi - LOW_TP_ARGS=(--max-num-batched-tokens 4096 --max-num-seqs "$MAX_NUM_SEQS") -fi - -# Pyxis shares the host network; port 8888 can already belong to a host service. -select_available_server_port -export AIPERF_SERVER_URL="http://localhost:${PORT}" -export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" -echo "Using vLLM endpoint ${AIPERF_SERVER_URL}" - -# Golden AL: golden_al_distribution/dsv41flash_dspark.yaml, thinking_on, five draft tokens. -# Accuracy evals keep real block rejection; throughput fixes acceptance to AL 3.51. -if [[ "${EVAL_ONLY}" == true ]]; then - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"block","enable_adaptive_verification":true}' -else - SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}' -fi -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP" - --language-model-only - --tokenizer-mode deepseek_v41 - --tool-call-parser deepseek_v41 --enable-auto-tool-choice - --reasoning-parser deepseek_v41 - --engram-config '{"cpu_offload":true}' - --speculative-config "$SPEC_CONFIG" - --max-model-len 1048576 - --max-cudagraph-capture-size "$CAPTURE_SIZE" - "${LOW_TP_ARGS[@]}" - --disable-uvicorn-access-log - "${LOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == true ]]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh deleted file mode 100755 index a70526fdc9..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_sglang_mtp.sh +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# DeepSeek-V4-Pro-0813 FP4 on B200 with SGLang DSpark K=6. -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" --validation-only -check_env_vars INFMAX_CONTAINER_WORKSPACE RESULT_DIR - -# The B200 DeepSeek-V4 image installs SGLang editable under /workspace, so its -# launcher mounts InferenceX at /ix. Resolve tooling and results against the -# actual repository mount. -if [[ "${RESULT_DIR:-}" == /workspace/* && "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - export RESULT_DIR="$INFMAX_CONTAINER_WORKSPACE/${RESULT_DIR#/workspace/}" -fi -source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" - -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source - -# AIPerf's Transformers-main dependency would replace the Transformers build -# pinned by the B200 SGLang image; the server keeps the image interpreter and -# AIPerf runs from an isolated venv when InferenceX is mounted at /ix. -SGLANG_PYTHON="$(command -v python3)" -if [[ "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - AGENTIC_VENV="/tmp/inferencex-agentic-venv" - "$SGLANG_PYTHON" -m venv "$AGENTIC_VENV" - export PATH="$AGENTIC_VENV/bin:$PATH" -fi -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 -export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # DeepSeek V4 HiCache currently rejects --hicache-size and supports - # DeepSeek V4 HiCache rejects --hicache-size; capacity is a host/device - # token ratio. DEP8 shards the host pools and fits ratio=8; replicated TP8 - # pools need 2.75 (~121 GiB/rank) to leave startup headroom on 1.7 TiB hosts. - HICACHE_RATIO=2.75 - if [ "$DP_ATTENTION" = "true" ]; then - HICACHE_RATIO=8 - fi - HICACHE_WRITE_POLICY="write_through" - HICACHE_IO_BACKEND="direct" - HICACHE_MEM_LAYOUT="page_first_direct" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" -fi - -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - ROUTER_POLICY_ARGS=() - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - SGLANG_ROUTER_CMD=("$SGLANG_PYTHON" -m sglang_router.launch_router) -fi - -PARALLEL_ARGS=(--tp "$TP") -METRICS_ARGS=(--enable-metrics --enable-cache-report) -CHUNKED_PREFILL_SIZE=8192 -SWA_FULL_TOKENS_RATIO=0.1 -MEM_FRACTION_STATIC=0.90 -if [ "$DP_ATTENTION" = "true" ]; then - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 - - # Leave HBM headroom for the FP4 indexer's context-dependent workspace. - MEM_FRACTION_STATIC=0.88 - PREFILL_DECODE_INTERVAL=24 - - # Keep DP admission and session routing uniform across the DEP8 curve. - PARALLEL_ARGS+=(--load-balance-method total_requests) - METRICS_ARGS+=(--load-snapshot-publish-interval 1) - export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=true - if [ "$CONC" -eq 160 ]; then - PREFILL_DECODE_INTERVAL=20 - ROUTER_POLICY_ARGS+=(--balance-abs-threshold 32) - fi - - PARALLEL_ARGS+=( - --dp "$TP" - --tokenizer-worker-num "$TP" - --prefill-decode-interval "$PREFILL_DECODE_INTERVAL" - --enable-dp-attention - --enable-dp-lm-head - --enable-dp-attention-local-control-broadcast - --incremental-streaming-output - --stream-interval 20 - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - --ep-size "$EP_SIZE" - --moe-a2a-backend megamoe - --enable-w4a4-mxfp4-megamoe - --enable-deepseek-v4-fp4-indexer - --disable-shared-experts-fusion - --disable-flashinfer-autotune - ) - # SGLang divides this global budget by dp_size. Keep 6144 tokens per rank - # for every DP-attention profile so the FP4 indexer retains HBM headroom. - CHUNKED_PREFILL_SIZE=$((6144 * TP)) - SWA_FULL_TOKENS_RATIO=0.02 -else - PARALLEL_ARGS+=( - --moe-runner-backend flashinfer_mxfp4 - --enable-deepseek-v4-fp4-indexer - --disable-flashinfer-autotune - ) -fi - -# The B300 compressed-attention/page-size overrides deadlock this image right -# after weight loading, so they are not passed here. - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$((2 * CONC)) -if [ "$DP_ATTENTION" = "true" ]; then - CUDA_GRAPH_MAX_BS=32 -fi -# SGLang v0.5.20 removed the deprecated --cuda-graph-max-bs alias -# (sgl-project/sglang#38375); --cuda-graph-max-bs-decode is its replacement. -CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS") - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 -export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 -export SGLANG_OPT_USE_JIT_NORM=1 -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 -export SGLANG_OPT_USE_TOPK_V2=1 -export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.77 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi -TRITON_PTXAS_PATH=$(find \ - /usr/local/cuda* \ - /usr/local/lib/python*/dist-packages/nvidia \ - /usr/local/lib/python*/site-packages/nvidia \ - -type f -name ptxas -perm -u+x -print -quit 2>/dev/null || true) -if [ -n "$TRITON_PTXAS_PATH" ]; then - export TRITON_PTXAS_PATH - echo "Using ptxas for Triton: $TRITON_PTXAS_PATH" -fi -SGLANG_CMD=( - "$SGLANG_PYTHON" -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" - --max-running-requests "$MAX_RUNNING_REQUESTS" - "${CUDA_GRAPH_ARGS[@]}" - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --tool-call-parser deepseekv4 - --reasoning-parser deepseek-v4 - --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" - --watchdog-timeout 1800 - --speculative-algorithm DSPARK - --speculative-dspark-block-size 6 - --speculative-num-steps 1 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 7 - # The B200 checkpoint lives on Lustre: prefetch sequentially across local - # ranks so post-load repacking reads from page cache instead of every rank - # issuing fragmented mmap faults. - --weight-loader-prefetch-checkpoints - --model-loader-extra-config '{"enable_multithread_load": true}' - "${METRICS_ARGS[@]}" - "${CACHE_ARGS[@]}" -) - -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" - -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B200..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_ready \ - --endpoint "http://localhost:$SGLANG_BACKEND_PORT/health" \ - --log "$SERVER_LOG" \ - --pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - "${SGLANG_ROUTER_CMD[@]}" \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy cache_aware \ - "${ROUTER_POLICY_ARGS[@]}" \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_ready \ - --endpoint "http://localhost:$PORT/health" \ - --log "$ROUTER_LOG" \ - --pid "$ROUTER_PID" -fi - -if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then - capture_cache_metrics - trap capture_cache_metrics EXIT -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh deleted file mode 100755 index 8a9fe5fc9d..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh +++ /dev/null @@ -1,325 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# DeepSeek-V4-Pro-0813 FP4 on B200 with vLLM DSpark (num_speculative_tokens=6). -# Throughput fixes synthetic acceptance to AL 3.77; EVAL_ONLY keeps real -# verification. Cudagraph capture sizes are in tokens (see the capture block). -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# DEP8 offloads KV to host DRAM with KV_OFFLOAD_BACKEND=vllm-simple. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars DCP_SIZE EVAL_ONLY PCP_SIZE - -VLLM_CP_ARGS=() -if [ "$DCP_SIZE" -gt 1 ]; then - VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") -fi -if [ "$PCP_SIZE" -gt 1 ]; then - VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") -fi - -GPU_COUNT=$((TP * PCP_SIZE)) -if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then - echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 - exit 1 -fi -export GPU_COUNT - -# Under DP-attention the DP world size equals TP, and the scheduler's global -# running-sequence budget is 2*CONC. Split that budget evenly across DP ranks. -if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then - echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2 - exit 1 -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -# vllm-router expands one HTTP backend into a logical worker per DP rank. -# AIPerf's X-Correlation-ID is stable across a conversation's turns; alias it -# to the router's X-Session-ID so every turn lands on the same rank. -USE_VLLM_ROUTER=false -VLLM_BACKEND_PORT="$PORT" -if [ "$DP_ATTENTION" = "true" ]; then - USE_VLLM_ROUTER=true - VLLM_BACKEND_PORT=$((PORT + 1)) - VLLM_ROUTER_VERSION=0.1.14 - VLLM_ROUTER_POLICY=consistent_hash - VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" -fi - -# AIPerf scrapes the public endpoint's /metrics, which is the router under -# DP-attention; add the engine endpoint explicitly (deduplicated for pure TP). -export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -export VLLM_ENGINE_READY_TIMEOUT_S=7200 - -# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while -# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's -# store mask. 32k matches the trace-replay tuning validated for this workload. -export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 -export VLLM_USE_V2_MODEL_RUNNER=1 -export VLLM_USE_RUST_FRONTEND=1 -export VLLM_DSV4_MEGA_FP8_COMBINE=1 -export VLLM_RPC_TIMEOUT=600000 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -ROUTER_PID="" -MOONCAKE_MASTER_PID="" - -OFFLOAD_ARGS=() -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - vllm-simple) - require_agentic_kv_offload_backend vllm-simple - CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) - # Identical prefixes must hash to identical block keys across DP ranks. - export PYTHONHASHSEED=42 - OFFLOAD_CONFIG=$(cat </dev/null - - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then - echo "Mooncake master died during startup." >&2 - cat "$MOONCAKE_MASTER_LOG" >&2 - exit 1 - fi - unset VLLM_USE_SIMPLE_KV_OFFLOAD - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) - ;; - *) - echo "Error: unsupported B200 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2 - exit 1 - ;; -esac - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -MODE_ARGS=() -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") - export PYTORCH_ALLOC_CONF=expandable_segments:True -fi - -if [ "$EP_SIZE" -gt 1 ]; then - MODE_ARGS+=( - --enable-expert-parallel - --enable-ep-weight-filter - --moe-backend deep_gemm_amxf4_mega_moe - ) -fi -if [ "$DP_ATTENTION" = "true" ]; then - DEP_KV_CACHE_BYTES=37580963840 - if [ "$TP" -eq 8 ] && { [ "$CONC" -eq 160 ] || [ "$CONC" -eq 192 ]; }; then - DEP_KV_CACHE_BYTES=36507222016 - fi - MODE_ARGS+=( - --prefill-schedule-interval 16 - --long-prefill-token-threshold 512 - --kv-cache-memory-bytes "$DEP_KV_CACHE_BYTES" - ) -fi - -# AgentX concurrency counts live session trees; subagent fan-out pushes the -# global request count above CONC, so keep 2x headroom split across DEP ranks. -if [ "$DP_ATTENTION" = "true" ]; then - MAX_NUM_SEQS=$((2 * CONC / TP)) -else - MAX_NUM_SEQS=$CONC -fi - -# Cudagraph capture sizes are in tokens: a decode batch of S seqs verifies -# S*(1+N) tokens, so capture the multiples (1+N)..MAX_NUM_SEQS*(1+N). vLLM -# rounds sizes up to multiples of (1+N) and dedups, so a plain 1..MAX_NUM_SEQS -# list would cover only MAX_NUM_SEQS/(1+N) sequences. -NUM_SPEC_TOKENS=6 -TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) -# Golden AL: golden_al_distribution/dsv4-pro-0813-dspark.yaml, thinking_on, -# probabilistic drafting, 6 draft tokens. EVAL_ONLY keeps real verification. -if [ "$EVAL_ONLY" = "true" ]; then - SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\"}" -else - SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 3.77}" -fi -# Reserve verification slots in addition to DEP's 8192-token prefill budget. -if [ "$DP_ATTENTION" = "true" ]; then - MAX_NUM_BATCHED_TOKENS=$((8192 + MAX_NUM_SEQS * TOKENS_PER_SEQ)) - MODE_ARGS+=(--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS") -else - MODE_ARGS+=(--max-num-batched-tokens 16384 --kv-cache-memory-bytes 53687091200) -fi -CAPTURE_SIZE_LIST=() -for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do - CAPTURE_SIZE_LIST+=("$((num_seqs * TOKENS_PER_SEQ))") -done -# TP also captures piecewise graphs for mixed prefill/decode batches, which -# needs the compiled graph (no "mode":0). The lists overlap past 100, so dedup. -if [ "$DP_ATTENTION" != "true" ]; then - CAPTURE_SIZE_LIST+=(100 200 300 400 500) -fi -CUDA_GRAPH_CAPTURE_SIZES=$(printf '%s\n' "${CAPTURE_SIZE_LIST[@]}" | sort -n -u | paste -sd, -) -if [ "$DP_ATTENTION" != "true" ]; then - COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}]}" -else - COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}" -fi - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_FLOAT32_MATMUL_PRECISION=high -GPU_MEMORY_UTILIZATION="0.90" - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$VLLM_BACKEND_PORT" - --trust-remote-code - --kv-cache-dtype fp8 - --block-size 256 - --max-model-len 1048576 - --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --numa-bind - --enable-cumem-allocator - --no-enable-flashinfer-autotune - --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --enable-auto-tool-choice - --reasoning-parser deepseek_v4 - --attention-config '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"indexer_kv_dtype":"mxfp4"}' - --speculative-config "$SPEC_CONFIG" - --no-disable-hybrid-kv-cache-manager - --disable-uvicorn-access-log - --compilation-config "$COMPILATION_CONFIG" - --max-num-seqs "$MAX_NUM_SEQS" - "${PARALLEL_ARGS[@]}" - "${VLLM_CP_ARGS[@]}" - "${MODE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_VLLM_ROUTER" = "true" ]; then - echo "Starting native vLLM router on port $PORT for $TP DP ranks..." - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy "$VLLM_ROUTER_POLICY" \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh deleted file mode 100755 index 7adf2c573b..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_sglang_mtp.sh +++ /dev/null @@ -1,310 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# DeepSeek-V4-Pro-0813 FP4 on B300 with SGLang DSpark K=6. -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -INFERENCEX_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" -source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" --validation-only -check_env_vars INFMAX_CONTAINER_WORKSPACE RESULT_DIR - -# The B200 DeepSeek-V4 image installs SGLang editable under /workspace, so its -# launcher mounts InferenceX at /ix. Resolve tooling and results against the -# actual repository mount. -if [[ "${RESULT_DIR:-}" == /workspace/* && "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - export RESULT_DIR="$INFMAX_CONTAINER_WORKSPACE/${RESULT_DIR#/workspace/}" -fi -source "$INFERENCEX_ROOT/benchmarks/benchmark_lib.sh" - -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source - -# AIPerf's Transformers-main dependency would replace the Transformers build -# pinned by the B200 SGLang image; the server keeps the image interpreter and -# AIPerf runs from an isolated venv when InferenceX is mounted at /ix. -SGLANG_PYTHON="$(command -v python3)" -if [[ "$INFMAX_CONTAINER_WORKSPACE" != /workspace ]]; then - AGENTIC_VENV="/tmp/inferencex-agentic-venv" - "$SGLANG_PYTHON" -m venv "$AGENTIC_VENV" - export PATH="$AGENTIC_VENV/bin:$PATH" -fi -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 -export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 - -CACHE_ARGS=() -WARMUP_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # DeepSeek V4 HiCache rejects --hicache-size; capacity is a host/device - # token ratio, and host bytes scale with ratio AND mem-fraction-static. - # TP8 ratio=4 at 0.93 left 5.84 GB free on a 2,964 GB node and the paged - # pool failed to allocate; ratio=3 keeps the tier near 2 TB with room for - # the paged pool, page cache, AIPerf and the router. - if [ "$TP" -ge 8 ]; then - HICACHE_RATIO=3 - else - HICACHE_RATIO=8 - fi - HICACHE_WRITE_POLICY="write_back" - HICACHE_IO_BACKEND="direct" - HICACHE_MEM_LAYOUT="page_first_direct" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - # AIPerf owns the AgentX warmup; SGLang's per-DP warmup can time out after - # the API is already healthy. - WARMUP_ARGS=(--skip-server-warmup) - echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" -fi - -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - SGLANG_ROUTER_CMD=("$SGLANG_PYTHON" -m sglang_router.launch_router) -fi - -PARALLEL_ARGS=(--tp "$TP") -METRICS_ARGS=(--enable-metrics --enable-cache-report) -MEM_FRACTION_STATIC=0.88 -CHUNKED_PREFILL_SIZE=8192 -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS+=( - --dp "$TP" - --tokenizer-worker-num "$TP" - --enable-prefill-delayer - --prefill-decode-interval 20 - --enable-dp-attention - --enable-dp-lm-head - --enable-dp-attention-local-control-broadcast - --incremental-streaming-output - --stream-interval 20 - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - --ep-size "$EP_SIZE" - --moe-a2a-backend megamoe - --enable-w4a4-mxfp4-megamoe - --enable-deepseek-v4-fp4-indexer - --disable-flashinfer-autotune - ) - if [ "$TP" -ge 8 ]; then - # Mega-MoE's transient workspace lives outside the static allocation - # and needs one ~7 GB contiguous block, so headroom grows with - # concurrency. At conc 256, 0.835 runs; 0.93 and 0.95 OOM one DP rank - # and hang the engine in the MLP-sync collective. - MEM_FRACTION_STATIC=0.93 - if [ "$CONC" -ge 512 ]; then - MEM_FRACTION_STATIC=0.86 - elif [ "$CONC" -ge 384 ]; then - MEM_FRACTION_STATIC=0.88 - elif [ "$CONC" -ge 32 ]; then - MEM_FRACTION_STATIC=0.90 - fi - else - # DEP4 weights take ~90% of each GPU, so the engine refuses to start - # below ~0.902, while megamoe still needs its ~7 GB workspace above - # the static budget; 0.93 leaves ~16 GB for it. - MEM_FRACTION_STATIC=0.93 - fi - # --chunked-prefill-size is a global budget divided by dp_size (=TP). - # Scale it so every DEP shape gets 8192 per rank; 16384/rank exceeds - # MegaMoE's per-rank token cap (startup ValueError). - CHUNKED_PREFILL_SIZE=$((8192 * TP)) -else - PARALLEL_ARGS+=( - --moe-runner-backend flashinfer_mxfp4 - --disable-flashinfer-autotune - ) -fi - -MODEL_ARGS=( - --attention-backend compressed - --page-size 256 - --disable-shared-experts-fusion -) - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -# Live requests exceed CONC under fan-out, so graphs sized at CONC would drop -# larger batches to eager decode; the runtime clamps to the request pool anyway. -CUDA_GRAPH_MAX_BS=$((CONC * 4)) -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -# --cuda-graph-max-bs is an alias whose dest is cuda_graph_max_bs_decode, so the -# two forms below are the same knob and must not both be passed. -CUDA_GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") -SWA_FULL_TOKENS_RATIO=0.1 -if [ "$DP_ATTENTION" = "true" ]; then - # Decode graphs must cover the padded speculative batch across all DP ranks, which - # exceeds CONC; capping at 64 would fall back to eager decode. - CUDA_GRAPH_ARGS=(--cuda-graph-max-bs-decode 544) - SWA_FULL_TOKENS_RATIO=0.075 -fi - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Agentic warmup dispatches hundreds of large prompts at once and SGLang's -# tokenizer can leave bytes unacknowledged past AIPerf's default 30 s -# TCP_USER_TIMEOUT, so Linux aborts live localhost connections. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# Outlast AIPerf's pooled connections so an inter-turn idle gap cannot race -# Uvicorn's five-second keep-alive closure. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 -export SGLANG_JIT_DEEPGEMM_FAST_WARMUP=1 -export SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT=1 -export SGLANG_OPT_USE_JIT_NORM=1 -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=1 -export SGLANG_OPT_USE_TOPK_V2=1 -export SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2=1 -if [ "$DP_ATTENTION" = "true" ]; then - # Must cover the per-rank prefill budget (8192) or startup raises; the - # extra 128 is headroom over the exact-fit boundary. - export SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320 -fi -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.77 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi -TRITON_PTXAS_PATH=$(find \ - /usr/local/cuda* \ - /usr/local/lib/python*/dist-packages/nvidia \ - /usr/local/lib/python*/site-packages/nvidia \ - -type f -name ptxas -perm -u+x -print -quit 2>/dev/null || true) -if [ -n "$TRITON_PTXAS_PATH" ]; then - export TRITON_PTXAS_PATH - echo "Using ptxas for Triton: $TRITON_PTXAS_PATH" -fi -SGLANG_CMD=( - "$SGLANG_PYTHON" -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" - --max-running-requests "$MAX_RUNNING_REQUESTS" - "${CUDA_GRAPH_ARGS[@]}" - --allow-auto-truncate - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --tool-call-parser deepseekv4 - --reasoning-parser deepseek-v4 - --chat-template "$SCRIPT_DIR/../chat_templates/deepseek_v4_thinking.jinja" - --watchdog-timeout 1800 - --speculative-algorithm DSPARK - --speculative-dspark-block-size 6 - --speculative-num-steps 1 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 7 - "${MODEL_ARGS[@]}" - "${METRICS_ARGS[@]}" - "${CACHE_ARGS[@]}" - "${WARMUP_ARGS[@]}" -) - -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" - -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B300..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$SGLANG_BACKEND_PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_ready \ - --endpoint "http://localhost:$SGLANG_BACKEND_PORT/health" \ - --log "$SERVER_LOG" \ - --pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - "${SGLANG_ROUTER_CMD[@]}" \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - `# A single transient router->engine send failure would otherwise` \ - `# surface as a 500, and AgentX aborts the whole run when a root` \ - `# warmup request fails ("ProfileAborted"). Measured at conc 512:` \ - `# 22 such transients in one 3600s run, spread over all 8 DP` \ - `# workers, every one of them recovered by the retry; with retries` \ - `# disabled a single one killed a 2h15m arm.` \ - --retry-max-retries 8 \ - --retry-initial-backoff-ms 500 \ - --retry-max-backoff-ms 10000 \ - --retry-backoff-multiplier 2 > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_ready \ - --endpoint "http://localhost:$PORT/health" \ - --log "$ROUTER_LOG" \ - --pid "$ROUTER_PID" -fi - -if [ "${#METRICS_ARGS[@]}" -gt 0 ]; then - capture_cache_metrics - trap capture_cache_metrics EXIT -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - git config --global --add safe.directory "$INFMAX_CONTAINER_WORKSPACE" - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh deleted file mode 100755 index 89cb781460..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# DeepSeek-V4-Pro FP4 on B300 with vLLM MTP (num_speculative_tokens=3). -# Throughput fixes synthetic acceptance to AL 2.49; EVAL_ONLY keeps real -# verification. Cudagraph capture sizes are in tokens (see the capture block). -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# TP8 and TP4 c8 are GPU-resident. TP4 c16, DEP4, and DEP8 use DRAM offload -# with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars DCP_SIZE EVAL_ONLY PCP_SIZE - -VLLM_CP_ARGS=() -if [ "$DCP_SIZE" -gt 1 ]; then - VLLM_CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE") -fi -if [ "$PCP_SIZE" -gt 1 ]; then - VLLM_CP_ARGS+=(--prefill-context-parallel-size "$PCP_SIZE") -fi - -GPU_COUNT=$((TP * PCP_SIZE)) -if [[ ! "$GPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then - echo "Error: GPU_COUNT must be a positive integer, got '$GPU_COUNT'" >&2 - exit 1 -fi -export GPU_COUNT - -# Under DP-attention the DP world size equals TP, and the DEP recipe sizes -# per-rank batch as MAX_NUM_SEQS = 2*CONC/TP, which must be an integer. -if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then - echo "Error: DEP requires 2*CONC divisible by TP, got CONC='$CONC' and TP='$TP'" >&2 - exit 1 -fi - -# DEP8 (TP8 + DP-attention) gets a larger prefill token budget and lower -# GPU-memory headroom than DEP4. -IS_DEP8=false -if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then - IS_DEP8=true -fi - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -# vllm-router expands one HTTP backend into a logical worker per DP rank. -# AIPerf's X-Correlation-ID is stable across a conversation's turns; alias it -# to the router's X-Session-ID so every turn lands on the same rank. -USE_VLLM_ROUTER=false -VLLM_BACKEND_PORT="$PORT" -if [ "$DP_ATTENTION" = "true" ]; then - USE_VLLM_ROUTER=true - VLLM_BACKEND_PORT=$((PORT + 1)) - VLLM_ROUTER_VERSION=0.1.14 - VLLM_ROUTER_POLICY=consistent_hash - VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" -fi - -# AIPerf scrapes the public endpoint's /metrics, which is the router under -# DP-attention; add the engine endpoint explicitly (deduplicated for pure TP). -export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -# Match the environment used by v4pro-b300.yaml. -export VLLM_USE_V2_MODEL_RUNNER=1 -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 -export VLLM_DSV4_MEGA_FP8_COMBINE=1 -export NCCL_NVLS_ENABLE=1 -export VLLM_USE_RUST_FRONTEND=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -ROUTER_PID="" -MOONCAKE_MASTER_PID="" - -OFFLOAD_ARGS=() -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - vllm-simple) - require_agentic_kv_offload_backend vllm-simple - CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / GPU_COUNT )) - # Identical prefixes must hash to identical block keys across DP ranks. - export PYTHONHASHSEED=42 - # DEP keeps eager offload for cross-rank block-hash stability; plain TP - # uses lazy offload. - SIMPLE_LAZY_OFFLOAD=false - if [ "$DP_ATTENTION" != "true" ]; then - SIMPLE_LAZY_OFFLOAD=true - fi - OFFLOAD_CONFIG=$(cat </dev/null - - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then - echo "Mooncake master died during startup." >&2 - cat "$MOONCAKE_MASTER_LOG" >&2 - exit 1 - fi - - unset VLLM_USE_SIMPLE_KV_OFFLOAD - OFFLOAD_CONFIG='{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG") - ;; - *) - echo "Error: unsupported B300 KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND'" >&2 - exit 1 - ;; -esac - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -TP_ARGS=() -if [ "$DP_ATTENTION" = "true" ]; then - export PYTORCH_ALLOC_CONF=expandable_segments:True -else - export VLLM_ALLREDUCE_USE_FLASHINFER=1 - export VLLM_FLASHINFER_ALLREDUCE_BACKEND=auto - TP_ARGS+=(--disable-custom-all-reduce) -fi - -MODE_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - MODE_ARGS+=( - --enable-expert-parallel - --enable-ep-weight-filter - --moe-backend deep_gemm_amxf4_mega_moe - ) -fi -if [ "$DP_ATTENTION" = "true" ]; then - MODE_ARGS+=( - --prefill-schedule-interval 8 - --long-prefill-token-threshold 512 - ) - if [ "$IS_DEP8" = "true" ]; then - MODE_ARGS+=(--max-num-batched-tokens 16384) - else - MODE_ARGS+=(--max-num-batched-tokens 8192) - fi -fi - -if [ "$DP_ATTENTION" = "true" ]; then - # The DEP source recipe enforces 2*CONC = DP_WORLD_SIZE*MAX_NUM_SEQS. - MAX_NUM_SEQS=$((2 * CONC / TP)) -else - # Headroom for AgentX subagent fan-out. - MAX_NUM_SEQS=$((2 * CONC)) -fi -# Cudagraph capture sizes are in tokens: a decode batch of S seqs verifies -# S*(1+N) tokens, so capture the multiples (1+N)..MAX_NUM_SEQS*(1+N). vLLM -# rounds sizes up to multiples of (1+N) and dedups, so a plain 1..MAX_NUM_SEQS -# list would cover only MAX_NUM_SEQS/(1+N) sequences. -NUM_SPEC_TOKENS=3 -TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) -# Golden AL: golden_al_distribution/dsv4_mtp.yaml, thinking_on, 3 draft tokens. -# EVAL_ONLY keeps real verification; synthetic acceptance bypasses it and -# zeroes the SWE-bench score. -if [ "${EVAL_ONLY}" = "true" ]; then - SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" -else - SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 2.49}" -fi -CUDA_GRAPH_CAPTURE_SIZES="" -for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do - if [ -n "$CUDA_GRAPH_CAPTURE_SIZES" ]; then - CUDA_GRAPH_CAPTURE_SIZES+="," - fi - CUDA_GRAPH_CAPTURE_SIZES+="$((num_seqs * TOKENS_PER_SEQ))" -done -COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[${CUDA_GRAPH_CAPTURE_SIZES}],\"mode\":0}" - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_FLOAT32_MATMUL_PRECISION=high - -# DEP8 leaves more headroom for its larger prefill token budget. -GPU_MEM_UTIL=0.95 -if [ "$IS_DEP8" = "true" ]; then - GPU_MEM_UTIL=0.92 -fi - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$VLLM_BACKEND_PORT" - --gpu-memory-utilization "$GPU_MEM_UTIL" - --trust-remote-code - --no-enable-flashinfer-autotune - --no-disable-hybrid-kv-cache-manager - --max-num-seqs "$MAX_NUM_SEQS" - --kv-cache-dtype fp8 - --block-size 256 - --max-model-len 1048576 - --attention-config '{"use_fp4_indexer_cache":true,"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true}' - --speculative-config "$SPEC_CONFIG" - --disable-uvicorn-access-log - --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --enable-auto-tool-choice - --reasoning-parser deepseek_v4 - --compilation-config "$COMPILATION_CONFIG" - "${PARALLEL_ARGS[@]}" - "${VLLM_CP_ARGS[@]}" - "${TP_ARGS[@]}" - "${MODE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_VLLM_ROUTER" = "true" ]; then - echo "Starting native vLLM router on port $PORT for $TP DP ranks..." - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy "$VLLM_ROUTER_POLICY" \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh deleted file mode 100755 index 2f908da72a..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_atom_mtp.sh +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro-0813 FP4 on MI355X using -# ATOM DSpark K6. All throughput runs use golden AL 3.77; eval uses real -# acceptance. The historical _mtp filename is also routed from draft_model. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars EVAL_ONLY - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -require_agentic_kv_offload_none - -echo "Attention mode: $([ "$DP_ATTENTION" = "true" ] && echo dp || echo tp) (DP_ATTENTION=$DP_ATTENTION, CONC=$CONC)" - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ "$MODEL" != "deepseek-ai/DeepSeek-V4-Pro-0813" ]]; then - echo "ERROR: DSpark requires the DeepSeek-V4-Pro-0813 checkpoint, got $MODEL" >&2 - exit 1 -fi -export DSV4_MODEL_REVISION=72e1d3230f6c080a530b0a1d46f8eb4602340597 -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --revision "$DSV4_MODEL_REVISION" --local-dir "$MODEL_PATH" - fi -else - # ATOM has no --revision flag. Serve the resolved immutable snapshot path. - MODEL_PATH=$(python3 - "$MODEL" "$DSV4_MODEL_REVISION" <<'PY' -import sys -from huggingface_hub import snapshot_download -print(snapshot_download(repo_id=sys.argv[1], revision=sys.argv[2])) -PY - ) -fi -export MODEL_PATH -export AGENTIC_TOKENIZER_PATH="$MODEL_PATH" -mkdir -p "$RESULT_DIR" -python3 "$(dirname "$0")/check_dsv4_dspark_checkpoint.py" \ - --model-path "$MODEL_PATH" --revision "$DSV4_MODEL_REVISION" \ - --output "$RESULT_DIR/checkpoint_preflight.json" - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps - -export AITER_BF16_FP8_MOE_BOUND=0 -export AITER_LOG_LEVEL=WARNING -export ATOM_MOE_GU_ITLV=1 -export ATOM_DISABLE_MMAP=true -export ATOM_DEBUG_PREFIX_HITS=1 -export ATOM_PROFILER_MORE=0 -export ATOM_PROFILER_TIMEOUT=1200 - -# EP is config-driven so the TP band remains TP-only while DEP uses one expert -# shard per GPU. -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -# The high-concurrency band uses ATOM's native RCCL DEP transport. Session -# affinity is required: otherwise consecutive turns can land on another DPA -# rank and lose access to the prefix KV produced by the previous turn. -DEP_ARGS=() -STATE_CHECKPOINT_INTERVAL_TOKENS=8192 -if [ "$DP_ATTENTION" = "true" ]; then - if [ "$EP_SIZE" -ne "$TP" ]; then - echo "ERROR: native RCCL DEP requires EP_SIZE=$TP for TP=$TP, got EP_SIZE=$EP_SIZE" >&2 - exit 1 - fi - # Keep only runtime controls that are not already expressed by DEP_ARGS. - export ATOM_DP_SESSION_AFFINITY=1 - export ATOM_DP_LB_REQ_EQUIV=512 - export ATOM_ENABLE_PREFILL_DELAYER=1 - export ATOM_PREFILL_DECODE_INTERVAL=10 - # Client-side counterpart to session affinity: make AIPerf emit a stable - # session id from its correlation id so the DPA router pins each - # conversation to one rank. - export AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID=1 - export AGENTIC_WARMUP_GRACE_PERIOD=3600 - DEP_ARGS=( - --enable-dp-attention - --all2all-backend rccl - --dp-load-balance least_tokens - --moe-backend standard - ) -fi - -# Long AgentX stalls exceed aiperf's default 30 s TCP_USER_TIMEOUT. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -export AIPERF_TIMING_CANCEL_DRAIN_TIMEOUT=300 -export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=0 -export AIPERF_DATASET_CONFIGURATION_TIMEOUT=1800 -export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800 -export AIPERF_UI_REALTIME_METRICS_ENABLED=true - -# Require ATOM Prometheus metrics in every official result. -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="atom:" - -wait_for_amd_gpu_clean - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -# Record the server interpreter's installed sources once, without importing GPU -# packages or resolving AITER's merged tuning table. The bundled CSV is evidence -# of image contents, not proof of which kernels a serving request executes. -python3 - "$RESULT_DIR/runtime_manifest.json" <<'PY' -import csv -import hashlib -import io -import json -import os -import subprocess -import sys -from importlib import metadata, util -from pathlib import Path - - -def package_manifest(name: str, distribution: str) -> dict: - info = {"distribution": distribution, "version": None, "origin": None, - "package_dir": None, "git": {"head": None, "dirty": None}} - try: - info["version"] = metadata.version(distribution) - except (metadata.PackageNotFoundError, OSError) as exc: - info["version_error"] = str(exc) - try: - spec = util.find_spec(name) - if spec is None or spec.origin is None: - info["source_error"] = "Package source was not found" - return info - origin = Path(spec.origin).resolve() - info["origin"] = str(origin) - info["package_dir"] = str(origin.parent) - - def git(*args: str) -> str: - return subprocess.run( - ["git", "-C", str(origin.parent), *args], check=True, - capture_output=True, text=True, timeout=5, - env={**os.environ, "GIT_OPTIONAL_LOCKS": "0"}, - ).stdout.strip() - - root = Path(git("rev-parse", "--show-toplevel")) - # Do not mistake an unrelated enclosing checkout for the package repo. - git("ls-files", "--error-unmatch", "--", str(origin)) - info["git"].update(root=str(root), head=git("rev-parse", "HEAD")) - info["git"]["dirty"] = bool(git("status", "--porcelain", "--untracked-files=no")) - except (OSError, ValueError, ImportError, subprocess.SubprocessError) as exc: - info["source_error"] = str(exc) - return info - - -packages = {name: package_manifest(name, dist) - for name, dist in (("atom", "atom"), ("aiter", "amd-aiter"))} -bundled = {"path": None, "sha256": None, "ep48_rows": [], - "scope": "Bundled CSV only; runtime overrides and kernel dispatch are not resolved"} -aiter_dir = packages["aiter"]["package_dir"] -if aiter_dir is not None: - path = Path(aiter_dir) / "configs/model_configs/dsv4_fp8fp4_tuned_fmoe.csv" - bundled["path"] = str(path) - try: - data = path.read_bytes() - bundled["sha256"] = hashlib.sha256(data).hexdigest() - for row in csv.DictReader(io.StringIO(data.decode("utf-8"))): - if (row.get("gfx") == "gfx950" and row.get("cu_num") == "256" - and row.get("model_dim") == "7168" - and row.get("inter_dim") == "3072" and row.get("expert") == "48" - and row.get("topk") == "6" - and row.get("token") in {"16384", "32768", "131072"}): - bundled["ep48_rows"].append({key: row.get(key) for key in ( - "gfx", "cu_num", "token", "model_dim", "inter_dim", "expert", "topk", - "block_m", "kernelName1", "kernelName2", - )}) - except (OSError, ValueError, csv.Error) as exc: - bundled["error"] = str(exc) -else: - bundled["error"] = "AITER package source was not found" - -manifest = { - "requested_image": os.environ.get("IMAGE"), - "python_executable": sys.executable, - "server_command_file": "server_command.txt", - "checkpoint": json.loads((Path(sys.argv[1]).parent / "checkpoint_preflight.json").read_text()), - "speculation": { - "method": "dspark", "num_speculative_tokens": 6, "target_verify_length": 7, - "forced_acceptance_length": None if os.environ.get("EVAL_ONLY") == "true" else 3.77, - "confidence_schedule": False, "ragged": False, - }, - "graph_evidence": "Requested FULL q7; capture completion must be checked in server.log", - "packages": packages, - "aiter_overrides": {key: os.environ.get(key) for key in ( - "AITER_CONFIG_FMOE", "AITER_BYPASS_TUNE_CONFIG", - )}, - "bundled_dsv4_fmoe": bundled, -} -Path(sys.argv[1]).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") -PY - -SERVER_PID="" -cleanup_atom_server() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "ATOM server" 60 - exit "$exit_code" -} -trap cleanup_atom_server EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -# AgentX concurrency counts session trees. Keep 2x scheduler headroom for the -# request bursts produced by subagent fan-out. -MAX_NUM_SEQS=$((2 * CONC)) - -# Use BF16 KV for every configured task at concurrency 16 and below. Keep FP8 -# KV for the higher-concurrency DEP band. -KV_CACHE_DTYPE=fp8 -if [ "$CONC" -le 16 ]; then - KV_CACHE_DTYPE=bf16 -fi - -# DPA splits the C48+ workload across eight ranks, so real decode batches are -# commonly 3, 5-7, and 9-15. ATOM's default power-of-two ladder rounds those -# shapes up and runs unnecessary attention, MoE, and collective work. Capture -# every small shape for DEP, while retaining larger graphs for the C96+ arms. -CUDAGRAPH_ARGS=() -if [ "$DP_ATTENTION" = "true" ]; then - CUDAGRAPH_CAPTURE_SIZES='[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128]' - if [ "$MAX_NUM_SEQS" -gt 128 ]; then - CUDAGRAPH_CAPTURE_SIZES='[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,32,48,64,128,256,512]' - fi - CUDAGRAPH_ARGS=(--cudagraph-capture-sizes "$CUDAGRAPH_CAPTURE_SIZES") -fi - -# golden_al_distribution/dsv4-pro-0813-dspark.yaml: thinking_on, K6 -> AL 3.77. -# K6 means six draft tokens plus one target token (q7), matching SGLang. -# Apply the golden value to both throughput bands; eval must verify real drafts. -NUM_SPEC_TOKENS=6 -SPEC_DECODE_AL=3.77 -SPEC_ARGS=( - --method dspark - --num-speculative-tokens "$NUM_SPEC_TOKENS" -) -if [ "${EVAL_ONLY}" != "true" ]; then - SPEC_ARGS+=(--spec-decode-acceptance-length "$SPEC_DECODE_AL") -fi - -echo "Starting ATOM server with MAX_NUM_SEQS=$MAX_NUM_SEQS NUM_SPEC_TOKENS=$NUM_SPEC_TOKENS KV_CACHE_DTYPE=$KV_CACHE_DTYPE STATE_CHECKPOINT_INTERVAL_TOKENS=$STATE_CHECKPOINT_INTERVAL_TOKENS DP_ATTENTION=$DP_ATTENTION EP_SIZE=$EP_SIZE EVAL_ONLY=${EVAL_ONLY:-false}" -ATOM_CMD=( - python3 -u -m atom.entrypoints.openai_server - --model "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --server-port "$PORT" - # uvicorn's default 5 s idle keep-alive is shorter than AIPerf's pooled - # socket reuse; a reset on a root warmup request aborts the whole run. - --timeout-keep-alive 900 - --tensor-parallel-size "$TP" - --data-parallel-size 1 - --kv-cache-dtype "$KV_CACHE_DTYPE" - --index-cache-dtype fp4 - --enable-prefix-caching - --gpu-memory-utilization 0.9 - --max-num-batched-tokens 16384 - --attn-prefill-chunk-size 16384 - --state-checkpoint-interval-tokens "$STATE_CHECKPOINT_INTERVAL_TOKENS" - --level 3 - --cudagraph-mode FULL - "${CUDAGRAPH_ARGS[@]}" - "${SPEC_ARGS[@]}" - "${EP_ARGS[@]}" - "${DEP_ARGS[@]}" - --max-num-seqs "$MAX_NUM_SEQS" -) -write_command "$RESULT_DIR/server_command.txt" "${ATOM_CMD[@]}" -"${ATOM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - # AgentX DSv4 traces already carry fully formed chat payloads; do not apply - # AIPerf's generic chat template on top of them. - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh deleted file mode 100644 index 8cafbaf91d..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro-0813 FP4 on MI355X using -# SGLang with DSpark speculative decoding. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars EVAL_ONLY - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# ROCR/HIP visibility under slurm cgroups. -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# A server killed minutes earlier can still be draining HBM (KFD reclaim takes -# minutes), and booting into a half-drained node fails RCCL init with HIP -# 'unhandled cuda error'. Idle GPUs sit at up to ~4% VRAM, draining ones at -# 50-90%, so require every GPU <= 10%. -GPU_CLEAN=false -for i in $(seq 1 90); do - VRAM_MAX=$(rocm-smi --showmemuse 2>/dev/null | grep -oE "GPU Memory Allocated \(VRAM%\): [0-9]+" | awk '{if ($NF > m) m = $NF} END {print m+0}') - if [ "${VRAM_MAX:-0}" -le 10 ]; then echo "GPUs clean (vram%max=$VRAM_MAX after $((i*10))s)"; GPU_CLEAN=true; break; fi - echo "waiting for prior-job GPU memory reclaim: vram%max=$VRAM_MAX"; sleep 10 -done -[ "$GPU_CLEAN" = "true" ] || { echo "Error: GPUs still draining prior job's memory after 15min" >&2; exit 1; } - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -mkdir -p "$RESULT_DIR" - -export PYTHONNOUSERSITE=1 -# Agentic warmup dispatches hundreds of large prompts at once; allow up to -# 15 minutes of TCP progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# AIPerf pins one pooled keep-alive connection per session while uvicorn's -# default keep-alive is 5 s; outlast the client pool so the reuse race cannot occur. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX measures the thinking-on regime, which is also the committed golden-AL curve. -export SGLANG_DEFAULT_THINKING=1 -export SGLANG_DSV4_REASONING_EFFORT=high -export SGLANG_USE_ROCM700A=0 -export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton -export AITER_BF16_FP8_MOE_BOUND=0 -export TORCH_BLAS_PREFER_HIPBLASLT=1 -export HSA_NO_SCRATCH_RECLAIM=0 -# aiter batched GEMM for the absorbed MLA projections; off by default in environ.py. -export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 - -# Unified radix tree with proactive release of out-of-window SWA slots. -# Without it in-flight requests pin SWA KV for their whole context and the -# trailing window of cached sessions is flushed under LRU, collapsing the -# prefix-cache hit rate on multi-turn agentic workloads. -export SGLANG_ENABLE_UNIFIED_RADIX_TREE=1 -export SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS=1 - -# Host pinned memory is roughly HICACHE_RATIO * per-rank device KV pool * TP -# and must stay under the node's ~2.7 TB; ratio 4 oversubscribes at TP8 with -# mem-fraction-static 0.85, so start from 1.5. -CACHE_ARGS=() -if agentic_kv_offload_enabled; then - case "$KV_OFFLOAD_BACKEND" in - hicache) - HICACHE_RATIO="1.5" - HICACHE_WRITE_POLICY="write_through" - HICACHE_IO_BACKEND="direct" - HICACHE_MEM_LAYOUT="page_first_direct" - echo "HiCache DSv4 CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT, dram_budget=${TOTAL_CPU_DRAM_GB} GB, tp=$TP" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected: hicache)" >&2 - exit 1 - ;; - esac -fi - -# ---- Parallelism ------------------------------------------------------------ -# The DP-attention path below is live: sglang-router fronts the DP ranks with -# cache-aware routing, which picks the rank holding the longest radix/hicache -# prefix match for the incoming prompt. -USE_SGLANG_ROUTER=false -ROUTER_POLICY_ARGS=() -SGLANG_BACKEND_PORT="$PORT" -# The flag is engine-wide and DP divides it by dp_size (=TP), so DP uses -# 8192*TP to keep 8192 per rank. -case "$TP" in - 4|8) ;; - *) echo "Error: unsupported TP '$TP' (expected: 4 or 8)" >&2; exit 1 ;; -esac -if [ "$DP_ATTENTION" = "true" ]; then - CHUNKED_PREFILL_SIZE=$((8192 * TP)) -elif [ "$TP" -eq 8 ]; then - CHUNKED_PREFILL_SIZE=16384 -else - CHUNKED_PREFILL_SIZE=8192 -fi -MEM_FRACTION_STATIC="0.86" -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -SHARED_EXPERTS_ARGS=(--enforce-shared-experts-fusion) -SWA_FULL_TOKENS_RATIO="0.10" -export GPU_MAX_HW_QUEUES="2" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router) - - export SGLANG_SHARED_EXPERT_TP1=1 - export SGLANG_DP_SHARED_EXPERT_LOCAL=1 - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES="5" - MEM_FRACTION_STATIC="0.92" - - if [ "$CONC" -gt 160 ]; then - ROUTER_POLICY_ARGS+=(--balance-abs-threshold 32) - fi - - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --enable-dp-lm-head - --enable-prefill-delayer - --enable-dp-attention-local-control-broadcast - --tokenizer-worker-num "$TP" - --stream-interval 20 - --prefill-decode-interval "20" - --prefill-delayer-token-usage-low-watermark "0.7" - ) -else - PARALLEL_ARGS+=(--prefill-decode-interval "20") -fi - -if [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--ep-size "$EP_SIZE") - SHARED_EXPERTS_ARGS=(--disable-shared-experts-fusion) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Subagent fan-out can push instantaneous request concurrency above CONC, so -# leave 2x headroom rather than clipping those bursts at the scheduler. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS -[ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 - -# Saturation arms carry a larger in-flight working set than the 30-minute -# default warmup drain allows. -if [ "$CONC" -ge 32 ]; then - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi - -# The DSpark draft head is bundled in the target checkpoint (dspark_* keys in -# config.json), so no separate draft path. gamma=6 is AL-optimal on the golden curve. -DSV4_DSPARK_GAMMA="6" - -SPEC_ARGS=( - --speculative-algorithm DSPARK - --speculative-dspark-block-size "$DSV4_DSPARK_GAMMA" - --speculative-num-steps 1 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens $((DSV4_DSPARK_GAMMA + 1)) -) - -# Golden AL: golden_al_distribution/dsv4-pro-0813-dspark.yaml, thinking_on, -# gamma 6 -> 3.77. Eval-only runs keep real target verification. -DSV4_GOLDEN_AL=3.77 -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN="$DSV4_GOLDEN_AL" - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi -echo "DSpark draft length: gamma=$DSV4_DSPARK_GAMMA (verify window $((DSV4_DSPARK_GAMMA + 1))), golden AL=$DSV4_GOLDEN_AL" - -# No --chat-template: deepseek_v4_thinking.jinja renders only -# system/user/assistant and silently drops tool definitions and tool messages, -# which would truncate prompts and distort ISL. -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --attention-backend dsv4 - --enable-deepseek-v4-fp4-indexer - --page-size 256 - --swa-full-tokens-ratio "$SWA_FULL_TOKENS_RATIO" - --kv-cache-dtype fp8_e4m3 - "${SHARED_EXPERTS_ARGS[@]}" - --tool-call-parser deepseekv4 - --reasoning-parser deepseek-v4 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - "${SPEC_ARGS[@]}" - "${CACHE_ARGS[@]}" - # Draft-token forward passes under long-context agentic load block the - # scheduler long enough to trip the 1800s watchdog mid-warmup. - --watchdog-timeout 3600 - --enable-metrics -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_* env vars at launch ===" - env | grep -E '^SGLANG_' | sort - echo "===================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for MI355X..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - "${SGLANG_ROUTER_CMD[@]}" \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy cache_aware \ - "${ROUTER_POLICY_ARGS[@]}" \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh deleted file mode 100644 index beea86d94c..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ /dev/null @@ -1,394 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# DeepSeek-V4-Pro-0813 FP4 on MI355X with vLLM DSpark K6 and golden synthetic -# acceptance. The script retains MTP K3 support for historical and ad hoc runs. -# Pure TP (DP_ATTENTION=false), TP+EP (EP_SIZE>1), and DEP (DP_ATTENTION=true) -# arms are supported. -# https://recipes.vllm.ai/deepseek-ai/DeepSeek-V4-Pro?hardware=mi355x -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-native or lmcache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR \ - DURATION EP_SIZE DP_ATTENTION -check_env_vars EVAL_ONLY - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -DSV4_MODEL_REVISION="" -if [[ "$MODEL" == "deepseek-ai/DeepSeek-V4-Pro-0813" ]]; then - DSV4_MODEL_REVISION=72e1d3230f6c080a530b0a1d46f8eb4602340597 -fi -if [[ "$SPEC_DECODING" == "draft_model" && -z "$DSV4_MODEL_REVISION" ]]; then - echo "ERROR: DSpark requires the DeepSeek-V4-Pro-0813 checkpoint, got $MODEL" >&2 - exit 1 -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - if [[ -n "$DSV4_MODEL_REVISION" ]]; then - hf download "$MODEL" --revision "$DSV4_MODEL_REVISION" --local-dir "$MODEL_PATH" - else - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - fi -elif [[ -n "$DSV4_MODEL_REVISION" ]]; then - MODEL_PATH=$(python3 - "$MODEL" "$DSV4_MODEL_REVISION" <<'PY' -import sys -from huggingface_hub import snapshot_download -print(snapshot_download(repo_id=sys.argv[1], revision=sys.argv[2])) -PY - ) - export MODEL_PATH -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -if [[ -n "$DSV4_MODEL_REVISION" ]]; then - export AGENTIC_TOKENIZER_PATH="$MODEL_PATH" - mkdir -p "$RESULT_DIR" - python3 "$(dirname "$0")/check_dsv4_dspark_checkpoint.py" \ - --model-path "$MODEL_PATH" --revision "$DSV4_MODEL_REVISION" \ - --output "$RESULT_DIR/checkpoint_preflight.json" -fi - -if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -resolve_trace_source -install_agentic_deps - -# The nightly ROCm image lacks these runtime deps. -agentic_pip_install --quiet Pillow fastapi uvicorn - -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 - -# vllm-router expands one HTTP backend into a logical worker per DP rank. -# AIPerf's X-Correlation-ID is stable across a conversation's turns; alias it -# to the router's X-Session-ID so every turn lands on the same rank. -USE_VLLM_ROUTER=false -VLLM_BACKEND_PORT="$PORT" -if [ "$DP_ATTENTION" = "true" ]; then - USE_VLLM_ROUTER=true - VLLM_BACKEND_PORT=$((PORT + 1)) - VLLM_ROUTER_VERSION=0.1.14 - VLLM_ROUTER_POLICY=consistent_hash - VLLM_ROUTER_METRICS_PORT=$((PORT + 10000)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION" -fi - -# AIPerf scrapes the public endpoint's /metrics, which is the router under -# DP-attention; add the engine endpoint explicitly (deduplicated for pure TP). -export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -# 805 GiB checkpoint; cold Weka loads take about two hours for the 64 shards. -export VLLM_ENGINE_READY_TIMEOUT_S=10800 - -# vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while -# vllm-project/vllm#44774 applies the same reachability policy to Mooncake's -# store mask. 32k matches the trace-replay tuning validated for this workload. -export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=32768 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -ROUTER_PID="" - -OFFLOAD_ARGS=() - -if agentic_kv_offload_enabled; then - check_env_vars KV_OFFLOAD_BACKEND - case "$KV_OFFLOAD_BACKEND" in - vllm-native) - require_agentic_kv_offload_backend vllm-native - unset VLLM_USE_SIMPLE_KV_OFFLOAD - TOTAL_CPU_DRAM_PARTITION_GB="$((TOTAL_CPU_DRAM_GB / (8 / TP)))" - # OffloadingConnector, not SimpleCPUOffloadConnector: VLLM_USE_SIMPLE_KV_OFFLOAD - # must stay unset. - - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_PARTITION_GB" - ) - - ;; - lmcache) - require_agentic_kv_offload_backend lmcache - LMCACHE_PID="" - - cleanup_lmcache_server() { - if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then - kill "$LMCACHE_PID" 2>/dev/null || true - wait "$LMCACHE_PID" 2>/dev/null || true - fi - } - - trap cleanup_lmcache_server EXIT - - cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$ROUTER_PID" "vLLM router" - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$exit_code" - } - trap cleanup_agentic_services EXIT - trap 'exit 130' INT - trap 'exit 143' TERM - - wait_for_lmcache_ready() { - { set +x; } 2>/dev/null - local attempts="120" - local tail_pid="" - - while [ ! -f "$LMCACHE_LOG" ]; do - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before creating log file. Exiting." >&2 - exit 1 - fi - sleep 10 - done - - tail -f -n +1 "$LMCACHE_LOG" & - tail_pid=$! - - for ((i = 1; i <= attempts; i++)); do - if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - return 0 - fi - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before becoming healthy. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 - fi - sleep 1 - done - - echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 - } - { set +x; } 2>/dev/null - unset VLLM_USE_SIMPLE_KV_OFFLOAD - - git clone https://github.com/LMCache/LMCache.git - cd LMCache - # https://github.com/LMCache/LMCache/pull/3853 - git checkout 9229067cec0b3a63bb8a39368d101db7ac0bc3c1 - pip install -r requirements/build.txt - pip install grpcio==1.78.0 - CXX=hipcc BUILD_WITH_HIP=1 pip install -e . --no-build-isolation - cd .. - - python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null - - TOTAL_CPU_DRAM_PARTITION_GB="$((TOTAL_CPU_DRAM_GB / (8 / TP)))" - # The external MP server owns the pool so vLLM does not split - # --kv-offloading-size across TP ranks. - LMCACHE_HOST="127.0.0.1" - LMCACHE_PORT="5555" - LMCACHE_HTTP_PORT="8080" - # LMCacheMPConnector concatenates lmcache.mp.host and port into the - # ZMQ endpoint, so the connector gets a ZMQ-style host string. - LMCACHE_CONNECT_HOST="tcp://$LMCACHE_HOST" - LMCACHE_L1_SIZE_GB="${TOTAL_CPU_DRAM_PARTITION_GB}" - if [ "$LMCACHE_L1_SIZE_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: LMCACHE_L1_SIZE_GB=$LMCACHE_L1_SIZE_GB exceeds configured capacity $TOTAL_CPU_DRAM_GB" >&2 - exit 1 - fi - LMCACHE_L1_INIT_SIZE_GB="20" - # Read locks are leases on chunks lookup promised vLLM can retrieve. - # TP8/conc32 can spend >300 s between lookup and retrieve while GPU - # KV is saturated, leaving the object in L1 but unreadable. - LMCACHE_L1_READ_TTL_SECONDS="7200" - LMCACHE_CHUNK_SIZE="256" - LMCACHE_MAX_WORKERS="$TP" - export PYTHONHASHSEED="0" - export LMCACHE_BLOCKING_TIMEOUT_SECS=1200 - LMCACHE_TX_MODE="lmcache_driven" - - echo "Starting LMCache MP server..." - LMCACHE_CMD=( - lmcache server - --host "$LMCACHE_HOST" - --port "$LMCACHE_PORT" - --http-host "$LMCACHE_HOST" - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" - --l1-read-ttl-seconds "$LMCACHE_L1_READ_TTL_SECONDS" - --chunk-size "$LMCACHE_CHUNK_SIZE" - --max-workers "$LMCACHE_MAX_WORKERS" - --eviction-policy LRU - --supported-transfer-mode "$LMCACHE_TX_MODE" - ) - printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" - printf '\n' >> "$RESULT_DIR/lmcache_command.txt" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - echo "LMCache server PID: $LMCACHE_PID" - wait_for_lmcache_ready - - PREFIX_CACHE_ARGS=(--enable-prefix-caching) - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT,\"lmcache.mp.mq_timeout\":6000.0}}" - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected: vllm-native, lmcache)" >&2 - exit 1 - ;; - esac -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -DP_SCHED_ARGS=() -if [ "$DP_ATTENTION" = "true" ]; then - DP_SCHED_ARGS=( - --prefill-schedule-interval 8 - --long-prefill-token-threshold 16384 - ) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Subagent fan-out can push instantaneous request concurrency above CONC, so -# leave 2x headroom rather than clipping those bursts at the scheduler. -MAX_NUM_SEQS=$((2 * CONC)) -if [ "$DP_ATTENTION" = "true" ]; then - MAX_NUM_SEQS="$CONC" -fi - -# MTP K3 uses the committed thinking-on AL 2.49. DSpark K6 uses the -# DeepSeek-V4-Pro-0813 thinking-on AL 3.77. Eval-only runs always use real -# target verification. -case "$SPEC_DECODING" in - mtp) - NUM_SPEC_TOKENS=3 - SYNTHETIC_ACCEPT_LEN=2.49 - if [ "${EVAL_ONLY}" = "true" ]; then - SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" - else - SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" - fi - ;; - draft_model) - export VLLM_USE_V2_MODEL_RUNNER=1 - NUM_SPEC_TOKENS=6 - SYNTHETIC_ACCEPT_LEN=3.77 - if [ "${EVAL_ONLY}" = "true" ]; then - SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\"}" - else - SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" - fi - ;; - *) - echo "Error: unsupported SPEC_DECODING='$SPEC_DECODING' for this recipe" >&2 - exit 1 - ;; -esac - -echo "Starting vllm server..." -set -x -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export VLLM_ROCM_USE_AITER_MOE=1 -# This checkpoint mixes packed MXFP4 routed experts with a full-width FP8 -# shared expert. The latest nightly otherwise admits the combination into the -# fused path and fails while loading incompatible scales/shapes. -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 -# vLLM only clamps torch threads after weight loading; cap from process start. -export OMP_NUM_THREADS=1 - -sleep 180 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$VLLM_BACKEND_PORT" - --trust-remote-code - --async-scheduling - --distributed-executor-backend mp - --kv-cache-dtype fp8 - --max-num-batched-tokens 8192 - "${PARALLEL_ARGS[@]}" - "${EP_ARGS[@]}" - "${DP_SCHED_ARGS[@]}" - --gpu-memory-utilization 0.86 - --moe-backend aiter - --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' - --speculative-config "$SPEC_CONFIG" - --tokenizer-mode deepseek_v4 - --tool-call-parser deepseek_v4 - --reasoning-parser deepseek_v4 - --enable-auto-tool-choice - --enable-prefix-caching - --no-disable-hybrid-kv-cache-manager - --max-num-seqs "$MAX_NUM_SEQS" - "${OFFLOAD_ARGS[@]}" -) - -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_VLLM_ROUTER" = "true" ]; then - echo "Starting native vLLM router on port $PORT for $TP DP ranks..." - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy "$VLLM_ROUTER_POLICY" \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh deleted file mode 100755 index 1edcaf2888..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp4_b200_sglang_mtp.sh +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# GLM-5.2 NVFP4 on B200 with SGLang EAGLE/MTP. Port of -# glm5.2_fp4_b300_sglang_mtp.sh; the B200 deltas are marked "B200:" below. -# DP_ATTENTION=false is the low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM); -# DP_ATTENTION=true is the DEP arm (TP8 + DP8 attention + --ep-size), kept -# intact but not wired into the master config. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE, DP_ATTENTION -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars EVAL_ONLY - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# B200: launch_b200-nscale-slurm.sh rewrites MODEL to a cluster-local path, so -# keep the HF repo id separately for the unstaged case. -HF_MODEL_ID="nvidia/GLM-5.2-NVFP4" - -# A non-empty directory is not a staged checkpoint: an aborted pull leaves -# config.json and friends with no tokenizer or weights, and SGLang then dies -# in AutoTokenizer.from_pretrained. Require the tokenizer, the shard index, -# and every shard it names. -checkpoint_is_complete() { - local dir="$1" - [[ -d "$dir" ]] || return 1 - [[ -f "$dir/tokenizer_config.json" ]] || return 1 - [[ -f "$dir/tokenizer.json" || -f "$dir/tokenizer.model" ]] || return 1 - [[ -f "$dir/model.safetensors.index.json" ]] || return 1 - CKPT_DIR="$dir" python3 - <<'PYEOF' -import json, os, sys -d = os.environ["CKPT_DIR"] -with open(os.path.join(d, "model.safetensors.index.json")) as fh: - shards = sorted(set(json.load(fh)["weight_map"].values())) -missing = [s for s in shards if not os.path.isfile(os.path.join(d, s))] -if missing: - print(f"{len(missing)}/{len(shards)} shards missing, e.g. {missing[:3]}", file=sys.stderr) - sys.exit(1) -PYEOF -} - -if [[ -n "${MODEL_PATH:-}" ]]; then - if ! checkpoint_is_complete "$MODEL_PATH"; then - # Every concurrency runs as its own allocation against the same Lustre - # path; one cell pulls the ~433 GB checkpoint and the rest wait on the - # lock. hf download resumes into a partially populated --local-dir. - mkdir -p "$MODEL_PATH" - MODEL_DOWNLOAD_LOCK="${MODEL_PATH%/}.download.lock" - echo "Checkpoint at $MODEL_PATH is incomplete; acquiring $MODEL_DOWNLOAD_LOCK" - exec 9>"$MODEL_DOWNLOAD_LOCK" - check_env_vars MODEL_DOWNLOAD_LOCK_TIMEOUT - flock -w "$MODEL_DOWNLOAD_LOCK_TIMEOUT" 9 || { - echo "Error: timed out waiting for another cell to stage $MODEL_PATH" >&2 - exit 1 - } - if checkpoint_is_complete "$MODEL_PATH"; then - echo "Another cell staged $MODEL_PATH while we waited" - else - hf download "$HF_MODEL_ID" --local-dir "$MODEL_PATH" - fi - flock -u 9 - exec 9>&- - checkpoint_is_complete "$MODEL_PATH" || { - echo "Error: $MODEL_PATH is still incomplete after hf download $HF_MODEL_ID." >&2 - exit 1 - } - fi -else - hf download "$HF_MODEL_ID" - export MODEL_PATH="$HF_MODEL_ID" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache spills evicted prefixes to a pinned host pool. On the 1M-context - # corpus the working set outgrows HBM past conc 8 (TP8) / 64 (DP8) and the - # radix hit rate collapses to <0.1, so every turn re-prefills its history. - # GLM-5.2 is MLA-family: every TP rank holds complete per-token KV. Ratio - # 0.75 gives only 1,257,728 host slots, so c12/c16 use a 169 GB/rank - # absolute pool (with the coupled 38.73 GB/rank DSA indexer, ~1,662 GB - # across TP8 of the 1,731 GB b200-nscale budget). - DEFAULT_HICACHE_RATIO=0.75 - DEFAULT_HICACHE_SIZE=0 - case "$CONC" in - 12|16) DEFAULT_HICACHE_SIZE=169 ;; - esac - MAX_HICACHE_SIZE=270 - HICACHE_SIZE="$DEFAULT_HICACHE_SIZE" - if ! [[ "$HICACHE_SIZE" =~ ^[0-9]+$ ]]; then - echo "Error: HICACHE_SIZE must be a non-negative integer, got $HICACHE_SIZE" >&2 - exit 1 - fi - if awk -v s="$HICACHE_SIZE" -v cap="$MAX_HICACHE_SIZE" 'BEGIN { exit !(s > cap) }'; then - echo "Error: HICACHE_SIZE=$HICACHE_SIZE exceeds configured limit $MAX_HICACHE_SIZE" >&2 - exit 1 - fi - HICACHE_RATIO="$DEFAULT_HICACHE_RATIO" - HICACHE_WRITE_POLICY="write_back" - HICACHE_IO_BACKEND="direct" - HICACHE_MEM_LAYOUT="page_first_direct" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - if awk -v s="$HICACHE_SIZE" 'BEGIN { exit !(s > 0) }'; then - echo "HiCache CPU tier: target_size=$HICACHE_SIZE GB, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" - CACHE_ARGS+=(--hicache-size "$HICACHE_SIZE") - else - if awk -v r="$HICACHE_RATIO" -v cap="$DEFAULT_HICACHE_RATIO" 'BEGIN { exit !(r > cap) }'; then - echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2 - exit 1 - fi - echo "HiCache CPU tier: ratio=$HICACHE_RATIO, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" - CACHE_ARGS+=(--hicache-ratio "$HICACHE_RATIO") - fi -fi - -# With attention-DP, front the DP ranks with sglang-router using consistent -# hashing on the AIPerf correlation id so multi-turn sessions stay on the DP -# rank that holds their radix-cache prefix. -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) -fi - -# GLM-5.2 ships its own nextn head, so EAGLE runs off the checkpoint. Three -# draft tokens per step is the draft length whose golden AL is pinned below. -SPEC_ARGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -CHUNKED_PREFILL_SIZE=8192 -if [ "$DP_ATTENTION" = "true" ]; then - # chunked-prefill-size is a whole-engine budget split across DP ranks: - # 8192 becomes 1,024 tokens/rank/step under dp8 and a conc-256 warmup - # could not drain within AIPerf's 1800 s grace period. 32768 = ~4096/rank. - CHUNKED_PREFILL_SIZE=32768 - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --tokenizer-worker-num "$TP" - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - ) - # The nextn layer is unquantized (hf_quant_config excludes model.layers.78*), - # so the draft MoE is bf16 and pinned to the triton runner; inheriting the - # target's FlashInfer all-to-all dies at init with "Pre-permute function for - # flashinfer to triton is not registered". SGLang only applies its fix on - # is_hip(), so set the ROCm values explicitly. Only matters with EP a2a. - SPEC_ARGS+=( - --speculative-moe-a2a-backend none - --speculative-moe-runner-backend triton - ) -else - # Cookbook low-latency levers; the DP-attention cell omits them. - PARALLEL_ARGS+=( - --kv-cache-dtype fp8_e4m3 - --bf16-gemm-backend cutedsl - --max-prefill-tokens 8192 - ) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -GRAPH_ARGS=() -if [ "$DP_ATTENTION" != "true" ]; then - # --cuda-graph-max-bs counts requests, not verification tokens; SGLang's - # spec-decode graph runner scales by --speculative-num-draft-tokens itself. - CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS - [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") -fi - -# B200: 180 GB HBM3e vs B300's 288 GB. B300's 0.85 leaves 43 GB headroom -# there but 27 GB here, and the EAGLE verification activations and 4-token -# graph capture come out of it on top of the DSA indexer temporaries. 0.83 -# restores ~31 GB. -MEM_FRACTION_STATIC="0.83" - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Each concurrency is a separate Slurm allocation on a shared home directory; -# keep FlashInfer autotune, Triton, Inductor and CUDA JIT caches -# allocation-local so concurrent cells cannot overwrite the same files. -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - export SGLANG_CACHE_DIR="/tmp/sglang-cache-${SLURM_JOB_ID}" -fi -# Agentic warmup dispatches hundreds of large prompts at once; allow up to -# 15 minutes of TCP progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# AIPerf pins one pooled keep-alive connection per session while uvicorn's -# default keep-alive is 5 s; an inter-turn idle gap can reuse a socket as the -# server closes it (ECONNRESET, terminal warmup failure). -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# Acceptance is pinned to the committed golden AL (golden_al_distribution/README.md): -# 2.99 is glm5.2_mtp.yaml at num_speculative_tokens=3, thinking_on. One curve -# per model: collected on FP8, and the NVFP4 checkpoint ships the same nextn head. -# SGLANG_SIMULATE_ACC_TOKEN_MODE exists from SGLang v0.5.16; an older image -# silently honors ACC_LEN/ACC_METHOD and ignores the token mode. -# EVAL_ONLY leaves it off: simulated acceptance commits drafted tokens -# regardless of target logits and the eval would score ~0. -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.99 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --quantization modelopt_fp4 - # GLM-5.2 emits the GLM-4.7 tool-call format; glm45 leaves calls as raw - # text and the SWE-bench eval dies with RepeatedFormatError. Neither parser - # affects replay throughput. - --tool-call-parser glm47 - --reasoning-parser glm45 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --max-running-requests "$MAX_RUNNING_REQUESTS" - "${SPEC_ARGS[@]}" - "${GRAPH_ARGS[@]}" - "${CACHE_ARGS[@]}" - --watchdog-timeout 1800 - --enable-metrics -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_SIMULATE_ACC_* env vars at launch (empty => real verification) ===" - env | grep -E '^SGLANG_SIMULATE_ACC_' | sort || true - echo "============================================================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B200..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - python3 -m sglang_router.launch_router \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - # The chat template defaults to reasoning_effort=Max when no - # chat_template_kwargs are passed (mini-swe-agent passes none), and the heavy - # thinking burns the shared 75-step budget (12/23 exited LimitsExceeded). - export SWEBENCH_AGENT_STEP_LIMIT=150 - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang_mtp.sh deleted file mode 100755 index 6646e406a8..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang_mtp.sh +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# GLM-5.2 NVFP4 on B300 with SGLang EAGLE/MTP. -# DP_ATTENTION=false is the low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM); -# DP_ATTENTION=true is the DEP arm (TP8 + DP8 attention + --ep-size), kept -# intact but not wired into the master config. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE, DP_ATTENTION -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars EVAL_ONLY - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache spills evicted prefixes to a pinned host pool. On the 1M-context - # corpus the working set outgrows HBM past conc 8 (TP8) / 64 (DP8) and the - # radix hit rate collapses to <0.1, so every turn re-prefills its history. - # GLM-5.2 is MLA-family: every TP rank holds complete per-token KV, so use - # an absolute pool size. In SGLang v0.5.16 --hicache-size sizes only the - # target KV host pool; the DSA indexer and MTP draft pools inherit its slot - # count. 270 GB measured as 335.34 GB/rank (2.683 TB across TP8) and left - # ~1.74 TiB node MemAvailable. - DEFAULT_HICACHE_SIZE=270 - MAX_HICACHE_SIZE=270 - HICACHE_SIZE="$DEFAULT_HICACHE_SIZE" - if ! [[ "$HICACHE_SIZE" =~ ^[0-9]+$ ]]; then - echo "Error: HICACHE_SIZE must be a positive integer, got $HICACHE_SIZE" >&2 - exit 1 - fi - if awk -v s="$HICACHE_SIZE" -v cap="$MAX_HICACHE_SIZE" 'BEGIN { exit !(s <= 0 || s > cap) }'; then - echo "Error: HICACHE_SIZE=$HICACHE_SIZE must be in (0, $MAX_HICACHE_SIZE]" >&2 - exit 1 - fi - HICACHE_WRITE_POLICY="write_back" - HICACHE_IO_BACKEND="direct" - HICACHE_MEM_LAYOUT="page_first_direct" - echo "HiCache CPU tier: conc=$CONC, target_size=$HICACHE_SIZE GB, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) -fi - -# With attention-DP, front the DP ranks with sglang-router using consistent -# hashing on the AIPerf correlation id so multi-turn sessions stay on the DP -# rank that holds their radix-cache prefix. -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) -fi - -# GLM-5.2 ships its own nextn head, so EAGLE runs off the checkpoint. Three -# draft tokens per step is the draft length whose golden AL is pinned below. -SPEC_ARGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -CHUNKED_PREFILL_SIZE=8192 -if [ "$DP_ATTENTION" = "true" ]; then - # chunked-prefill-size is a whole-engine budget split across DP ranks: - # 8192 becomes 1,024 tokens/rank/step under dp8 and a conc-256 warmup - # could not drain within AIPerf's 1800 s grace period. 32768 = ~4096/rank. - CHUNKED_PREFILL_SIZE=32768 - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --tokenizer-worker-num "$TP" - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - ) - # The nextn layer is unquantized (hf_quant_config excludes model.layers.78*), - # so the draft MoE is bf16 and pinned to the triton runner; inheriting the - # target's FlashInfer all-to-all dies at init with "Pre-permute function for - # flashinfer to triton is not registered". SGLang only applies its fix on - # is_hip(), so set the ROCm values explicitly. Only matters with EP a2a. - SPEC_ARGS+=( - --speculative-moe-a2a-backend none - --speculative-moe-runner-backend triton - ) -else - # Cookbook low-latency levers; the DP-attention cell omits them. - PARALLEL_ARGS+=( - --kv-cache-dtype fp8_e4m3 - --bf16-gemm-backend cutedsl - --max-prefill-tokens 8192 - ) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -GRAPH_ARGS=() -if [ "$DP_ATTENTION" != "true" ]; then - # --cuda-graph-max-bs counts requests, not verification tokens; SGLang's - # spec-decode graph runner scales by --speculative-num-draft-tokens itself. - CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS - [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") -fi - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Agentic warmup dispatches hundreds of large prompts at once; allow up to -# 15 minutes of TCP progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# AIPerf pins one pooled keep-alive connection per session while uvicorn's -# default keep-alive is 5 s; an inter-turn idle gap can reuse a socket as the -# server closes it (ECONNRESET, terminal warmup failure). -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# Acceptance is pinned to the committed golden AL (golden_al_distribution/README.md): -# 2.99 is glm5.2_mtp.yaml at num_speculative_tokens=3, thinking_on. One curve -# per model: collected on FP8, and the NVFP4 checkpoint ships the same nextn head. -# SGLANG_SIMULATE_ACC_TOKEN_MODE exists from SGLang v0.5.16, which is why the -# image is pinned there; an older image silently honors ACC_LEN/ACC_METHOD and -# ignores the token mode. -# EVAL_ONLY leaves it off: simulated acceptance commits drafted tokens -# regardless of target logits and the eval would score ~0. -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.99 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --quantization modelopt_fp4 - # GLM-5.2 emits the GLM-4.7 tool-call format; glm45 leaves calls as raw - # text and the SWE-bench eval dies with RepeatedFormatError. Neither parser - # affects replay throughput. - --tool-call-parser glm47 - --reasoning-parser glm45 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --mem-fraction-static 0.85 - --max-running-requests "$MAX_RUNNING_REQUESTS" - "${SPEC_ARGS[@]}" - "${GRAPH_ARGS[@]}" - "${CACHE_ARGS[@]}" - --watchdog-timeout 1800 - --enable-metrics -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_SIMULATE_ACC_* env vars at launch (empty => real verification) ===" - env | grep -E '^SGLANG_SIMULATE_ACC_' | sort || true - echo "============================================================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B300..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - python3 -m sglang_router.launch_router \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - # The chat template defaults to reasoning_effort=Max when no - # chat_template_kwargs are passed (mini-swe-agent passes none), and the heavy - # thinking burns the shared 75-step budget (12/23 exited LimitsExceeded). - export SWEBENCH_AGENT_STEP_LIMIT=150 - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh deleted file mode 100644 index 74aaf037a6..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - - export EVAL_FRAMEWORK="lm-eval" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars EVAL_ONLY - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# ROCR/HIP visibility under slurm cgroups. -if [ -n "$ROCR_VISIBLE_DEVICES" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# A server killed minutes earlier can still be draining HBM (KFD reclaim takes -# minutes), and booting into a half-drained node fails RCCL init with HIP -# 'unhandled cuda error'. Idle GPUs sit at up to ~4% VRAM, draining ones at -# 50-90%, so require every GPU <= 10%. -GPU_CLEAN=false -for i in $(seq 1 90); do - VRAM_MAX=$(rocm-smi --showmemuse 2>/dev/null | grep -oE "GPU Memory Allocated \(VRAM%\): [0-9]+" | awk '{if ($NF > m) m = $NF} END {print m+0}') - if [ "${VRAM_MAX:-0}" -le 10 ]; then echo "GPUs clean (vram%max=$VRAM_MAX after $((i*10))s)"; GPU_CLEAN=true; break; fi - echo "waiting for prior-job GPU memory reclaim: vram%max=$VRAM_MAX"; sleep 10 -done -[ "$GPU_CLEAN" = "true" ] || { echo "Error: GPUs still draining prior job's memory after 15min" >&2; exit 1; } - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -mkdir -p "$RESULT_DIR" - -export PYTHONNOUSERSITE=1 -# Agentic warmup dispatches hundreds of large prompts at once; allow up to -# 15 minutes of TCP progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# AIPerf pins one pooled keep-alive connection per session while uvicorn's -# default keep-alive is 5 s; outlast the client pool so the reuse race cannot occur. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 -# Use the precompiled HIP Top-K path; at c8 it improves P90 interactivity with -# essentially unchanged throughput per GPU compared with fused Top-K v2. -export SGLANG_OPT_USE_TOPK_V2=false - -# HiCache L2 (host DRAM), optionally with Mooncake L3. The TP4 HiCache arm -# uses 180 GB/rank; including the DSA indexer, it leaves about 125 GB inside -# the SA runner's 0.85 DRAM budget. Dormant DP and Mooncake paths retain ratio sizing. -CACHE_ARGS=() -if agentic_kv_offload_enabled; then - if [ "$DP_ATTENTION" = "true" ]; then - HICACHE_RATIO="0.5" - else - HICACHE_RATIO="1.0" - HICACHE_SIZE_GB="180" - fi - HICACHE_WRITE_POLICY="write_through" - case "$KV_OFFLOAD_BACKEND" in - hicache) - if [ "$DP_ATTENTION" = "true" ]; then - HICACHE_SIZING_ARGS=(--hicache-ratio "$HICACHE_RATIO") - echo "HiCache (GPU+host DRAM only): ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY" - else - HICACHE_SIZING_ARGS=(--hicache-size "$HICACHE_SIZE_GB") - echo "HiCache (GPU+host DRAM only): size=${HICACHE_SIZE_GB}GB/rank, write_policy=$HICACHE_WRITE_POLICY" - fi - CACHE_ARGS=( - --enable-hierarchical-cache - "${HICACHE_SIZING_ARGS[@]}" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) - ;; - mooncake) - L3_PER_RANK_GB="40" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - echo "HiCache+Mooncake: ratio=$HICACHE_RATIO, l3_per_rank=${L3_PER_RANK_GB} GB, dram_budget=${TOTAL_CPU_DRAM_GB} GB" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-size 0 - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-storage-backend mooncake - --hicache-storage-prefetch-policy wait_complete - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND '$KV_OFFLOAD_BACKEND' (expected: hicache or mooncake)" >&2 - exit 1 - ;; - esac -fi - -# The DP-attention arm is dormant (no dp-attn arms in amd-master.yaml): DSA + -# dp-attention hangs a collective under long-context prefill on ROCm v0.5.14 -# (watchdog kills the scheduler with zero completions). When active, -# sglang-router fronts the DP ranks with consistent hashing on the AIPerf -# correlation id so sessions stay on the rank holding their prefix. -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -MEM_FRACTION_STATIC=0.85 -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router) - PARALLEL_ARGS+=(--dp "$TP" --enable-dp-attention) - CHUNKED_PREFILL_SIZE=32768 - export AGENTIC_WARMUP_GRACE_PERIOD=3600 - # gatherv/reduce-scatter DP collectives (the only green DP-attention config - # on this cluster/image); the defaults hang under long-context prefill. - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 -elif [ "$CONC" -le 16 ]; then - # 32k chunks let the scheduler interleave decode between prefill chunks; - # they also drop per-chunk activation headroom from ~7 GiB to ~1.7 GiB per - # rank, which is what makes 0.85 safe (it OOMed at 131k chunks). - CHUNKED_PREFILL_SIZE=32768 - MEM_FRACTION_STATIC=0.85 -else - CHUNKED_PREFILL_SIZE=32768 - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi -# 2×CONC in-flight slots: MTP draft+verify transiently batches more tokens -# than CONC sessions; headroom prevents scheduler stalls under burst. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -[ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 -# Cap at 64 to bound graph-capture memory. --cuda-graph-max-bs was a deprecated -# alias for the decode setting and the 20260910 image removed it. -CUDA_GRAPH_MAX_BS_DECODE=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) - -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.61 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --kv-cache-dtype fp8_e4m3 - --dsa-prefill-backend triton - --dsa-decode-backend triton - # GLM-5.2 emits the GLM-4.7-style tool-call format; glm47 is required for - # structured message.tool_calls (SWE-bench agentic evals die without it). - # The glm45 reasoning parser keeps hybrid thinking in reasoning_content. - --tool-call-parser glm47 - --reasoning-parser glm45 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE" - --speculative-algorithm EAGLE - --speculative-num-steps 5 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 6 - "${CACHE_ARGS[@]}" - --watchdog-timeout 1800 - --enable-metrics -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -echo "Starting SGLang server for MI355X..." -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - "${SGLANG_ROUTER_CMD[@]}" \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh deleted file mode 100755 index 52522dc40f..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp8_b200_sglang_mtp.sh +++ /dev/null @@ -1,386 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for GLM-5.2 FP8 on B200 using SGLang with -# EAGLE/MTP speculative decoding. First GLM-5.2 FP8 AgentX recipe on B200; it -# uses speculative decoding. MODELS.md also permits non-speculative recipes -# that contribute to the Pareto frontier; separate A/B baselines are not required. -# -# Port of the validated agentic/glm5.2_fp4_b200_sglang_mtp.sh. The FP8 deltas -# are the blocks marked "FP8:" below -- the checkpoint (zai-org/GLM-5.2-FP8, -# ~756 GB of block-quantized e4m3 weights against ~465 GB for GLM-5.2-NVFP4), -# --quantization fp8 in place of modelopt_fp4, and the memory notes that follow -# from the larger resident weights. Everything else is the NVFP4 B200 script -# unchanged so the two precision curves stay comparable. -# -# Server flags follow the SGLang cookbook GLM-5.x single-node recipes -# (https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.2; the published -# GLM-5.1-FP8 cookbook entry uses the same EAGLE shape with quantization: fp8): -# DP_ATTENTION=false -> low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM) -# DP_ATTENTION=true -> high-throughput DEP arm (TP8 + DP8 attention-DP + -# EP_SIZE expert-parallel MoE via --ep-size) -# Only the low-latency arm is wired into the master config for this MTP recipe -# (see the entry comment on glm5.2-fp8-b200-sglang-agentic-mtp); the DEP branch -# is kept intact so the throughput arm can be added without re-deriving it. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE, DP_ATTENTION -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# B200: runners/launch_b200-nscale-slurm.sh resolves the checkpoint to a -# cluster-local path and then rewrites MODEL to that path, so `hf download -# "$MODEL"` cannot work on this runner. Keep the HF repo id separate for the -# day-zero case where GLM-5.2-FP8 has not been staged yet. -# FP8: the upstream zai-org release; the golden AL below was measured on it. -HF_MODEL_ID="${HF_MODEL_ID:-zai-org/GLM-5.2-FP8}" - -# A non-empty directory is NOT a staged checkpoint. The NVFP4 sibling found -# /lustre/fsw/gharunners/models/GLM-5.2-NVFP4 holding config.json, -# generation_config.json, hf_quant_config.json, chat_template.jinja, README.md -# and .quant_summary.txt and NOTHING else -- an aborted or metadata-only pull. -# An `ls -A` emptiness guard accepts that, so all five cells of run 30729467646 -# skipped the download and went straight to serve; SGLang read config.json -# fine, then died in AutoTokenizer.from_pretrained with "Couldn't instantiate -# the backend tokenizer" because neither the tokenizer files nor a single -# weight shard were on disk. Check for a COMPLETE checkpoint instead: the -# tokenizer, the shard index, and every shard the index names. -checkpoint_is_complete() { - local dir="$1" - [[ -d "$dir" ]] || return 1 - [[ -f "$dir/tokenizer_config.json" ]] || return 1 - [[ -f "$dir/tokenizer.json" || -f "$dir/tokenizer.model" ]] || return 1 - [[ -f "$dir/model.safetensors.index.json" ]] || return 1 - CKPT_DIR="$dir" python3 - <<'PYEOF' -import json, os, sys -d = os.environ["CKPT_DIR"] -with open(os.path.join(d, "model.safetensors.index.json")) as fh: - shards = sorted(set(json.load(fh)["weight_map"].values())) -missing = [s for s in shards if not os.path.isfile(os.path.join(d, s))] -if missing: - print(f"{len(missing)}/{len(shards)} shards missing, e.g. {missing[:3]}", file=sys.stderr) - sys.exit(1) -PYEOF -} - -if [[ -n "${MODEL_PATH:-}" ]]; then - if ! checkpoint_is_complete "$MODEL_PATH"; then - # Every concurrency of this sweep runs as its own allocation against - # the same shared path, so serialize: one cell pulls the ~756 GB - # checkpoint (141 shards) and the rest wait on it rather than five - # racing writers. `hf download` resumes into a partially-populated - # --local-dir, so a metadata-only stub is fine to download on top of. - mkdir -p "$MODEL_PATH" - MODEL_DOWNLOAD_LOCK="${MODEL_PATH%/}.download.lock" - echo "Checkpoint at $MODEL_PATH is incomplete; acquiring $MODEL_DOWNLOAD_LOCK" - exec 9>"$MODEL_DOWNLOAD_LOCK" - flock -w "${MODEL_DOWNLOAD_LOCK_TIMEOUT:-21600}" 9 || { - echo "Error: timed out waiting for another cell to stage $MODEL_PATH" >&2 - exit 1 - } - if checkpoint_is_complete "$MODEL_PATH"; then - echo "Another cell staged $MODEL_PATH while we waited" - else - hf download "$HF_MODEL_ID" --local-dir "$MODEL_PATH" - fi - flock -u 9 - exec 9>&- - checkpoint_is_complete "$MODEL_PATH" || { - echo "Error: $MODEL_PATH is still incomplete after hf download $HF_MODEL_ID." >&2 - exit 1 - } - fi -else - hf download "$HF_MODEL_ID" - export MODEL_PATH="$HF_MODEL_ID" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention: prefixes evicted from the HBM KV pool - # spill to a pinned host pool instead of being recomputed. On the - # 1M-context agentic corpus the live working set outgrows HBM past - # conc 8 (TP8) / 64 (DP8) and the radix hit rate collapses to <0.1 - # against a ~0.97 theoretical ceiling, so every turn re-prefills its - # whole history; the host tier restores those hits at C2C bandwidth. - # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every TP rank holds - # complete per-token KV. The ratio 0.75 sizes the host pool off the HBM KV - # pool, which is smaller here than on NVFP4 (see MEM_FRACTION_STATIC), so - # ratio mode yields a proportionally smaller pinned tier at c1-c8. Use the - # same 169 GB/rank absolute target pool at c12/c16 as the NVFP4 sibling: - # that figure is bounded by host DRAM, which the precision does not change. - # - # cluster:b200-nscale advertises 2,063,920 MiB and this config exposes 80%, - # giving the benchmark 1,731 GB. A 169 GB/rank packed target+MTP pool plus - # the coupled 38.73 GB/rank DSA indexer uses about 1,662 GB across TP8. - # Keep the 270 GB ceiling so deployments with more usable host DRAM can - # explicitly override the default. - DEFAULT_HICACHE_RATIO=0.75 - DEFAULT_HICACHE_SIZE=0 - case "$CONC" in - 12|16) DEFAULT_HICACHE_SIZE=169 ;; - esac - MAX_HICACHE_SIZE=270 - HICACHE_SIZE="${HICACHE_SIZE:-$DEFAULT_HICACHE_SIZE}" - if ! [[ "$HICACHE_SIZE" =~ ^[0-9]+$ ]]; then - echo "Error: HICACHE_SIZE must be a non-negative integer, got $HICACHE_SIZE" >&2 - exit 1 - fi - if awk -v s="$HICACHE_SIZE" -v cap="$MAX_HICACHE_SIZE" 'BEGIN { exit !(s > cap) }'; then - echo "Error: HICACHE_SIZE=$HICACHE_SIZE exceeds configured limit $MAX_HICACHE_SIZE" >&2 - exit 1 - fi - HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - if awk -v s="$HICACHE_SIZE" 'BEGIN { exit !(s > 0) }'; then - echo "HiCache CPU tier: target_size=$HICACHE_SIZE GB, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" - CACHE_ARGS+=(--hicache-size "$HICACHE_SIZE") - else - if awk -v r="$HICACHE_RATIO" -v cap="$DEFAULT_HICACHE_RATIO" 'BEGIN { exit !(r > cap) }'; then - echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2 - exit 1 - fi - echo "HiCache CPU tier: ratio=$HICACHE_RATIO, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" - CACHE_ARGS+=(--hicache-ratio "$HICACHE_RATIO") - fi -fi - -# With attention-DP, front the DP ranks with sglang-router using consistent -# hashing on the AIPerf correlation id so multi-turn sessions stay on the DP -# rank that holds their radix-cache prefix. -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) -fi - -# MTP: GLM-5.2 ships its own nextn head (num_nextn_predict_layers=1), so EAGLE -# runs off the checkpoint with no external draft model. num-steps 3 / -# eagle-topk 1 / num-draft-tokens 4 is 3 speculative tokens per verification -# step -- the same shape the NVFP4 B200/B300 siblings and the GLM-5.1-FP8 -# cookbook speculative-mtp entry use, and the draft length whose golden AL is -# pinned below. -SPEC_ARGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -CHUNKED_PREFILL_SIZE=8192 -if [ "$DP_ATTENTION" = "true" ]; then - # chunked-prefill-size is a whole-engine budget split across DP ranks: - # the cookbook HT cell's 8192 becomes 1,024 tokens/rank/step under dp8, - # which starves prefill on the 1M-context agentic corpus (observed: a - # conc-256 warmup could not drain within AIPerf's 1800s grace period - # while KV usage sat at ~0.01). Use the cookbook's own dp8 lever from - # the B200 cells (32768 = ~4096/rank). - CHUNKED_PREFILL_SIZE=32768 - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --tokenizer-worker-num "$TP" - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - ) - # Carried over from the NVFP4 sibling, where the draft MoE is bf16 - # (hf_quant_config excludes model.layers.78*) and inheriting the target - # model's FlashInfer all-to-all dies at init with "Pre-permute function - # for flashinfer to triton is not registered". FP8: GLM-5.2-FP8 quantizes - # the nextn experts like every other layer (modules_to_not_convert lists - # only layer-78 norms and biases), so the draft MoE takes the FP8 runner - # and this pin is likely unnecessary here; it is kept so the DEP arm, if - # wired, starts from the configuration that is known to boot. Only - # relevant once expert parallelism puts an a2a in the MoE path -- the - # plain-TP arm below has none. - SPEC_ARGS+=( - --speculative-moe-a2a-backend none - --speculative-moe-runner-backend triton - ) -else - # Cookbook low-latency levers; the DP-attention cell omits them. - PARALLEL_ARGS+=( - --kv-cache-dtype fp8_e4m3 - --bf16-gemm-backend cutedsl - --max-prefill-tokens 8192 - ) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -GRAPH_ARGS=() -if [ "$DP_ATTENTION" != "true" ]; then - # Cookbook low-latency captures graphs up to its request cap; the - # DP-attention cell leaves the CUDA-graph batch list at SGLang defaults. - # --cuda-graph-max-bs counts requests, not verification tokens: SGLang's - # spec-decode graph runner scales each captured batch by - # --speculative-num-draft-tokens itself. - CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS - [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") -fi - -# B200: 180 GB HBM3e per GPU. 0.83 leaves ~31 GB of non-static headroom for -# the EAGLE draft head's verification-batch activations, the extra CUDA-graph -# capture at 4 draft tokens, and GLM-5.2's DSA indexer temporaries; that -# headroom is a fraction of the card and does not depend on the checkpoint. -# FP8: what does change is the KV pool inside the static share. The ~756 GB -# checkpoint is ~94.5 GB/GPU across TP8 (NVFP4: ~58 GB/GPU), so the fp8 KV -# pool is roughly 55 GB/GPU here against ~91 GB/GPU on NVFP4. HiCache absorbs -# the difference as host spill; if c12/c16 show HBM pressure, raise the -# fraction here (overridable) before touching the concurrency grid. -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.83}" - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Each concurrency in a full sweep is a separate Slurm allocation, while the -# Nscale home directory is shared. Keep SGLang's FlashInfer autotune, Triton, -# Inductor, and CUDA JIT caches allocation-local so concurrent cells cannot -# overwrite the same per-rank runtime-cache files. Non-Slurm launchers can -# provide an explicit SGLANG_CACHE_DIR override. -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - export SGLANG_CACHE_DIR="${SGLANG_CACHE_DIR:-/tmp/sglang-cache-${SLURM_JOB_ID}}" -fi -# Agentic warmup dispatches hundreds of large prompts at once; allow up to -# 15 minutes of TCP progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# AIPerf pins one pooled keep-alive connection per session (client-side -# keep-alive 300s) while uvicorn's default SGLANG_TIMEOUT_KEEP_ALIVE is 5s; -# inter-turn idle gaps (capped at 10s) can reuse a socket exactly as the -# server closes it -> ECONNRESET -> terminal warmup failure. Outlast the -# client pool so the race cannot occur. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX pins acceptance to the committed golden AL so submissions are compared -# on system performance at a fixed acceptance target rather than on draft-head -# quality (golden_al_distribution/README.md). 2.99 is the GLM-5.2 curve at -# num_speculative_tokens=3, thinking_on -# (golden_al_distribution/glm5.2_mtp.yaml, SPEED-Bench coding, run 28058352479). -# FP8: that curve was measured on this very checkpoint (glm-5.2-fp8), so no -# cross-precision assumption is involved here. -# -# SGLANG_SIMULATE_ACC_TOKEN_MODE only exists from SGLang v0.5.16. An older -# image would silently honor ACC_LEN/ACC_METHOD and ignore the token-mode half -# of the contract. -# -# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens -# regardless of the target logits, so generated text is wrong and the eval -# would score ~0. -if [ "${EVAL_ONLY:-false}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.99 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - # FP8: zai-org/GLM-5.2-FP8 ships quantization_config quant_method=fp8 with - # 128x128 weight blocks and dynamic e4m3 activations; pass the method - # explicitly as the Qwen3.5 FP8 B200/B300 SGLang siblings do. - --quantization fp8 - # GLM-5.2 emits the GLM-4.7-style // format; - # the glm47 parser is required for structured message.tool_calls (glm45 - # leaves calls as raw text). Without it the SWE-bench mini-swe-agent eval - # dies with RepeatedFormatError ("No tool calls found in the response") on - # every instance and scores 0. Reasoning parser keeps hybrid-thinking - # output in reasoning_content instead of polluting content. Neither flag - # affects trace-replay throughput (pre-canned replay discards live - # responses). - --tool-call-parser glm47 - --reasoning-parser glm45 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --max-running-requests "$MAX_RUNNING_REQUESTS" - "${SPEC_ARGS[@]}" - "${GRAPH_ARGS[@]}" - "${CACHE_ARGS[@]}" - --watchdog-timeout 1800 - --enable-metrics -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_SIMULATE_ACC_* env vars at launch (empty => real verification) ===" - env | grep -E '^SGLANG_SIMULATE_ACC_' | sort || true - echo "============================================================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B200..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - python3 -m sglang_router.launch_router \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - # GLM-5.2's chat template defaults to reasoning_effort=Max when the - # client passes no chat_template_kwargs (mini-swe-agent doesn't), and the - # heavy thinking burns the default 75-step budget: on the 23-instance - # slice, 12/23 trajectories exited LimitsExceeded unsubmitted while 10 of - # the 11 that submitted resolved. Double the step budget for this recipe; - # other recipes keep the shared 75 default. - export SWEBENCH_AGENT_STEP_LIMIT=150 - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp8_b300_sglang_mtp.sh deleted file mode 100755 index b7a9c7ea8c..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp8_b300_sglang_mtp.sh +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for GLM-5.2 FP8 on B300 using SGLang with -# EAGLE/MTP speculative decoding. First GLM-5.2 FP8 AgentX recipe on B300; it -# uses speculative decoding. MODELS.md also permits non-speculative recipes -# that contribute to the Pareto frontier; separate A/B baselines are not required. -# -# Port of agentic/glm5.2_fp8_b200_sglang_mtp.sh (the FP8 B200 sibling, itself a -# port of the validated NVFP4 B200 script) with the B300 deltas the NVFP4 B300 -# sibling (agentic/glm5.2_fp4_b300_sglang_mtp.sh) carries, marked "B300:" -# below: --mem-fraction-static 0.85 on the 288 GB part and a fixed 270 GB/rank -# HiCache target pool at every concurrency. The FP8 deltas, marked "FP8:", are -# the checkpoint (zai-org/GLM-5.2-FP8, ~756 GB of block-quantized e4m3 weights -# against ~465 GB for GLM-5.2-NVFP4) and --quantization fp8 in place of -# modelopt_fp4. Serve flags are otherwise identical to both siblings so the -# FP8/NVFP4 curves on B300 and the FP8 curves across B200/B300 stay comparable. -# -# Server flags follow the SGLang cookbook GLM-5.x single-node recipes -# (https://docs.sglang.io/cookbook/autoregressive/GLM/GLM-5.2; the published -# GLM-5.1-FP8 cookbook entry uses the same EAGLE shape with quantization: fp8): -# DP_ATTENTION=false -> low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM) -# DP_ATTENTION=true -> high-throughput DEP arm (TP8 + DP8 attention-DP + -# EP_SIZE expert-parallel MoE via --ep-size) -# Only the low-latency arm is wired into the master config for this MTP recipe -# (see the entry comment on glm5.2-fp8-b300-sglang-agentic-mtp); the DEP branch -# is kept intact so the throughput arm can be added without re-deriving it. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE, DP_ATTENTION -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# B300: runners/launch_b300-dsxe.sh lists GLM-5.2-FP8 in STAGED_MODELS, so it -# exports MODEL_PATH=/scratch/models/GLM-5.2-FP8 (node-local NVMe, read-only -# from the job's point of view) and probes config.json on the allocated node -# before this script runs. The checkpoint is pre-staged there like the other -# DSXE models; this script never downloads it. -# FP8: the upstream zai-org release; the golden AL below was measured on it. - -# A non-empty directory is NOT a staged checkpoint. The NVFP4 B200 sibling found -# /lustre/fsw/gharunners/models/GLM-5.2-NVFP4 holding config.json, -# generation_config.json, hf_quant_config.json, chat_template.jinja, README.md -# and .quant_summary.txt and NOTHING else -- an aborted or metadata-only pull. -# An `ls -A` emptiness guard accepts that, so all five cells of run 30729467646 -# skipped straight to serve; SGLang read config.json fine, then died in -# AutoTokenizer.from_pretrained with "Couldn't instantiate the backend -# tokenizer" because neither the tokenizer files nor a single weight shard were -# on disk. Check for a COMPLETE checkpoint instead: the tokenizer, the shard -# index, and every shard the index names -- and fail fast if anything is -# missing rather than pulling ~756 GB onto the read-only staged root. -checkpoint_is_complete() { - local dir="$1" - [[ -d "$dir" ]] || return 1 - [[ -f "$dir/tokenizer_config.json" ]] || return 1 - [[ -f "$dir/tokenizer.json" || -f "$dir/tokenizer.model" ]] || return 1 - [[ -f "$dir/model.safetensors.index.json" ]] || return 1 - CKPT_DIR="$dir" python3 - <<'PYEOF' -import json, os, sys -d = os.environ["CKPT_DIR"] -with open(os.path.join(d, "model.safetensors.index.json")) as fh: - shards = sorted(set(json.load(fh)["weight_map"].values())) -missing = [s for s in shards if not os.path.isfile(os.path.join(d, s))] -if missing: - print(f"{len(missing)}/{len(shards)} shards missing, e.g. {missing[:3]}", file=sys.stderr) - sys.exit(1) -PYEOF -} - -if [[ -n "${MODEL_PATH:-}" ]]; then - checkpoint_is_complete "$MODEL_PATH" || { - echo "Error: staged checkpoint at $MODEL_PATH is incomplete. Stage zai-org/GLM-5.2-FP8 there before running; this recipe does not download it." >&2 - exit 1 - } -else - # Stand-alone runs outside the launcher: hand SGLang the HF id and let it - # resolve the checkpoint from HF_HUB_CACHE. - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention: prefixes evicted from the HBM KV pool - # spill to a pinned host pool instead of being recomputed. On the - # 1M-context agentic corpus the live working set outgrows HBM past - # conc 8 (TP8) and the radix hit rate collapses to <0.1 against a ~0.97 - # theoretical ceiling, so every turn re-prefills its whole history; the - # host tier restores those hits at C2C bandwidth. - # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every TP rank holds - # complete per-token KV. - # - # B300: use the NVFP4 B300 sibling's fixed 270 GB/rank target pool at every - # concurrency (#2651) rather than the B200 recipe's ratio mode at c1-c8. - # From SGLang v0.5.16 --hicache-size sizes only the target KV host pool; - # the coupled DSA indexer host pool (~38.73 GB/rank) is allocated on top. - # cluster:b300-dsxe advertises 3,977,095 MiB of host DRAM and this config - # exposes 80% (~3.18 TB); 270 GB/rank target + indexer across TP8 uses - # about 2.47 TB, leaving startup headroom. Overridable via HICACHE_SIZE. - DEFAULT_HICACHE_SIZE=270 - MAX_HICACHE_SIZE=270 - HICACHE_SIZE="${HICACHE_SIZE:-$DEFAULT_HICACHE_SIZE}" - if ! [[ "$HICACHE_SIZE" =~ ^[0-9]+$ ]]; then - echo "Error: HICACHE_SIZE must be a positive integer, got $HICACHE_SIZE" >&2 - exit 1 - fi - if awk -v s="$HICACHE_SIZE" -v cap="$MAX_HICACHE_SIZE" 'BEGIN { exit !(s <= 0 || s > cap) }'; then - echo "Error: HICACHE_SIZE=$HICACHE_SIZE must be in (0, $MAX_HICACHE_SIZE]" >&2 - exit 1 - fi - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - echo "HiCache CPU tier: conc=$CONC, target_size=$HICACHE_SIZE GB, total_capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) -fi - -# With attention-DP, front the DP ranks with sglang-router using consistent -# hashing on the AIPerf correlation id so multi-turn sessions stay on the DP -# rank that holds their radix-cache prefix. -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -ROUTER_LOG="$RESULT_DIR/router.log" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) -fi - -# MTP: GLM-5.2 ships its own nextn head (num_nextn_predict_layers=1), so EAGLE -# runs off the checkpoint with no external draft model. num-steps 3 / -# eagle-topk 1 / num-draft-tokens 4 is 3 speculative tokens per verification -# step -- the same shape the NVFP4 B200/B300 siblings and the GLM-5.1-FP8 -# cookbook speculative-mtp entry use, and the draft length whose golden AL is -# pinned below. -SPEC_ARGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -CHUNKED_PREFILL_SIZE=8192 -if [ "$DP_ATTENTION" = "true" ]; then - # chunked-prefill-size is a whole-engine budget split across DP ranks: - # the cookbook HT cell's 8192 becomes 1,024 tokens/rank/step under dp8, - # which starves prefill on the 1M-context agentic corpus (observed: a - # conc-256 warmup could not drain within AIPerf's 1800s grace period - # while KV usage sat at ~0.01). Use the cookbook's own dp8 lever from - # the B200 cells (32768 = ~4096/rank). - CHUNKED_PREFILL_SIZE=32768 - PARALLEL_ARGS+=( - --dp "$TP" - --enable-dp-attention - --tokenizer-worker-num "$TP" - --dist-init-addr "127.0.0.1:$((PORT + 2000))" - ) - # Carried over from the NVFP4 sibling, where the draft MoE is bf16 - # (hf_quant_config excludes model.layers.78*) and inheriting the target - # model's FlashInfer all-to-all dies at init with "Pre-permute function - # for flashinfer to triton is not registered". FP8: GLM-5.2-FP8 quantizes - # the nextn experts like every other layer (modules_to_not_convert lists - # only layer-78 norms and biases), so the draft MoE takes the FP8 runner - # and this pin is likely unnecessary here; it is kept so the DEP arm, if - # wired, starts from the configuration that is known to boot. Only - # relevant once expert parallelism puts an a2a in the MoE path -- the - # plain-TP arm below has none. - SPEC_ARGS+=( - --speculative-moe-a2a-backend none - --speculative-moe-runner-backend triton - ) -else - # Cookbook low-latency levers; the DP-attention cell omits them. - PARALLEL_ARGS+=( - --kv-cache-dtype fp8_e4m3 - --bf16-gemm-backend cutedsl - --max-prefill-tokens 8192 - ) -fi - -# AgentX concurrency counts live session trees, not individual requests. -# Allow subagent fan-out to exceed CONC without clipping request bursts. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -GRAPH_ARGS=() -if [ "$DP_ATTENTION" != "true" ]; then - # Cookbook low-latency captures graphs up to its request cap; the - # DP-attention cell leaves the CUDA-graph batch list at SGLang defaults. - # --cuda-graph-max-bs counts requests, not verification tokens: SGLang's - # spec-decode graph runner scales each captured batch by - # --speculative-num-draft-tokens itself. - CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS - [ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - GRAPH_ARGS=(--cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS") -fi - -# B300: 288 GB HBM3e per GPU. The NVFP4 B300 sibling runs 0.85, leaving ~43 GB -# of non-static headroom for the EAGLE draft head's verification-batch -# activations, the extra CUDA-graph capture at 4 draft tokens, and GLM-5.2's -# DSA indexer temporaries; that headroom is a fraction of the card and does -# not depend on the checkpoint. -# FP8: what does change is the KV pool inside the static share. The ~756 GB -# checkpoint is ~94.5 GB/GPU across TP8 (NVFP4: ~58 GB/GPU), so the fp8 KV -# pool is roughly 150 GB/GPU here against ~187 GB/GPU on NVFP4 B300 (and -# ~55 GB/GPU on FP8 B200). HiCache absorbs the difference as host spill. -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.85}" - -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Each concurrency in a full sweep is a separate Slurm allocation, while the -# DSXE runner home directory is shared. Keep SGLang's FlashInfer autotune, Triton, -# Inductor, and CUDA JIT caches allocation-local so concurrent cells cannot -# overwrite the same per-rank runtime-cache files. Non-Slurm launchers can -# provide an explicit SGLANG_CACHE_DIR override. -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - export SGLANG_CACHE_DIR="${SGLANG_CACHE_DIR:-/tmp/sglang-cache-${SLURM_JOB_ID}}" -fi -# Agentic warmup dispatches hundreds of large prompts at once; allow up to -# 15 minutes of TCP progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# AIPerf pins one pooled keep-alive connection per session (client-side -# keep-alive 300s) while uvicorn's default SGLANG_TIMEOUT_KEEP_ALIVE is 5s; -# inter-turn idle gaps (capped at 10s) can reuse a socket exactly as the -# server closes it -> ECONNRESET -> terminal warmup failure. Outlast the -# client pool so the race cannot occur. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 - -# AgentX pins acceptance to the committed golden AL so submissions are compared -# on system performance at a fixed acceptance target rather than on draft-head -# quality (golden_al_distribution/README.md). 2.99 is the GLM-5.2 curve at -# num_speculative_tokens=3, thinking_on -# (golden_al_distribution/glm5.2_mtp.yaml, SPEED-Bench coding, run 28058352479). -# FP8: that curve was measured on this very checkpoint (glm-5.2-fp8), so no -# cross-precision assumption is involved here. -# -# SGLANG_SIMULATE_ACC_TOKEN_MODE only exists from SGLang v0.5.16. An older -# image would silently honor ACC_LEN/ACC_METHOD and ignore the token-mode half -# of the contract. -# -# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens -# regardless of the target logits, so generated text is wrong and the eval -# would score ~0. -if [ "${EVAL_ONLY:-false}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=2.99 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - # FP8: zai-org/GLM-5.2-FP8 ships quantization_config quant_method=fp8 with - # 128x128 weight blocks and dynamic e4m3 activations; pass the method - # explicitly as the Qwen3.5 FP8 B200/B300 SGLang siblings do. - --quantization fp8 - # GLM-5.2 emits the GLM-4.7-style // format; - # the glm47 parser is required for structured message.tool_calls (glm45 - # leaves calls as raw text). Without it the SWE-bench mini-swe-agent eval - # dies with RepeatedFormatError ("No tool calls found in the response") on - # every instance and scores 0. Reasoning parser keeps hybrid-thinking - # output in reasoning_content instead of polluting content. Neither flag - # affects trace-replay throughput (pre-canned replay discards live - # responses). - --tool-call-parser glm47 - --reasoning-parser glm45 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --max-running-requests "$MAX_RUNNING_REQUESTS" - "${SPEC_ARGS[@]}" - "${GRAPH_ARGS[@]}" - "${CACHE_ARGS[@]}" - --watchdog-timeout 1800 - --enable-metrics -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_SIMULATE_ACC_* env vars at launch (empty => real verification) ===" - env | grep -E '^SGLANG_SIMULATE_ACC_' | sort || true - echo "============================================================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for B300..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - python3 -m sglang_router.launch_router \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - # GLM-5.2's chat template defaults to reasoning_effort=Max when the - # client passes no chat_template_kwargs (mini-swe-agent doesn't), and the - # heavy thinking burns the default 75-step budget: on the 23-instance - # slice, 12/23 trajectories exited LimitsExceeded unsubmitted while 10 of - # the 11 that submitted resolved. Double the step budget for this recipe; - # other recipes keep the shared 75 default. - export SWEBENCH_AGENT_STEP_LIMIT=150 - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh deleted file mode 100755 index 990e110624..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# GLM-5.2 FP8 on 8xMI325X with native EAGLE MTP; GPU-resident KV only. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -check_env_vars EVAL_ONLY - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# GLM-5.2 natively supports 1M context, so use the complete AgentX corpus. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" -SERVER_PID="" -SERVER_PGID="" - -cleanup() { - if [[ -n "$SERVER_PGID" ]]; then - kill -TERM -- "-$SERVER_PGID" 2>/dev/null || true - elif [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then - kill -TERM "$SERVER_PID" 2>/dev/null || true - fi - [[ -z "$SERVER_PID" ]] || wait "$SERVER_PID" 2>/dev/null || true -} -trap cleanup EXIT - -if agentic_kv_offload_enabled; then - echo "Error: this refreshed baseline supports GPU-resident KV only" >&2 - exit 1 -fi - -export PYTHONNOUSERSITE=1 -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" -export SGLANG_TIMEOUT_KEEP_ALIVE=900 -# The MI30X image's DSA top-k v2 JIT includes CUDA-only headers when compiled -# for gfx942. Use the portable Torch path and disable the fused top-k path. -export SGLANG_DSA_FUSE_TOPK=false -export SGLANG_OPT_USE_TOPK_V2=false - -# Golden AL: golden_al_distribution/glm5.2_mtp.yaml, thinking_on, K=3. -# Evals keep real target verification. -if [[ "${EVAL_ONLY}" != "true" ]]; then - export SGLANG_SIMULATE_ACC_LEN=2.99 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -MAX_RUNNING_REQUESTS=$((2 * CONC)) - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - --tp "$TP" - --ep-size "$EP_SIZE" - --dsa-prefill-backend tilelang - --dsa-decode-backend tilelang - --dsa-topk-backend torch - --kv-cache-dtype bfloat16 - --tool-call-parser glm47 - --reasoning-parser glm45 - --context-length 1048576 - --max-total-tokens 1048576 - --chunked-prefill-size 131072 - --mem-fraction-static 0.85 - --max-running-requests "$MAX_RUNNING_REQUESTS" - # SGLang v0.5.20 retired the deprecated --cuda-graph-max-bs alias - # (sgl-project/sglang#38375); the decode-phase flag is the same setting. - --cuda-graph-max-bs-decode "$MAX_RUNNING_REQUESTS" - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --watchdog-timeout 1800 - --enable-metrics - --enable-cache-report -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -echo "Starting SGLang server for MI325X..." -setsid "${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -SERVER_PGID=$SERVER_PID -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "${EVAL_ONLY}" == "true" ]]; then - export SWEBENCH_AGENT_STEP_LIMIT=150 - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - # Long AgentX responses admitted near the end of the measurement window can - # take several minutes to finish. Bound their post-window drain without - # extending the measured request-admission window. - REPLAY_CMD+=" --benchmark-grace-period 1800" - REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/glm5.2_fp8_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp8_mi355x_sglang_mtp.sh deleted file mode 100755 index 40d76bd646..0000000000 --- a/benchmarks/single_node/agentic/glm5.2_fp8_mi355x_sglang_mtp.sh +++ /dev/null @@ -1,288 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for GLM-5.2 FP8 on MI355X using SGLang with -# EAGLE/MTP speculative decoding. First GLM-5.2 FP8 AgentX recipe on MI355X; -# this recipe uses speculative decoding. MODELS.md also permits non-speculative -# recipes that contribute to the Pareto frontier; separate A/B baselines are not required. -# -# Port of the validated agentic/glm5.2_fp4_mi355x_sglang_mtp.sh (amd/GLM-5.2-MXFP4). -# The FP8 deltas are the blocks marked "FP8:" below -- the checkpoint -# (zai-org/GLM-5.2-FP8, ~756 GB of block-quantized e4m3 weights, 141 shards, -# against ~380 GB for the MXFP4 checkpoint), the memory notes that follow from -# the larger resident weights, and the TP8-only arm selection. Serve flags are -# otherwise the MXFP4 script unchanged so the two precision curves on this SKU -# stay comparable. The ROCm GLM-5.2 FP8 sibling on MI325X -# (agentic/glm5.2_fp8_mi325x_mtp.sh) is the precedent for serving this -# checkpoint on gfx9 SGLang: quantization is auto-detected from the -# checkpoint's quantization_config (quant_method=fp8, 128x128 weight blocks), -# so no --quantization flag is passed. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, -# EP_SIZE, DP_ATTENTION -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# ROCR/HIP visibility under slurm cgroups. -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -# FP8: runners/launch_mi355x-amds.sh mounts the NFS hf-hub cache for this -# checkpoint (like MiniMax-M3) so the ~756 GB pull happens once for the -# cluster instead of once per node-local NVMe cache. `hf download` resumes -# into a partially populated cache, so concurrent cells converge on one copy. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# A server killed on this node minutes earlier (previous job, crashed run) -# can still be draining its HBM: KFD reclaim takes minutes, and booting into a -# half-drained node fails RCCL init with HIP 'unhandled cuda error' / -# 'invalid argument'. Wait for the GPUs to come back before launching. -# Per-GPU threshold: idle nodes hold a small driver/firmware VRAM baseline -# (observed up to ~4%/GPU, node-dependent), while a draining or occupied -# GPU sits at 50-90%. Require every GPU <= 10%. -GPU_CLEAN=false -for i in $(seq 1 90); do - VRAM_MAX=$(rocm-smi --showmemuse 2>/dev/null | grep -oE "GPU Memory Allocated \(VRAM%\): [0-9]+" | awk '{if ($NF > m) m = $NF} END {print m+0}') - if [ "${VRAM_MAX:-0}" -le 10 ]; then echo "GPUs clean (vram%max=$VRAM_MAX after $((i*10))s)"; GPU_CLEAN=true; break; fi - echo "waiting for prior-job GPU memory reclaim: vram%max=$VRAM_MAX"; sleep 10 -done -[ "$GPU_CLEAN" = "true" ] || { echo "Error: GPUs still draining prior job's memory after 15min" >&2; exit 1; } - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -mkdir -p "$RESULT_DIR" - -export PYTHONNOUSERSITE=1 -# Agentic warmup dispatches hundreds of large prompts at once; allow up to -# 15 minutes of TCP progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -# AIPerf pins one pooled keep-alive connection per session (client-side -# keep-alive 300s) while uvicorn's default SGLANG_TIMEOUT_KEEP_ALIVE is 5s; -# inter-turn idle gaps can reuse a socket exactly as the server closes it. -# Outlast the client pool so the race cannot occur. -export SGLANG_TIMEOUT_KEEP_ALIVE=900 -# The DSA indexer's top-k v2 kernel (default since v0.5.14) is JIT-compiled -# from CUDA-only source (cooperative_groups.h) and cannot build for gfx950; -# v1 dispatches to the precompiled HIP op in sgl-kernel (upstream MI355X CI -# runs DSA models the same way). Still honored by v0.5.19 (environ.py). -export SGLANG_OPT_USE_TOPK_V2=false - -# HiCache L2 (host DRAM). KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. -# -# FP8: with the ~756 GB checkpoint resident at TP8 (~94.5 GB/rank) inside -# --mem-fraction-static 0.85 of 288 GB, the device KV pool is roughly 150 GB -# per rank (MXFP4 TP8: ~182.7 GB/rank). ratio 1.5 therefore pins about 1.8 TB -# of host DRAM across TP8, comfortably inside the ~3.0 TB available on -# cluster:mi355x-amds nodes (the MXFP4 TP arm's ratio 1.5 pins ~2.9 TB there). -# The agentic-coding corpus saturates any fixed DRAM pool at conc >= 10; -# ratio 2.5 yields more throughput at conc 10-16 but exceeds physical DRAM on -# these nodes and must be set via HICACHE_RATIO on nodes that can take it. -# The DP-attention arm keeps the MXFP4 script's 0.5. -CACHE_ARGS=() -if agentic_kv_offload_enabled; then - if [ "$DP_ATTENTION" = "true" ]; then - HICACHE_RATIO="${HICACHE_RATIO:-0.5}" - else - HICACHE_RATIO="${HICACHE_RATIO:-1.5}" - fi - # write_through_selective skips DRAM writes for non-reusable KV blocks, - # reducing host-bus traffic without affecting the cache hit rate. - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" - case "${KV_OFFLOAD_BACKEND:-}" in - hicache) - echo "HiCache (GPU+host DRAM only): ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) - ;; - *) - # The MXFP4 script also wires a Mooncake L3 tier; it is deliberately - # not carried here (HiCache host DRAM is the supported AMD tier). - echo "Error: unsupported KV_OFFLOAD_BACKEND '${KV_OFFLOAD_BACKEND:-}' (expected: hicache)" >&2 - exit 1 - ;; - esac -fi - -# Arm selection. FP8: TP8-only in the master config. The ~756 GB checkpoint -# does not fit below TP8 with room for KV (TP4 would leave ~56 GB/rank for the -# KV pool, DSA indexer, and EAGLE verification batches), which is also why the -# MI325X FP8 sibling is TP8-only. Two arms: GPU-resident KV at low -# concurrency (latency), HiCache host offload at conc >= 8 (throughput). -# -# NOTE: the DP-attention path below is DORMANT (no dp-attn arms in -# amd-master.yaml): DSA + dp-attention hangs a collective under long-context -# prefill on ROCm (reproduced on v0.5.14 with and without HiCache). Kept -# intact from the MXFP4 script so the arm can be enabled without re-deriving it. -USE_SGLANG_ROUTER=false -SGLANG_BACKEND_PORT="$PORT" -PARALLEL_ARGS=(--tp "$TP" --ep-size "$EP_SIZE") -MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.85}" -if [ "$DP_ATTENTION" = "true" ]; then - USE_SGLANG_ROUTER=true - export AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID=true - SGLANG_BACKEND_PORT=$((PORT + 1)) - SGLANG_ROUTER_METRICS_PORT=$((PORT + 10000)) - SGLANG_ROUTER_CMD=(python3 -m sglang_router.launch_router) - PARALLEL_ARGS+=(--dp "$TP" --enable-dp-attention) - CHUNKED_PREFILL_SIZE=32768 - export AGENTIC_WARMUP_GRACE_PERIOD=3600 - # Swap the DP gather collectives to gatherv/reduce-scatter on ROCm - # (dsv4_fp4_mi355x_sglang.sh precedent): with the defaults the DSA DP path - # hangs a collective under long-context prefill load. - export SGLANG_DP_USE_GATHERV=1 - export SGLANG_DP_USE_REDUCE_SCATTER=1 - export GPU_MAX_HW_QUEUES=5 -elif [ "$CONC" -le 16 ]; then - # Chunked prefill 32k: smaller chunks let the scheduler interleave decode - # steps between prefill chunks, reducing TPOT for concurrent sessions. - # Per-chunk activation headroom is ~1.7 GiB/rank at 32k (vs ~7 GiB/rank - # at 131k, which OOMed the MXFP4 recipe at 0.85), so 0.85 is safe here too: - # the non-static headroom is a fraction of the 288 GB card and does not - # depend on the checkpoint size. - CHUNKED_PREFILL_SIZE=32768 -else - CHUNKED_PREFILL_SIZE=32768 - export AGENTIC_WARMUP_GRACE_PERIOD=3600 -fi -# 2xCONC in-flight slots: MTP draft+verify transiently batches more tokens -# than CONC sessions; headroom prevents scheduler stalls under burst. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -[ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 -# SGLang interpolates a bs list [1..max_bs] automatically; cap at 64 to -# keep graph-capture memory bounded. Passed as --cuda-graph-max-bs-decode: -# the old --cuda-graph-max-bs alias is gone from the 2026-09 ROCm builds -# (the MXFP4 sibling already made this switch). -# keep graph-capture memory bounded without giving up coverage. -CUDA_GRAPH_MAX_BS=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) - -# AgentX pins acceptance to the committed golden AL so submissions are compared -# on system performance at a fixed acceptance target rather than on draft-head -# quality (golden_al_distribution/README.md). Same MTP depth as the MXFP4 -# MI355X sibling (num-steps 5, 6 draft tokens = 5 speculative tokens) so the -# two precision curves on this SKU share one acceptance target: 3.61 is the -# GLM-5.2 thinking_on curve at K=5 (golden_al_distribution/glm5.2_mtp.yaml, -# SPEED-Bench coding, run 28058352479). FP8: that curve was measured on -# glm-5.2-fp8, i.e. on this checkpoint. -# -# EVAL_ONLY leaves simulated acceptance off: it commits drafted tokens -# regardless of the target logits, so generated text is wrong and the eval -# would score ~0. -if [ "${EVAL_ONLY:-false}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.61 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$SGLANG_BACKEND_PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - # gfx950 has calibrated FP8 attention scales (the MXFP4 MI355X sibling and - # the DSv4 MI355X recipe both run fp8_e4m3 KV); only gfx942 keeps bf16 KV. - --kv-cache-dtype fp8_e4m3 - # DSA indexer kernels: the tilelang prefill/decode backends are the ones - # the MXFP4 MI355X sibling validated on gfx950 and remain options in v0.5.19. - --dsa-prefill-backend tilelang - --dsa-decode-backend tilelang - # GLM-5.2 emits the GLM-4.7-style tool-call format; glm47 is required for - # structured message.tool_calls (SWE-bench agentic evals die without it). - # The glm45 reasoning parser keeps hybrid thinking in reasoning_content. - --tool-call-parser glm47 - --reasoning-parser glm45 - --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" - --mem-fraction-static "$MEM_FRACTION_STATIC" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --speculative-algorithm EAGLE - --speculative-num-steps 5 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 6 - "${CACHE_ARGS[@]}" - --watchdog-timeout 1800 - --enable-metrics -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" - -{ - echo "=== SGLANG_SIMULATE_ACC_* env vars at launch (empty => real verification) ===" - env | grep -E '^SGLANG_SIMULATE_ACC_' | sort || true - echo "============================================================================" -} | tee "$SERVER_LOG" - -echo "Starting SGLang server for MI355X..." -"${SGLANG_CMD[@]}" >> "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$SGLANG_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$USE_SGLANG_ROUTER" = "true" ]; then - echo "Starting SGLang router on port $PORT for $TP DP ranks..." - "${SGLANG_ROUTER_CMD[@]}" \ - --worker-urls "http://localhost:$SGLANG_BACKEND_PORT" \ - --policy consistent_hashing \ - --request-id-headers x-correlation-id \ - --dp-aware \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$SGLANG_ROUTER_METRICS_PORT" \ - --connect-timeout-secs 900 \ - --request-timeout-secs 14400 \ - --disable-health-check \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - echo "Router PID: $ROUTER_PID" - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY:-false}" = "true" ]; then - # GLM-5.2's chat template defaults to reasoning_effort=Max when the client - # passes no chat_template_kwargs (mini-swe-agent doesn't); the heavy - # thinking burns the shared 75-step budget. Double it, as the GLM-5.2 B200 - # and MI325X recipes do. - export SWEBENCH_AGENT_STEP_LIMIT=150 - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$SGLANG_BACKEND_PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh deleted file mode 100755 index c1b63b195d..0000000000 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ /dev/null @@ -1,295 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Kimi-K3 MXFP4 on B300: TP8 x DCP8, TokenspeedMLA, Mooncake external KV tier. -# Concurrency selects the arm: -# conc <= 8 DSpark level 7, golden AL 3.84 -# conc 16 DSpark level 3, golden AL 3.00 -# conc > 16 no drafting -# Keep the arms disjoint in concurrency: exp-name carries conc and spec but not -# the arm, so a shared concurrency would collide. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION -# Optional: -# DCP_SIZE (default 8), KV_OFFLOAD_BACKEND (mooncake, or empty for resident) -# -# TP8 only: the ~1.5 TB MXFP4 checkpoint would need ~375 GB/GPU at TP4. -# The draft's real acceptance on this corpus is 1.16-2.01; synthetic acceptance -# measures the system at a prescribed AL, not the draft's fitness at 100k+ ISL. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars DCP_SIZE EVAL_ONLY SPEC_DECODING - -if [ "$TP" -ne 8 ]; then - echo "Error: Kimi-K3 on B300 requires TP=8, got TP='$TP'" >&2 - exit 1 -fi - -if [[ -n "${EP_SIZE:-}" && "${EP_SIZE}" -gt 1 ]]; then - echo "Error: this recipe ships the pure-TP8 profile; EP_SIZE='$EP_SIZE' is not wired" >&2 - exit 1 -fi - -# DCP shards decode KV across the TP ranks, so it must divide TP. -if [ $((TP % DCP_SIZE)) -ne 0 ]; then - echo "Error: TP='$TP' must be divisible by DCP_SIZE='$DCP_SIZE'" >&2 - exit 1 -fi -CP_ARGS=() -if [ "$DCP_SIZE" -gt 1 ]; then - CP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) -fi - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -DRAFT_MODEL="Inferact/Kimi-K3-DSpark" - -# The draft must not land next to a pre-staged target: dirname(MODEL_PATH) is a -# read-only mount, so use the launcher's writable models dir. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - check_env_vars WRITABLE_MODELS_DIR - DRAFT_MODEL_PATH="${WRITABLE_MODELS_DIR}/${DRAFT_MODEL##*/}" - # Other sweep cells share this directory and nonempty may mean a download - # is in progress; let HF validate/resume under one lock. - mkdir -p "$(dirname "$DRAFT_MODEL_PATH")" - check_env_vars MODEL_DOWNLOAD_LOCK_TIMEOUT - flock -w "$MODEL_DOWNLOAD_LOCK_TIMEOUT" "${DRAFT_MODEL_PATH}.download.lock" \ - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -export VLLM_ALLREDUCE_USE_FLASHINFER=1 -export VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION=1 -export VLLM_USE_V2_MODEL_RUNNER=1 -# These default to auto, which self-enables on B300; name them so the measured -# DCP a2a path is the one that runs. -export VLLM_USE_DIRECT_DCP_A2A=1 -export VLLM_USE_DIRECT_DCP_Q_GATHER=1 -export VLLM_USE_DIRECT_DCP_KV_GATHER=1 -# ~1.5 TB of MXFP4 shards loads well past the default readiness window. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_RPC_TIMEOUT=600000 -export VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0 -export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 -export PYTHONNOUSERSITE=1 -export TORCH_CUDA_ARCH_LIST=10.0 -# Identical prefixes must hash to identical block keys run-to-run. -export PYTHONHASHSEED=42 -# AIPerf reuses one pooled connection per session; outlast its idle gaps or a -# socket race aborts warmup. -export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=900 -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 - -SERVER_LOG="$RESULT_DIR/server.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" -MOONCAKE_MASTER_PID="" - -cleanup() { - if [[ -n "$MOONCAKE_MASTER_PID" ]]; then - kill "$MOONCAKE_MASTER_PID" 2>/dev/null || true - fi -} -trap cleanup EXIT - -OFFLOAD_ARGS=() -case "${KV_OFFLOAD_BACKEND:-}" in - "") - require_agentic_kv_offload_none - ;; - mooncake) - require_agentic_kv_offload_backend mooncake - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - # Rail-isolated nodes: two RNICs cannot reach each other even within a - # node, so every rank uses one rail. Chosen at runtime because mlx5_0 is - # down on some nodes (b300-016, b300-017) and topology discovery then - # finds 0 HCAs and every rank dies in a retry loop. - MOONCAKE_RAIL="" - for _d in mlx5_0 mlx5_1 mlx5_2 mlx5_3 mlx5_4 mlx5_5 mlx5_8 mlx5_9 \ - mlx5_10 mlx5_11 mlx5_16 mlx5_17 mlx5_20 mlx5_21 mlx5_22 mlx5_23; do - if grep -q ACTIVE "/sys/class/infiniband/$_d/ports/1/state" 2>/dev/null; then - MOONCAKE_RAIL="$_d" - break - fi - done - if [ -z "$MOONCAKE_RAIL" ]; then - echo "Error: no active RDMA rail on $(hostname); Mooncake cannot initialise" >&2 - exit 1 - fi - echo "Mooncake rail: $MOONCAKE_RAIL" - - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - if ! kill -0 "$MOONCAKE_MASTER_PID" 2>/dev/null; then - echo "Mooncake master died during startup." >&2 - cat "$MOONCAKE_MASTER_LOG" >&2 - exit 1 - fi - - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_load_failure_policy":"recompute","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_offload":false}}' - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or mooncake)" >&2 - exit 1 - ;; -esac - -if [ "${SPEC_DECODING}" != "mtp" ]; then - echo "Error: this recipe expects spec-decoding=mtp for every arm, got '${SPEC_DECODING:-}'" >&2 - exit 1 -fi -# Draft length by concurrency. The golden AL must track it, from the -# probabilistic curve in golden_al_distribution/. -if [ "$CONC" -le 8 ]; then - NUM_SPEC_TOKENS=7 - SYNTHETIC_ACCEPT_LEN=3.84 -elif [ "$CONC" -le 16 ]; then - NUM_SPEC_TOKENS=3 - SYNTHETIC_ACCEPT_LEN=3.00 -else - NUM_SPEC_TOKENS=0 -fi - -SPEC_ARGS=() -if [ "$NUM_SPEC_TOKENS" -gt 0 ]; then - # EVAL_ONLY needs real verification: synthetic acceptance commits drafts - # regardless of target logits and would zero the eval score. - if [ "${EVAL_ONLY}" = "true" ]; then - SPEC_CONFIG="{\"method\": \"dspark\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TOKENSPEED_MLA\", \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"block\"}" - else - SPEC_CONFIG="{\"method\": \"dspark\", \"model\": \"$DRAFT_MODEL_PATH\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TOKENSPEED_MLA\", \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" - fi - SPEC_ARGS=(--speculative-config "$SPEC_CONFIG") -fi - -MAX_NUM_SEQS=$((2 * CONC)) - -# 1 - util is the buffer for what is not sized against the budget (cudagraph -# pool, FlashInfer MoE workspace, fragmentation); only c56 and c70 ran out. -if [ "$CONC" -ge 56 ]; then - GPU_MEM_UTIL=0.90 -else - GPU_MEM_UTIL=0.92 -fi - -# Capture sizes: step * 1..min(max-num-seqs, 128), then the fixed powers of two -# above that. Sizes are tokens when drafting, sequences when not; the 128 caps -# the number of dense entries, not their value. -CAPTURE_STEP=$((1 + NUM_SPEC_TOKENS)) -DENSE_COUNT=$MAX_NUM_SEQS -if [ "$DENSE_COUNT" -gt 128 ]; then - DENSE_COUNT=128 -fi -CAPTURE_SIZES="" -for ((n = 1; n <= DENSE_COUNT; n++)); do - CAPTURE_SIZES+="${CAPTURE_SIZES:+,}$((n * CAPTURE_STEP))" -done -DENSE_MAX=$((DENSE_COUNT * CAPTURE_STEP)) -for t in 64 128 256 512 1024 2048 4096 8192; do - if [ "$t" -gt "$DENSE_MAX" ]; then - CAPTURE_SIZES+=",$t" - fi -done -COMPILATION_CONFIG="{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[${CAPTURE_SIZES}]}" - -echo "Starting vllm server..." - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - "${CP_ARGS[@]}" - --max-num-seqs "$MAX_NUM_SEQS" - --gpu-memory-utilization "$GPU_MEM_UTIL" - --max-num-batched-tokens 16384 - --trust-remote-code - --language-model-only - --enable-auto-tool-choice - --tool-call-parser kimi_k3 - --reasoning-parser kimi_k3 - --load-format fastsafetensors - --moe-backend auto - --no-enable-flashinfer-autotune - --enable-cumem-allocator - --enable-prefix-caching - --prefix-match-unit 128 - --kv-cache-dtype fp8 - --stream-interval 10 - --attention-backend TOKENSPEED_MLA - --attention-config '{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - "${SPEC_ARGS[@]}" - --compilation-config "$COMPILATION_CONFIG" - --disable-uvicorn-access-log - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh deleted file mode 100755 index 954007c8f0..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# MiniMax-M3 NVFP4 on B200 with EAGLE3-GQA and synthetic acceptance; DRAM KV -# offload uses SimpleCPUOffloadConnector in lazy mode. Port of -# minimaxm3_fp4_b300_mtp.sh; the B200 deltas are marked "B200:" below. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -NUM_SPEC_TOKENS=3 -# Golden AL for the GQA draft head: golden_al_distribution/minimaxm3_eagle3_gqa.yaml -# minimax-m3.thinking_on[3]. The non-GQA curve (minimaxm3_eagle3.yaml) reads 2.83. -SYNTHETIC_ACCEPT_LEN=2.78 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# A non-empty directory is not a staged checkpoint: an aborted pull leaves -# config.json with no weights. Require every shard the index names, and accept -# the single-file layout the EAGLE3 draft head ships in (no index). -checkpoint_is_complete() { - local dir="$1" - [[ -d "$dir" && -f "$dir/config.json" ]] || return 1 - CKPT_DIR="$dir" python3 - <<'PYEOF' -import glob, json, os, sys - -d = os.environ["CKPT_DIR"] -index = os.path.join(d, "model.safetensors.index.json") -if os.path.isfile(index): - with open(index) as fh: - shards = sorted(set(json.load(fh)["weight_map"].values())) - missing = [s for s in shards if not os.path.isfile(os.path.join(d, s))] - if missing: - print( - f"{len(missing)}/{len(shards)} shards missing, e.g. {missing[:3]}", - file=sys.stderr, - ) - sys.exit(1) -elif not glob.glob(os.path.join(d, "*.safetensors")): - print("no shard index and no .safetensors present", file=sys.stderr) - sys.exit(1) -PYEOF -} - -# B200: launch_b200-nscale-slurm.sh rewrites MODEL to a cluster-local path, so -# keep the HF repo id separately for the unstaged case. -HF_MODEL_ID="nvidia/MiniMax-M3-NVFP4" - -if [[ -n "${MODEL_PATH:-}" ]]; then - if ! checkpoint_is_complete "$MODEL_PATH"; then - # Every concurrency runs as its own allocation against the same shared - # path; one cell pulls the ~250 GB checkpoint and the rest wait on the - # lock. hf download resumes into a partially populated --local-dir. - mkdir -p "$MODEL_PATH" - MODEL_DOWNLOAD_LOCK="${MODEL_PATH%/}.download.lock" - echo "Checkpoint at $MODEL_PATH is incomplete; acquiring $MODEL_DOWNLOAD_LOCK" - exec 9>"$MODEL_DOWNLOAD_LOCK" - check_env_vars MODEL_DOWNLOAD_LOCK_TIMEOUT - flock -w "$MODEL_DOWNLOAD_LOCK_TIMEOUT" 9 || { - echo "Error: timed out waiting for another cell to stage $MODEL_PATH" >&2 - exit 1 - } - if checkpoint_is_complete "$MODEL_PATH"; then - echo "Another cell staged $MODEL_PATH while we waited" - else - hf download "$HF_MODEL_ID" --local-dir "$MODEL_PATH" - fi - flock -u 9 - exec 9>&- - checkpoint_is_complete "$MODEL_PATH" || { - echo "Error: $MODEL_PATH is still incomplete after hf download $HF_MODEL_ID." >&2 - exit 1 - } - fi -else - hf download "$HF_MODEL_ID" - export MODEL_PATH="$HF_MODEL_ID" -fi - -# B200: /data/models does not exist on b200-nscale and the launcher bind-mounts -# only $MODEL_PATH, so stage the draft in the (per-job, writable) container -# overlay next to it rather than inside the shared checkpoint directory. -DRAFT_MODEL_PATH="$(dirname "${MODEL_PATH%/}")/${DRAFT_MODEL##*/}" -if ! checkpoint_is_complete "$DRAFT_MODEL_PATH"; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - checkpoint_is_complete "$DRAFT_MODEL_PATH" || { - echo "Error: $DRAFT_MODEL_PATH is incomplete after hf download $DRAFT_MODEL." >&2 - exit 1 - } -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend vllm-simple; then - CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_CONFIG=$(printf \ - '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":%d,"lazy_offload":true}}' \ - "$CPU_OFFLOAD_BYTES") - OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG") -fi - -export PYTHONNOUSERSITE=1 -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm - -# B200's 180 GB leaves little beyond the ~250 GB checkpoint: TP2 cannot -# host 1M-context KV for one request at 0.9, so TP4 is the smallest topology. -GPU_MEMORY_UTILIZATION="0.9" - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -if [ "${EVAL_ONLY:-}" = "true" ]; then - SPEC_CONFIG=$(printf \ - '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN"}' \ - "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS") -else - SPEC_CONFIG=$(printf \ - '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":%.2f}' \ - "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS" "$SYNTHETIC_ACCEPT_LEN") -fi - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --block-size 128 - --language-model-only - --enable-prefix-caching - --no-enable-flashinfer-autotune - --reasoning-parser minimax_m3 - --tool-call-parser minimax_m3 - --enable-auto-tool-choice - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --attention-config '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - --kv-cache-dtype fp8 - --max-cudagraph-capture-size 512 - --max-num-batched-tokens 16384 - --stream-interval 20 - --trust-remote-code - --speculative-config "$SPEC_CONFIG" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" -set -x - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh deleted file mode 100755 index 49f405fb77..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_trt_mtp.sh +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# MiniMax-M3 NVFP4 on B200 with TRT-LLM EAGLE3-GQA. DRAM KV offload uses -# TRT-LLM's native secondary-memory pool; kv_cache_config.host_cache_size is -# pinned per topology in ser.yaml (200 GiB at TP8, 250 GiB at TP4), not derived -# from TOTAL_CPU_DRAM_GB. KV_OFFLOADING / KV_OFFLOAD_BACKEND come from the -# master config; do not override them here. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC PORT KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EVAL_ONLY - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -NUM_SPEC_TOKENS=3 -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/prometheus/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="trtllm_kv_cache_utilization" - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="/lustre/fsw/gharunners/models/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - mkdir -p "$DRAFT_MODEL_PATH" - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -# rc23 gates Prometheus and its expensive per-step timing collector behind one -# option; keep request/iteration metrics without timing payloads. -disable_trtllm_detailed_perf_metrics - -# BFCL's stock OpenAI client sends the standard `store=false` field. TRT-LLM -# 1.3 rejects that field even though this server never persists responses. -python3 "$(dirname "$0")/../../../runners/patch_trtllm_chat_store.py" - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "TRTLLM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -CAPTURE_TOKENS_LIST=(1 512 1024 2048) -CAPTURE_TOKENS_LIST=$(printf "%s, " "${CAPTURE_TOKENS_LIST[@]}") - -MAX_BATCH=$CONC -if (( MAX_BATCH <= 20 )); then - CAPTURE_BATCH_LIST=( $(seq 1 $MAX_BATCH) ) -elif (( MAX_BATCH == 25 )); then - CAPTURE_BATCH_LIST=( $(seq 1 15) 17 19 21 23 25 ) -elif (( MAX_BATCH == 30 )); then - CAPTURE_BATCH_LIST=( $(seq 1 12) $(seq 14 2 24) 27 30 ) -elif (( MAX_BATCH == 35 )); then - CAPTURE_BATCH_LIST=( $(seq 1 11) $(seq 14 3 35) ) -elif (( MAX_BATCH == 40 )); then - CAPTURE_BATCH_LIST=( $(seq 2 2 40) ) -elif (( MAX_BATCH == 45 )); then - CAPTURE_BATCH_LIST=( $(seq 2 2 18) $(seq 21 3 45) ) -elif (( MAX_BATCH == 50 )); then - CAPTURE_BATCH_LIST=( $(seq 2 2 8) 11 $(seq 14 2 18) $(seq 21 3 48) 50 ) -elif (( MAX_BATCH == 55 )); then - CAPTURE_BATCH_LIST=( $(seq 2 3 14) $(seq 16 2 18) $(seq 21 3 48) 50 53 55 ) -fi -CAPTURE_BATCH_LIST=$(printf "%s, " "${CAPTURE_BATCH_LIST[@]}") - -if [[ $TP == 8 ]]; then - mem_off=214748364800 -else - mem_off=268435456000 -fi - -cat << EOF > ser.yaml -max_seq_len: 1048576 -max_num_tokens: 16384 -max_batch_size: $MAX_BATCH -cuda_graph_config: - enable_padding: true - batch_sizes: [${CAPTURE_BATCH_LIST%, }] -torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_userbuffers: true - max_num_streams: 3 -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true -kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: $mem_off -speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: $DRAFT_MODEL_PATH -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION -enable_chunked_prefill: true -enable_autotuner: true -trust_remote_code: true -reasoning_parser: minimax_m3 -stream_interval: 20 -print_iter_log: true -enable_iter_perf_stats: true -return_perf_metrics: true -num_postprocess_workers: 8 -enable_attention_dp: false -EOF - -export TLLM_LOG_LEVEL=INFO -export TRTLLM_SERVER_DISABLE_GC=1 -export TRTLLM_WORKER_DISABLE_GC=1 -export TLLM_PROFILE_LOG_RANKS=all -# aiperf resolves its tokenizer by HF repo id ($MODEL), so do not set -# HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE here. -export PYTHONNOUSERSITE=1 -export TRTLLM_ENABLE_PDL=1 -export ENROOT_ALLOW_DEV=yes -export NCCL_GRAPH_MIXING_SUPPORT=0 -export MIMALLOC_PURGE_DELAY=0 -export TQDM_DISABLE=1 -export HF_HUB_DISABLE_PROGRESS_BARS=1 -export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -export TRTLLM_SERVE_ENABLE_MSGSPEC=1 -export TRTLLM_TORCH_COMPILE_CONTEXT_ONLY=1 -# Golden AL 2.78 = one target token plus 1.78 accepted draft tokens. The force -# knob overwrites the verifier's count, so accuracy evals must leave it unset. -if [ "$EVAL_ONLY" = "true" ]; then - unset TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -else - export TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1.78 -fi - -{ set +x; } 2>/dev/null -TRTLLM_CMD=( - mpirun -n 1 --oversubscribe --allow-run-as-root - trtllm-serve "$MODEL_PATH" - --tp_size "$TP" - --host 0.0.0.0 - --port "$PORT" - --chat_template "$MODEL_PATH/chat_template.jinja" - --config ser.yaml -) -printf '%q ' "${TRTLLM_CMD[@]}" | tee "$RESULT_DIR/trtllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/trtllm_command.txt" -"${TRTLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" -set -x - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh deleted file mode 100755 index 508c0d0359..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# MiniMax-M3 NVFP4 B300 AgentX with EAGLE3-GQA and synthetic acceptance. -# DRAM KV offload uses vLLM's SimpleCPUOffloadConnector in lazy mode. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -NUM_SPEC_TOKENS=3 -# Golden AL for the GQA draft head: golden_al_distribution/minimaxm3_eagle3_gqa.yaml -# minimax-m3.thinking_on[3]. The non-GQA curve (minimaxm3_eagle3.yaml) reads 2.83. -SYNTHETIC_ACCEPT_LEN=2.78 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend vllm-simple; then - CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_CONFIG=$(printf \ - '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":%d,"lazy_offload":true}}' \ - "$CPU_OFFLOAD_BYTES") - OFFLOAD_ARGS=(--kv-transfer-config "$OFFLOAD_CONFIG") -fi - -export PYTHONNOUSERSITE=1 -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_FLOAT32_MATMUL_PRECISION=high -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -if [ "${EVAL_ONLY:-}" = "true" ]; then - SPEC_CONFIG=$(printf \ - '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN"}' \ - "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS") -else - SPEC_CONFIG=$(printf \ - '{"method":"eagle3","model":"%s","num_speculative_tokens":%d,"attention_backend":"FLASH_ATTN","rejection_sample_method":"synthetic","synthetic_acceptance_length":%.2f}' \ - "$DRAFT_MODEL_PATH" "$NUM_SPEC_TOKENS" "$SYNTHETIC_ACCEPT_LEN") -fi - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --gpu-memory-utilization 0.9 - --block-size 128 - --language-model-only - --enable-prefix-caching - --no-enable-flashinfer-autotune - --reasoning-parser minimax_m3 - --tool-call-parser minimax_m3 - --enable-auto-tool-choice - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --attention-config '{"backend":"FLASHINFER","use_trtllm_attention":true,"indexer_kv_dtype":"fp8"}' - --kv-cache-dtype fp8 - --max-cudagraph-capture-size 512 - --max-num-batched-tokens 16384 - --stream-interval 20 - --trust-remote-code - --speculative-config "$SPEC_CONFIG" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" -set -x - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh deleted file mode 100755 index efc2d0afc4..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_trt_mtp.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# MiniMax-M3 NVFP4 on B300 with TRT-LLM EAGLE3-GQA. DRAM KV offload uses -# TRT-LLM's native secondary-memory pool; kv_cache_config.host_cache_size is -# pinned per topology in ser.yaml, not derived from TOTAL_CPU_DRAM_GB. -# KV_OFFLOADING / KV_OFFLOAD_BACKEND come from the master config; do not -# override them here. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC PORT KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EVAL_ONLY - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -NUM_SPEC_TOKENS=3 -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/prometheus/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="trtllm_kv_cache_utilization" - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi - -nvidia-smi -resolve_trace_source -install_agentic_deps -# rc23 gates Prometheus and its expensive per-step timing collector behind one -# option; keep request/iteration metrics without timing payloads. -disable_trtllm_detailed_perf_metrics - -# BFCL's stock OpenAI client sends the standard `store=false` field. TRT-LLM -# 1.3 rejects that field even though this server never persists responses. -python3 "$(dirname "$0")/../../../runners/patch_trtllm_chat_store.py" - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "TRTLLM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -CAPTURE_TOKENS_LIST=(1 512 1024 2048) -CAPTURE_TOKENS_LIST=$(printf "%s, " "${CAPTURE_TOKENS_LIST[@]}") - -MAX_BATCH=$CONC -if (( MAX_BATCH <= 20 )); then - CAPTURE_BATCH_LIST=( $(seq 1 $MAX_BATCH) ) -elif (( MAX_BATCH == 25 )); then - CAPTURE_BATCH_LIST=( $(seq 1 15) 17 19 21 23 25 ) -elif (( MAX_BATCH == 30 )); then - CAPTURE_BATCH_LIST=( $(seq 1 12) $(seq 14 2 24) 27 30 ) -fi -CAPTURE_BATCH_LIST=$(printf "%s, " "${CAPTURE_BATCH_LIST[@]}") - -if [[ $TP == 8 ]]; then - mem_off=309237645312 -else - mem_off=388554555392 -fi - -cat << EOF > ser.yaml -max_seq_len: 1048576 -max_num_tokens: 16384 -max_batch_size: $MAX_BATCH -cuda_graph_config: - enable_padding: true - batch_sizes: [${CAPTURE_BATCH_LIST%, }] -torch_compile_config: - enable_fullgraph: true - enable_inductor: false - enable_piecewise_cuda_graph: true - capture_num_tokens: [${CAPTURE_TOKENS_LIST%, }] - enable_userbuffers: true - max_num_streams: 3 -moe_config: - backend: TRTLLM - use_low_precision_moe_combine: true -sparse_attention_config: - algorithm: minimax_m3 - implementation: msa - indexer_kv_dtype: fp8 - sparse_disable_index_value: true - fuse_qkv_index_projection: true -kv_cache_config: - free_gpu_memory_fraction: 0.94 - enable_block_reuse: true - block_reuse_policy: per_conversation - tokens_per_block: 128 - use_kv_cache_manager_v2: true - dtype: fp8 - event_buffer_max_size: 0 - host_cache_size: $mem_off -speculative_config: - decoding_type: Eagle3 - max_draft_len: 3 - speculative_model: $DRAFT_MODEL_PATH -scheduler_config: - capacity_scheduler_policy: MAX_UTILIZATION -enable_chunked_prefill: true -enable_autotuner: true -trust_remote_code: true -reasoning_parser: minimax_m3 -stream_interval: 20 -print_iter_log: true -enable_iter_perf_stats: true -return_perf_metrics: true -num_postprocess_workers: 8 -enable_attention_dp: false -EOF - -export TLLM_LOG_LEVEL=INFO -export TRTLLM_SERVER_DISABLE_GC=1 -export TRTLLM_WORKER_DISABLE_GC=1 -export TLLM_PROFILE_LOG_RANKS=all -# aiperf resolves its tokenizer by HF repo id ($MODEL), so do not set -# HF_HUB_OFFLINE / TRANSFORMERS_OFFLINE here. -export PYTHONNOUSERSITE=1 -export TRTLLM_ENABLE_PDL=1 -export ENROOT_ALLOW_DEV=yes -export NCCL_GRAPH_MIXING_SUPPORT=0 -export MIMALLOC_PURGE_DELAY=0 -export TQDM_DISABLE=1 -export HF_HUB_DISABLE_PROGRESS_BARS=1 -export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -export TRTLLM_SERVE_ENABLE_MSGSPEC=1 -export TRTLLM_TORCH_COMPILE_CONTEXT_ONLY=1 -# Golden AL 2.78 = one target token plus 1.78 accepted draft tokens. The force -# knob overwrites the verifier's count, so accuracy evals must leave it unset. -if [ "$EVAL_ONLY" = "true" ]; then - unset TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS -else - export TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=1.78 -fi - -{ set +x; } 2>/dev/null -TRTLLM_CMD=( - mpirun -n 1 --oversubscribe --allow-run-as-root - trtllm-serve "$MODEL_PATH" - --tp_size "$TP" - --host 0.0.0.0 - --port "$PORT" - --chat_template "$MODEL_PATH/chat_template.jinja" - --config ser.yaml -) -printf '%q ' "${TRTLLM_CMD[@]}" | tee "$RESULT_DIR/trtllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/trtllm_command.txt" -"${TRTLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" -set -x - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh deleted file mode 100755 index 9e5f8fc8d6..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100_mtp.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# H100 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional Mooncake DRAM KV offload. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION PORT EVAL_ONLY - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -# Concurrent downloads on the shared HF cache can hit transient stale handles. -for attempt in 1 2 3 4 5; do - hf download "$DRAFT_MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download of $DRAFT_MODEL failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 -done -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -MOONCAKE_MASTER_PID="" -cleanup_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" 30 - exit "$exit_code" -} -trap cleanup_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -OFFLOAD_ARGS=() -MODEL_CPU_OFFLOAD_GB=26 -MODEL_CHECKPOINT_PAGE_CACHE_GIB=414 -MOONCAKE_LOCAL_BUFFER_GIB=4 -if [ "$KV_OFFLOADING" = "none" ]; then - require_agentic_kv_offload_none -elif [ "$KV_OFFLOADING" = "dram" ]; then - require_agentic_kv_offload_backend mooncake - TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824)) - PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB)) - if (( PER_RANK_GIB <= 0 )); then - echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and KV offload" >&2 - exit 1 - fi - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -else - echo "Error: unsupported KV_OFFLOADING='$KV_OFFLOADING'" >&2 - exit 1 -fi - -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -NUM_SPEC_TOKENS=3 -TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) - -# Golden AL is minimaxm3_eagle3_gqa.yaml thinking_on[3]; eval uses real verification. -SYNTHETIC_ACCEPT_LEN=2.78 -if [ "$EVAL_ONLY" = "true" ]; then - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" -else - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" -fi - -MAX_NUM_SEQS=$((2 * CONC)) -# MTP verifies four tokens per sequence, so CUDA graph capture is token-sized. -MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * TOKENS_PER_SEQ)) - -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --data-parallel-size 1 - --gpu-memory-utilization 0.90 - --cpu-offload-gb "$MODEL_CPU_OFFLOAD_GB" - --kv-cache-dtype fp8 - --attention-backend TRITON_ATTN - --block-size 128 - --language-model-only - --enable-prefix-caching - --enable-prompt-tokens-details - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --max-num-seqs "$MAX_NUM_SEQS" - --max-cudagraph-capture-size "$MAX_CUDAGRAPH_CAPTURE_SIZE" - --speculative-config "$SPEC_CONFIG" - --tool-call-parser minimax_m3 - --reasoning-parser minimax_m3 - --enable-auto-tool-choice - --safetensors-load-strategy lazy - --trust-remote-code - "${OFFLOAD_ARGS[@]}" -) -write_command "$RESULT_DIR/vllm_command.txt" "${VLLM_CMD[@]}" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$EVAL_ONLY" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h200_mtp.sh deleted file mode 100755 index 428f5e0949..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_h200_mtp.sh +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# H200 MiniMax-M3 MXFP8 AgentX with EAGLE3 and optional Mooncake DRAM KV offload. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION PORT EVAL_ONLY - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" - -resolve_complete_model_snapshot() { - python3 - "$1" <<'PY' -import json -import sys -from pathlib import Path - -model_cache_dir = Path(sys.argv[1]) -try: - revision = model_cache_dir.joinpath("refs/main").read_text().strip() -except OSError: - raise SystemExit - -if not revision or Path(revision).name != revision: - raise SystemExit - -snapshot = model_cache_dir / "snapshots" / revision -index_path = snapshot / "model.safetensors.index.json" -required_files = ( - snapshot / "config.json", - snapshot / "tokenizer_config.json", - index_path, -) -if not all(path.is_file() for path in required_files): - raise SystemExit -try: - weight_map = json.loads(index_path.read_text())["weight_map"] -except (KeyError, json.JSONDecodeError, OSError): - raise SystemExit -shards = {snapshot / filename for filename in weight_map.values()} -if shards and all(path.is_file() for path in shards): - print(snapshot) -PY -} - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - check_env_vars HF_HUB_CACHE - MODEL_CACHE_ROOT="$HF_HUB_CACHE" - MODEL_CACHE_DIR="$MODEL_CACHE_ROOT/models--${MODEL//\//--}" - mkdir -p "$MODEL_CACHE_ROOT" - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - exec 9>"$MODEL_CACHE_ROOT/.minimaxm3-download.lock" - flock -w 3600 9 - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - DOWNLOADED_MODEL_PATH=$(hf download "$MODEL") - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - echo "Downloaded model snapshot is incomplete: $DOWNLOADED_MODEL_PATH" >&2 - exit 1 - fi - fi - flock -u 9 - fi - echo "Using complete cached model snapshot: $MODEL_PATH" - export MODEL_PATH -fi - -# Concurrent downloads on the shared HF cache can hit transient stale handles. -for attempt in 1 2 3 4 5; do - hf download "$DRAFT_MODEL" && break - if [ "$attempt" = 5 ]; then echo "hf download of $DRAFT_MODEL failed after $attempt attempts" >&2; exit 1; fi - echo "hf download attempt $attempt failed; retrying in 60s" >&2 - sleep 60 -done -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export PYTHONNOUSERSITE=1 -# Humming for dense MXFP8 linear layers; atomic reduction for the Marlin MXFP8 MoE path. -export VLLM_MARLIN_USE_ATOMIC_ADD=1 - -SERVER_LOG="$RESULT_DIR/server.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -MOONCAKE_MASTER_PID="" -cleanup_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - stop_background_process_tree "$MOONCAKE_MASTER_PID" "Mooncake master" 30 - exit "$exit_code" -} -trap cleanup_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -OFFLOAD_ARGS=() -MODEL_CHECKPOINT_PAGE_CACHE_GIB=414 -MOONCAKE_LOCAL_BUFFER_GIB=4 -if [ "$KV_OFFLOADING" = "none" ]; then - require_agentic_kv_offload_none - GPU_MEMORY_UTILIZATION=0.95 -elif [ "$KV_OFFLOADING" = "dram" ]; then - require_agentic_kv_offload_backend mooncake - GPU_MEMORY_UTILIZATION=0.90 - TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824)) - PER_RANK_GB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MOONCAKE_LOCAL_BUFFER_GIB)) - if (( PER_RANK_GB <= 0 )); then - echo "Error: CPU DRAM budget is too small for checkpoint cache and KV offload" >&2 - exit 1 - fi - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -else - echo "Error: unsupported KV_OFFLOADING='$KV_OFFLOADING'" >&2 - exit 1 -fi - -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -NUM_SPEC_TOKENS=3 -TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) - -# Golden AL is minimaxm3_eagle3_gqa.yaml thinking_on[3]; eval uses real verification. -SYNTHETIC_ACCEPT_LEN=2.78 -if [ "$EVAL_ONLY" = "true" ]; then - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\"}" -else - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"FLASH_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" -fi - -MAX_NUM_SEQS=$((2 * CONC)) -# MTP verifies four tokens per sequence, so CUDA graph capture is token-sized. -MAX_CUDAGRAPH_CAPTURE_SIZE=$((MAX_NUM_SEQS * TOKENS_PER_SEQ)) - -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --data-parallel-size 1 - --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" - --kv-cache-dtype fp8 - --attention-backend FLASH_ATTN - --linear-backend humming - --block-size 128 - --language-model-only - --enable-prefix-caching - --enable-prompt-tokens-details - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --max-num-seqs "$MAX_NUM_SEQS" - --max-cudagraph-capture-size "$MAX_CUDAGRAPH_CAPTURE_SIZE" - --speculative-config "$SPEC_CONFIG" - --tool-call-parser minimax_m3 - --reasoning-parser minimax_m3 - --enable-auto-tool-choice - --trust-remote-code - "${OFFLOAD_ARGS[@]}" -) -write_command "$RESULT_DIR/vllm_command.txt" "${VLLM_CMD[@]}" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$EVAL_ONLY" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh deleted file mode 100755 index c13d3e9143..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# MiniMax-M3 MXFP8 MI300X AgentX with EAGLE3-GQA and optional LMCache MP. - -source "$(dirname "$0")/../../benchmark_lib.sh" - - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -NUM_SPEC_TOKENS=3 -SYNTHETIC_ACCEPT_LEN=2.78 - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi -if [[ -n "$ROCR_VISIBLE_DEVICES" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -hf download "$DRAFT_MODEL" - -rocm-smi || true -amd-smi || true -resolve_trace_source -install_agentic_deps - -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -LMCACHE_PIDS=() -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - local i - for i in "${!LMCACHE_PIDS[@]}"; do - stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" - done - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -OFFLOAD_ARGS=() -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - lmcache) - require_agentic_kv_offload_backend lmcache - LMCACHE_VERSION="0.5.3" - LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm" - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-rocm-7-0==14.1.1" \ - "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" - python3 -c \ - "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ - >/dev/null - - LMCACHE_L1_SHARD_GB=$((TOTAL_CPU_DRAM_GB / TP)) - if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then - echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2 - exit 1 - fi - - LMCACHE_SERVER_URLS=() - LMCACHE_HTTP_PORTS=() - LMCACHE_LOGS=() - : > "$RESULT_DIR/lmcache_command.txt" - for shard in $(seq 0 $((TP - 1))); do - shard_port=$((5555 + shard)) - shard_http_port=$((8080 + shard)) - shard_log="${LMCACHE_LOG%.log}_${shard}.log" - LMCACHE_CMD=( - lmcache server - --host 127.0.0.1 - --port "$shard_port" - --http-host 127.0.0.1 - --http-port "$shard_http_port" - --l1-size-gb "$LMCACHE_L1_SHARD_GB" - --l1-init-size-gb 10 - --l1-read-ttl-seconds 7200 - --chunk-size 256 - --max-workers 2 - --eviction-policy LRU - --supported-transfer-mode lmcache_driven - ) - append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" - "${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 & - LMCACHE_PIDS+=($!) - LMCACHE_HTTP_PORTS+=("$shard_http_port") - LMCACHE_LOGS+=("$shard_log") - LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}") - done - for shard in "${!LMCACHE_PIDS[@]}"; do - wait_for_ready \ - --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \ - --log "${LMCACHE_LOGS[$shard]}" \ - --pid "${LMCACHE_PIDS[$shard]}" \ - --sleep-interval 1 \ - --timeout 600 - done - LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}") - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}" - ) - ;; - *) - echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND" >&2 - exit 1 - ;; -esac - -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -if [ "$EVAL_ONLY" = "true" ]; then - SPEC_CONFIG="{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"attention_backend\":\"TRITON_ATTN\"}" -else - SPEC_CONFIG="{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"attention_backend\":\"TRITON_ATTN\",\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN}" -fi - -export PYTHONNOUSERSITE=1 -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MHA=0 -export TORCH_BLAS_PREFER_HIPBLASLT=1 -export NCCL_MIN_NCHANNELS=112 -export GPU_MAX_HW_QUEUES=2 - -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - "${PARALLEL_ARGS[@]}" - --trust-remote-code - --block-size 128 - --gpu-memory-utilization 0.90 - # The upstream nightly does not torch-compile MiniMaxM3SparseForConditionalGeneration, - # so with VLLM_USE_BREAKABLE_CUDAGRAPH=0 the default FULL_AND_PIECEWISE - # mode aborts at init ("piecewise CUDA graphs unavailable"). - --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' - --enable-chunked-prefill - --max-num-batched-tokens 16384 - --language-model-only - --enable-prefix-caching - --attention-backend TRITON_ATTN - --kv-cache-dtype fp8 - --tool-call-parser minimax_m3 - --reasoning-parser minimax_m3 - --enable-auto-tool-choice - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --max-num-seqs "$((2 * CONC))" - --stream-interval 20 - --speculative-config "$SPEC_CONFIG" - "${OFFLOAD_ARGS[@]}" -) -write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "$EVAL_ONLY" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh deleted file mode 100755 index 28d901417b..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - - -check_env_vars MODEL TP CONC KV_OFFLOADING RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY - -DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" -NUM_SPEC_TOKENS=3 -SYNTHETIC_ACCEPT_LEN=2.78 - -if [[ -n "$SLURM_JOB_ID" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -if [[ -n "$ROCR_VISIBLE_DEVICES" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "$MODEL_PATH" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -hf download "$DRAFT_MODEL" - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -require_agentic_kv_offload_none -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" - -if [ "$EVAL_ONLY" = "true" ]; then - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" -else - SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" -fi - -export PYTHONNOUSERSITE=1 -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 - -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --gpu-memory-utilization 0.90 - --kv-cache-dtype fp8 - --block-size 128 - --language-model-only - --attention-backend TRITON_ATTN - --enable-prefix-caching - --enable-chunked-prefill - --max-num-batched-tokens 32768 - --max-num-seqs "$((2 * CONC))" - --speculative-config "$SPEC_CONFIG" - --tool-call-parser minimax_m3 - --reasoning-parser minimax_m3 - --enable-auto-tool-choice - --default-chat-template-kwargs '{"thinking_mode":"enabled"}' - --trust-remote-code - --stream-interval 20 -) -write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_ready \ - --endpoint "http://0.0.0.0:${PORT}/health" \ - --log "$SERVER_LOG" \ - --pid "$SERVER_PID" - -if [ "$EVAL_ONLY" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh deleted file mode 100755 index 0d4cc74f14..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# AgentX trace replay for Qwen3.5-397B-A17B NVFP4 on B200 with SGLang -# native NEXTN MTP. Throughput uses the committed golden synthetic AL; evals -# retain real target-model verification. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. - HICACHE_ALIGNMENT_RESERVE_GB=$TP - HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then - echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 - exit 1 - fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 - exit 1 - fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy write_through_selective - ) -fi - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) - -# TP4 needs parallel tokenization to keep 256k AgentX warmups below the client -# request timeout. Keep TP2 on SGLang's single-worker default: multi-tokenizer -# startup races with the TP2 HiCache shared-memory initialization path. -TOKENIZER_ARGS=() -if [ "$TP" -ge 4 ]; then - TOKENIZER_ARGS=(--tokenizer-worker-num 6) -fi - -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -# Keep server-side connections alive beyond AIPerf's 300-second client pool -# timeout so bursty AgentX trajectories cannot reuse a closing idle socket. -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --enable-symm-mem - --quantization modelopt_fp4 - --fp4-gemm-backend flashinfer_cutlass - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend trtllm_mha - --moe-runner-backend flashinfer_trtllm - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 16384 - --chunked-prefill-size 16384 - --mem-fraction-static 0.80 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --enable-metrics - --enable-cache-report - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -capture_cache_metrics -trap capture_cache_metrics EXIT - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh deleted file mode 100644 index 88311f2b04..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# AgentX trace replay for Qwen3.5-397B-A17B MXFP4 on MI355X with SGLang -# native EAGLE MTP. Throughput uses the committed golden synthetic -# acceptance length; evaluation retains real target-model verification. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=60 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -rocm-smi || true -amd-smi || true - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "SGLang server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - HICACHE_RATIO="1.5" - HICACHE_WRITE_POLICY="write_through" - HICACHE_IO_BACKEND="kernel" - HICACHE_MEM_LAYOUT="page_first" - echo "HiCache CPU tier: ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT, dram_budget=${TOTAL_CPU_DRAM_GB} GB, tp=$TP" - CACHE_ARGS=( - --enable-hierarchical-cache - --hicache-ratio "$HICACHE_RATIO" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - ) -fi - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) - -TOKENIZER_ARGS=() -if [ "$TP" -ge 4 ]; then - TOKENIZER_ARGS=(--tokenizer-worker-num 6) -fi - -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS=$MAX_RUNNING_REQUESTS -[ "$CUDA_GRAPH_MAX_BS" -gt 128 ] && CUDA_GRAPH_MAX_BS=128 - -export PYTHONNOUSERSITE=1 -export SGLANG_USE_AITER=1 -export SGLANG_USE_AITER_UNIFIED_ATTN=1 -export AITER_FLYDSL_FORCE=1 -export SGLANG_MAMBA_SSM_DTYPE=bfloat16 -export ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --attention-backend aiter - --mem-fraction-static 0.80 - --model-loader-extra-config '{"enable_multithread_load": true}' - --watchdog-timeout 1200 - --page-size 16 - --kv-cache-dtype fp8_e4m3 - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 16384 - --chunked-prefill-size 16384 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --stream-interval 50 - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --enable-metrics - --enable-cache-report - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh deleted file mode 100755 index 2a2d4b62e2..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# AgentX trace replay for Qwen3.5-397B-A17B FP8 on B200 with SGLang -# native NEXTN MTP. Throughput uses the committed golden synthetic AL; evals -# retain real target-model verification. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. - HICACHE_ALIGNMENT_RESERVE_GB=$TP - HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then - echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 - exit 1 - fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 - exit 1 - fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy write_through_selective - ) -fi - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) - -# Parallel tokenization keeps 256k AgentX warmups below the client timeout. -TOKENIZER_ARGS=(--tokenizer-worker-num 6) - -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -# Keep server-side connections alive beyond AIPerf's 300-second client pool -# timeout so bursty AgentX trajectories cannot reuse a closing idle socket. -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --enable-symm-mem - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend trtllm_mha - --moe-runner-backend flashinfer_trtllm - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 16384 - --chunked-prefill-size 16384 - --mem-fraction-static 0.80 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --enable-metrics - --enable-cache-report - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -capture_cache_metrics -trap capture_cache_metrics EXIT - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh deleted file mode 100755 index 6a931059a7..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Qwen3.5 FP8 on H100 with SGLang EAGLE MTP. 80 GB HBM3 keeps -# mem-fraction-static at 0.75; attention is flashinfer (trtllm_mha is -# Blackwell-only). -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR \ - DURATION EP_SIZE -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# H100 max_model_len caps at 131k. The unfiltered corpus has requests up to -# ~1M tokens the server would reject; the 256k-capped variant (470 traces) -# keeps the rejection rate low. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank. - HICACHE_HOST_POOL_COUNT="2" - HICACHE_WRITE_POLICY="write_through_selective" - HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SPEC_ARGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -# Acceptance is pinned to the committed golden AL (golden_al_distribution/README.md): -# 3.39 is qwen3.5_mtp.yaml at num_speculative_tokens=3, thinking_on. -# SGLANG_SIMULATE_ACC_TOKEN_MODE exists from SGLang v0.5.16, which is why the -# image is pinned there. EVAL_ONLY leaves it off: simulated acceptance commits -# drafted tokens regardless of target logits and the eval would score ~0. -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path="$MODEL_PATH" --served-model-name="$MODEL" - --host=0.0.0.0 - --port="$PORT" - --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" - --trust-remote-code - --tensor-parallel-size="$TP" - --data-parallel-size=1 - --expert-parallel-size="$EP_SIZE" - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend flashinfer - --enable-flashinfer-allreduce-fusion - --mem-fraction-static 0.75 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${SPEC_ARGS[@]}" - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh deleted file mode 100755 index fcdee818a1..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Qwen3.5 FP8 on H200 with SGLang EAGLE MTP; attention is flashinfer -# (trtllm_mha is Blackwell-only). -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR \ - DURATION EP_SIZE -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# 256k-capped with-subagents corpus (470 traces): the unfiltered corpus has -# requests up to ~1M tokens the server would reject at this model's TP8 ceiling. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank. - HICACHE_HOST_POOL_COUNT="2" - HICACHE_WRITE_POLICY="write_through_selective" - HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SPEC_ARGS=( - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -# Acceptance is pinned to the committed golden AL (golden_al_distribution/README.md): -# 3.39 is qwen3.5_mtp.yaml at num_speculative_tokens=3, thinking_on. -# SGLANG_SIMULATE_ACC_TOKEN_MODE exists from SGLang v0.5.16, which is why the -# image is pinned there. EVAL_ONLY leaves it off: simulated acceptance commits -# drafted tokens regardless of target logits and the eval would score ~0. -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path="$MODEL_PATH" --served-model-name="$MODEL" - --host=0.0.0.0 - --port="$PORT" - --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" - --trust-remote-code - --tensor-parallel-size="$TP" - --data-parallel-size=1 - --expert-parallel-size="$EP_SIZE" - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend flashinfer - --enable-flashinfer-allreduce-fusion - --mem-fraction-static 0.8 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${SPEC_ARGS[@]}" - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh deleted file mode 100755 index c845a9c905..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi300x_mtp.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Qwen3.5-397B-A17B FP8 on MI300X with native EAGLE/NEXTN MTP. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -require_agentic_kv_offload_none - -SCHEDULER_RECV_INTERVAL=30 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -PARALLEL_ARGS=( - --tensor-parallel-size "$TP" - --data-parallel-size 1 - --ep-size "$EP_SIZE" -) - -# AgentX concurrency counts live session trees, not individual HTTP requests. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export PYTHONNOUSERSITE=1 -export SGLANG_ENABLE_SPEC_V2=1 -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --attention-backend aiter - --enable-aiter-allreduce-fusion - --mamba-ssm-dtype bfloat16 - --tokenizer-worker-num 6 - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 32768 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --mem-fraction-static 0.75 - --tokenizer-path "$MODEL" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --enable-metrics - --enable-cache-report -) - -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - # Aggregate serving exposes one logical SGLang Prometheus target. - export AIPERF_SERVER_METRICS_URLS="http://localhost:$PORT/metrics" - export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh deleted file mode 100755 index a6534ca076..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi325x_mtp.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# AgentX trace replay for Qwen3.5-397B-A17B FP8 on MI325X with SGLang -# native EAGLE/NEXTN MTP. Throughput uses the committed golden synthetic -# acceptance length; evaluation retains real target-model verification. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=30 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -rocm-smi || true -amd-smi || true - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -# Single aggregate engine: one backend metrics endpoint; AIPerf deduplicates -# the public copy build_replay_cmd discovers. -export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" -export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="sglang:" - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "SGLang server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) - -TOKENIZER_ARGS=() -if [ "$TP" -ge 4 ]; then - TOKENIZER_ARGS=(--tokenizer-worker-num 6) -fi - -# AgentX concurrency counts live session trees rather than HTTP requests. -# Keep capacity for subagent fan-out while bounding graph-capture memory. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -# With EAGLE enabled, SGLang reserves 15% of this budget for the draft model. -# TP2 needs an effective 0.8075 target-model budget to fit Qwen3.5's weights, -# hybrid state, and KV pools; TP4/TP8 retain the established 0.68 budget. -MEM_FRACTION_STATIC=0.80 -if [ "$TP" -eq 2 ]; then - MEM_FRACTION_STATIC=0.95 -fi - -export PYTHONNOUSERSITE=1 -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 -export SGLANG_USE_AITER=1 -export SGLANG_USE_AITER_UNIFIED_ATTN=1 -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -# Golden AL 3.39: committed Qwen3.5 thinking-on value for three speculative -# tokens. Evals use real target-model verification. -if [ "${EVAL_ONLY}" != "true" ]; then - export SGLANG_SIMULATE_ACC_LEN=3.39 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - --attention-backend aiter - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --mem-fraction-static "$MEM_FRACTION_STATIC" - --model-loader-extra-config '{"enable_multithread_load": true}' - --watchdog-timeout 1200 - --enable-aiter-allreduce-fusion - --page-size 16 - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$MAX_RUNNING_REQUESTS" - --max-prefill-tokens 32768 - --chunked-prefill-size 32768 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --stream-interval 50 - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --reasoning-parser qwen3 - --tool-call-parser qwen3_coder - --speculative-algorithm EAGLE - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --enable-metrics - --enable-cache-report -) - -write_command "$RESULT_DIR/sglang_command.txt" "${SGLANG_CMD[@]}" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --apply-chat-template" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh deleted file mode 100755 index ef23585e29..0000000000 --- a/benchmarks/single_node/agentic/qwen3.8next_fp4_b200_sglang_mtp.sh +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Qwen3.8-Flash-Next NVFP4 on B200 with SGLang native NEXTN MTP. The -# RadixArk/Qwen3.8-Flash-Next-NVFP4 checkpoint (126 GiB, quant_method = -# modelopt) ships native MTP modules, so NEXTN needs no external drafter, and -# fits on one GPU, so the cookbook command is --tp 1. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -model_checkpoint_is_complete() { - local directory="$1" - local index="$directory/model.safetensors.index.json" - - [[ -f "$index" ]] || return 1 - [[ -z "$(find "$directory" -name '*.incomplete' -print -quit 2>/dev/null)" ]] || return 1 - python3 - "$directory" <<'PYEOF' -import json -import os -import sys - -directory = sys.argv[1] -index = os.path.join(directory, "model.safetensors.index.json") -try: - with open(index, encoding="utf-8") as index_file: - shards = set(json.load(index_file)["weight_map"].values()) -except (OSError, KeyError, TypeError, ValueError): - sys.exit(1) -if not shards or any(not isinstance(shard, str) for shard in shards): - sys.exit(1) -sys.exit(any(not os.path.isfile(os.path.join(directory, shard)) for shard in shards)) -PYEOF -} - -if [[ -z "${MODEL_PATH:-}" ]] || ! model_checkpoint_is_complete "$MODEL_PATH"; then - echo "Error: complete staged Qwen3.8-Flash-Next NVFP4 checkpoint not found at ${MODEL_PATH:-}" >&2 - exit 1 -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. - HICACHE_ALIGNMENT_RESERVE_GB=$TP - HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then - echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 - exit 1 - fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 - exit 1 - fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy write_through_selective - ) -fi - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) - -# Parallel tokenization keeps 256k AgentX warmups below the client timeout (TP1 here). -TOKENIZER_ARGS=(--tokenizer-worker-num 6) - -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$MAX_RUNNING_REQUESTS" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -MEM_FRACTION_STATIC=0.80 -MAMBA_CACHE_ARGS=() -if [ "$CONC" -eq 16 ]; then - MEM_FRACTION_STATIC=0.90 - MAMBA_CACHE_ARGS=(--max-mamba-cache-size 160) -fi - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -# Keep server-side connections alive beyond AIPerf's 300-second client pool -# timeout so bursty AgentX trajectories cannot reuse a closing idle socket. -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - # golden_al_distribution/qwen3.8next_mtp.yaml: thinking_on[3] = 2.32 - # (3 speculative tokens per step; AgentX replays run with thinking on). - export SGLANG_SIMULATE_ACC_LEN=2.32 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - # Quantization is read from the checkpoint, so no --quantization flag; the - # hybrid GDN linear-attention layers take their own backends. - --linear-attn-prefill-backend flashinfer - --linear-attn-decode-backend flashinfer - # SGLang rejects flashinfer linear-attn decode on SM100+ without - # --mamba-ssm-dtype bfloat16; Hopper needs float32 instead (the - # gated_delta_rule_mtp verify kernel asserts a float32 state). - --mamba-ssm-dtype bfloat16 - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --reasoning-parser auto - # NEXTN silently resets --max-running-requests to 48 when it is unset, so - # this must stay explicit and sized to the AgentX concurrency. - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --mem-fraction-static "$MEM_FRACTION_STATIC" - "${MAMBA_CACHE_ARGS[@]}" - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --enable-metrics - --enable-cache-report - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -capture_cache_metrics -trap capture_cache_metrics EXIT - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh deleted file mode 100755 index f363b9fd2f..0000000000 --- a/benchmarks/single_node/agentic/qwen3.8next_fp4_b300_sglang_mtp.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Qwen3.8-Flash-Next NVFP4 on B300 with SGLang native NEXTN MTP. The -# RadixArk/Qwen3.8-Flash-Next-NVFP4 checkpoint (126 GiB, quant_method = -# modelopt) ships native MTP modules, so NEXTN needs no external drafter, and -# fits on one GPU, so the cookbook command is --tp 1. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars \ - MODEL TP CONC EP_SIZE KV_OFFLOADING \ - TOTAL_CPU_DRAM_GB RESULT_DIR DURATION -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126_256k -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # SGLang applies --hicache-size independently to Qwen's target KV and - # Mamba pools. Native NEXTN also creates a draft KV pool with the same - # slot count; its one attention layer adds 1/15 of the target KV bytes. - # Reserve 1 GB/rank for page alignment and enforce H * 31/15 per rank. - HICACHE_ALIGNMENT_RESERVE_GB=$TP - HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then - echo "Error: insufficient DRAM after HiCache alignment reserve" >&2 - exit 1 - fi - HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 15 / TP / 31)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB must be positive" >&2 - exit 1 - fi - PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 31 + 14) / 15 + HICACHE_ALIGNMENT_RESERVE_GB)) - if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - echo "HiCache CPU pools: ${HICACHE_SIZE_GB} GB target + Mamba + 1/15 draft per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB <= ${TOTAL_CPU_DRAM_GB} GB" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy write_through_selective - ) -fi - -PARALLEL_ARGS=( - --tp "$TP" - --dp 1 - --ep-size "$EP_SIZE" -) - -# TP4 needs parallel tokenization to keep 256k AgentX warmups below the client -# request timeout. Keep TP2 on SGLang's single-worker default: multi-tokenizer -# startup races with the TP2 HiCache shared-memory initialization path. -TOKENIZER_ARGS=() -if [ "$TP" -ge 4 ]; then - TOKENIZER_ARGS=(--tokenizer-worker-num 6) -fi - -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -[ "$CUDA_GRAPH_MAX_BS" -gt 64 ] && CUDA_GRAPH_MAX_BS=64 - -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true -# Keep server-side connections alive beyond AIPerf's 300-second client pool -# timeout so bursty AgentX trajectories cannot reuse a closing idle socket. -export SGLANG_TIMEOUT_KEEP_ALIVE=1800 - -if [ "${EVAL_ONLY}" != "true" ]; then - # golden_al_distribution/qwen3.8next_mtp.yaml: thinking_on[3] = 2.32 - # (3 speculative tokens per step; AgentX replays run with thinking on). - export SGLANG_SIMULATE_ACC_LEN=2.32 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - "${PARALLEL_ARGS[@]}" - # Quantization is read from the checkpoint, so no --quantization flag; the - # hybrid GDN linear-attention layers take their own backends. - --linear-attn-prefill-backend flashinfer - --linear-attn-decode-backend flashinfer - # SGLang rejects flashinfer linear-attn decode on SM100+ without - # --mamba-ssm-dtype bfloat16; Hopper needs float32 instead (the - # gated_delta_rule_mtp verify kernel asserts a float32 state). - --mamba-ssm-dtype bfloat16 - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 - --reasoning-parser auto - # NEXTN silently resets --max-running-requests to 48 when it is unset, so - # this must stay explicit and sized to the AgentX concurrency. - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --mem-fraction-static 0.80 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - "${TOKENIZER_ARGS[@]}" - --tokenizer-path "$MODEL" - --enable-metrics - --enable-cache-report - "${CACHE_ARGS[@]}" -) - -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -capture_cache_metrics() { - { - echo "=== SGLang cache metrics snapshot $(date --iso-8601=seconds) ===" - curl -fsS "http://localhost:$PORT/metrics" 2>/dev/null \ - | grep -E '^(sglang:(cache_hit_rate|cached_tokens_total|prompt_tokens_total|hicache_host_used_tokens|hicache_host_total_tokens|token_usage|num_requests_running|num_requests_waiting))' \ - || true - echo "============================================================" - } >> "$SERVER_LOG" -} - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -capture_cache_metrics -trap capture_cache_metrics EXIT - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - REPLAY_CMD+=" --server-metrics http://localhost:$PORT/metrics" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh deleted file mode 100755 index 51a9a09ba3..0000000000 --- a/benchmarks/single_node/agentic/qwen3.8next_fp8_h200_sglang_mtp.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Qwen3.8-Flash-Next FP8 on H200 with SGLang NEXTN MTP. Hopper has no NVFP4 -# tensor cores, so this arm serves Qwen/Qwen3.8-Flash-Next-FP8 (172.8 GiB); -# attention stays flashinfer (trtllm_mha is Blackwell-only). -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars \ - MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR \ - DURATION EP_SIZE -check_env_vars EVAL_ONLY - -SCHEDULER_RECV_INTERVAL=10 - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# 256k-capped with-subagents corpus (470 traces): the unfiltered corpus has -# requests up to ~1M tokens the server would reject at this model's TP8 ceiling. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank. - HICACHE_HOST_POOL_COUNT="2" - HICACHE_WRITE_POLICY="write_through_selective" - HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 -export SGLANG_ENABLE_SPEC_V2=1 - -SPEC_ARGS=( - --speculative-algorithm NEXTN - --speculative-num-steps 3 - --speculative-eagle-topk 1 - --speculative-num-draft-tokens 4 -) - -# Acceptance is pinned to the committed golden AL (golden_al_distribution/README.md). -# EVAL_ONLY leaves it off: simulated acceptance commits drafted tokens -# regardless of target logits and the eval would score ~0. -if [ "${EVAL_ONLY}" != "true" ]; then - # golden_al_distribution/qwen3.8next_mtp.yaml: thinking_on[3] = 2.32 - # (3 speculative tokens per step; AgentX replays run with thinking on). - export SGLANG_SIMULATE_ACC_LEN=2.32 - export SGLANG_SIMULATE_ACC_METHOD=match-expected - export SGLANG_SIMULATE_ACC_TOKEN_MODE=real-draft-token -fi - -{ set +x; } 2>/dev/null -# AgentX concurrency counts live session trees; leave room for subagent -# fan-out without spending HBM on graphs above useful batch sizes. -MAX_RUNNING_REQUESTS=$((2 * CONC)) -CUDA_GRAPH_MAX_BS="$CONC" -if [ "$CUDA_GRAPH_MAX_BS" -gt 64 ]; then - CUDA_GRAPH_MAX_BS=64 -fi - -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --trust-remote-code - # Cookbook H200 / FP8 low-latency flags: TP4 with EP4 shards the 512-expert - # MoE with expert parallelism rather than sharding attention eight ways. - --tp-size "$TP" - --ep-size "$EP_SIZE" - --dp-size 1 - --mem-fraction-static 0.85 - --chunked-prefill-size 8192 - --linear-attn-prefill-backend flashinfer - --linear-attn-decode-backend flashinfer - # float32, not the cookbook's bfloat16: with NEXTN the GDN backend verifies - # through flashinfer's gated_delta_rule_mtp, which asserts - # initial_state.dtype == torch.float32 and aborts CUDA graph capture - # (flashinfer/gdn_decode.py:761 via gdn_backend.py target_verify). - --mamba-ssm-dtype float32 - "${SPEC_ARGS[@]}" - --reasoning-parser auto - # NEXTN silently resets --max-running-requests to 48 when it is unset, so - # this must stay explicit and sized to the AgentX concurrency. - --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi