diff --git a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh index 764e40e46..13155407d 100644 --- a/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_mi355x_atom_mtp.sh @@ -129,6 +129,13 @@ esac export ATOM_ENABLE_REPLAYSSM export AITER_REUSE_IDENTICAL_COMM_GROUPS +# From concurrency 16 up, hold a ready prefill for 4 decode passes instead of +# interleaving it into every step; MAX_QUEUE_MS bounds the wait. 14 runs without it. +if [ "$CONC" -ge 16 ]; then + export ATOM_PREFILL_DECODE_INTERVAL=4 + export ATOM_PREFILL_DELAYER_MAX_QUEUE_MS=5000 +fi + # Full CUDA graphs over [1 .. window * (1 + draft tokens)]: the verify step # submits one row per draft token on top of the accepted token, so capturing # only up to the window would send every speculative decode down the eager path. @@ -199,6 +206,8 @@ export AITER_FLYDSL_STAGE2_FP8=1 # costs more in evictions than its reuse is worth on these traces. export ATOM_STATE_CHECKPOINT_DEMAND=0 export ATOM_GDN_SSM_DTYPE=fp16 +# FlyDSL FP8 prefill attention; ATOM defaults it off. +export ATOM_USE_FLYDSL_FP8_PREFILL_ATTN=1 # https://github.com/SemiAnalysisAI/InferenceX/blob/main/golden_al_distribution/kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml # 7 draft tokens -> AL 3.84 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 543c40458..c771f07db 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -690,7 +690,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: # wider in-flight window needs the deeper pool to keep the paged KV # resident. kimik3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0911 + image: rocm/atom-dev:nightly_202609251613 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1bc132cd3..b6108908a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8950,3 +8950,14 @@ description: - "Update SGLang image from v0.5.19-cu130 to v0.5.20-cu130 and pass --cuda-graph-max-bs-decode after the alias removal." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3334 + +- config-keys: + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Move the MI355X Kimi-K3 FP4 ATOM AgentX submission to image rocm/atom-dev:ubuntu24.04_py3.12_pytorch_release_2.10.0_kimi_k3_agentic_0924, tracking recipes/Agentic-Kimi-K3.md as retuned in ROCm/ATOM#2382." + - "Enable FlyDSL FP8 prefill attention (ATOM_USE_FLYDSL_FP8_PREFILL_ATTN=1) on every concurrency; ATOM defaults it off." + - "Hold a ready prefill for four decode passes from concurrency 16 up (ATOM_PREFILL_DECODE_INTERVAL=4, ATOM_PREFILL_DELAYER_MAX_QUEUE_MS=5000). Concurrency 1, 4 and 14 are unchanged." + - "Keep the published concurrency set [1, 4, 14, 16, 48, 56, 72], max-num-seqs, max-num-batched-tokens, gpu-memory-utilization, the CUDA-graph ladder, dcp-size, draft depth, synthetic acceptance, ReplaySSM placement and LMCache sizing unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3407