diff --git a/crates/grill-perf/examples/prefill-ladder-v1.json b/crates/grill-perf/examples/prefill-ladder-v1.json new file mode 100644 index 0000000..0501c3a --- /dev/null +++ b/crates/grill-perf/examples/prefill-ladder-v1.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "name": "prefill-ladder-v1", + "request": { + "profile": "vllm-fixed-v1", + "stream": true, + "output": { "tokens": 1, "mode": "cap" }, + "cache": "reported-prefix-zero", + "temperature_milli": 0, + "top_p_milli": 1000, + "seed": null, + "thinking_control": { "kind": "vllm-enable-thinking-v1", "enabled": false } + }, + "limits": { + "total_ms": 2700000, + "idle_ms": 2700000, + "response_bytes": 65536, + "wave_buffer_bytes": 4194304 + }, + "cases": [ + { + "id": "prefill-2k", + "messages": [ + { "role": "user", "content": "[prefill-ladder {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 2048 } + }, + { + "id": "prefill-8k", + "messages": [ + { "role": "user", "content": "[prefill-ladder {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 8192 } + }, + { + "id": "prefill-32k", + "messages": [ + { "role": "user", "content": "[prefill-ladder {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 32768 } + }, + { + "id": "prefill-128k", + "messages": [ + { "role": "user", "content": "[prefill-ladder {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 131072 } + } + ], + "cells": [ + { "id": "prefill-2k-1", "case": "prefill-2k", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "prefill-8k-1", "case": "prefill-8k", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "prefill-32k-1", "case": "prefill-32k", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "prefill-128k-1", "case": "prefill-128k", "concurrency": 1, "warmup_trials": 1, "trials": 3 } + ] +} diff --git a/crates/grill-perf/tests/cli.rs b/crates/grill-perf/tests/cli.rs index 0bf55b6..db786e6 100644 --- a/crates/grill-perf/tests/cli.rs +++ b/crates/grill-perf/tests/cli.rs @@ -1172,6 +1172,73 @@ fn absent_fill_preserves_literal_placeholders_and_plan_identity() { ); } +#[test] +fn prefill_ladder_admits_and_requires_reported_zero_prefix_evidence() { + let workload: Value = + serde_json::from_str(include_str!("../examples/prefill-ladder-v1.json")).unwrap(); + for (name, cached, error) in [ + ("zero", Some(0), None), + ( + "missing", + None, + Some("provider_prefix_cache_usage_unavailable"), + ), + ( + "nonzero", + Some(1), + Some("provider_reported_prefix_cache_nonzero"), + ), + ] { + let temp = Temp::new(); + let server = Server::new(move |mut s, _, request| { + assert_eq!( + request["chat_template_kwargs"], + json!({"enable_thinking":false}) + ); + header(&mut s, "text/event-stream"); + frame(&mut s, json!({"choices":[{"delta":{"content":"x"}}]})); + finish(&mut s, Some(1), cached); + }); + let output = run(&temp, &server, name, &workload); + if let Some(error) = error { + assert_eq!(output.status.code(), Some(2)); + let receipt = wave(&temp, name, 0); + assert_eq!(receipt["attempts"][0]["status"], "complete"); + assert!( + receipt["attempts"][0]["eligibility_errors"] + .as_array() + .unwrap() + .iter() + .any(|value| value == error) + ); + assert!(receipt["achieved_completion_tokens_per_second"].is_null()); + } else { + successful(&output); + let plan = value(temp.path(name).join("plan.json")); + for spec in plan["waves"].as_array().unwrap() { + let receipt = wave(&temp, name, spec["index"].as_u64().unwrap() as usize); + assert_eq!(receipt["attempts"][0]["status"], "complete"); + assert!( + receipt["attempts"][0]["eligibility_errors"] + .as_array() + .unwrap() + .is_empty() + ); + assert!(receipt["achieved_completion_tokens_per_second"].is_number()); + } + successful( + &cli() + .arg("compare") + .arg(temp.path(name)) + .arg(temp.path(name)) + .arg("--json") + .output() + .unwrap(), + ); + } + } +} + #[test] fn fill_bytes_count_toward_each_cells_wave_buffer_bound() { let temp = Temp::new(); diff --git a/docs/performance/README.md b/docs/performance/README.md index 80e75f4..2b93f2d 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -66,6 +66,60 @@ explicit declarations when a longer quiet prefill is intended; raising total alone does not fix idle expiry. This is a bounded policy, not a server-runtime guarantee, and longer budgets can lengthen cooperative pause/wave drain. +[`prefill-ladder-v1.json`](../../crates/grill-perf/examples/prefill-ladder-v1.json) +is a separate strict-cold ladder with approximate prompt targets of +2k/8k/32k/128k, not a revision of the sparkDash workload. It uses the existing +`" the"` fill generator with an early per-attempt `{salt}`, concurrency 1, +one warmup and three measured trials per size. Repeat counts are size proxies, +not measured tokenizer counts: the template, salt, header/footer and tokenizer +determine actual prompt tokens. Check reported usage and the server's context +limit before interpreting a size label as a token count. + +The ladder streams with an output **cap** of 1, not exact output, temperature +zero and top_p one. It explicitly requests +`thinking_control: {"kind":"vllm-enable-thinking-v1","enabled":false}`. +The template must support `chat_template_kwargs.enable_thinking`; this example +does not establish live provider compatibility or prove the setting was honored. +Reported reasoning and observed answer channels remain separate evidence. +Both selected deadlines are 2,700,000 ms, within the 3,600,000 ms ceiling. + +`reported-prefix-zero` requires explicit provider-reported zero cached prompt +tokens. Missing or nonzero cache evidence remains ineligible; neither the salt +nor a prefill rate proves a cold cache. Switching to `observe` changes the +workload and its claim. Real tokenizer counts, long-context endpoint support, +thinking-control compliance and live prefill rates remain unverified. + +### Larger-prompt buffer sizing + +The ladder's response cap is 65,536 bytes, independent of request size. +For streaming, the admitted per-wave allowance is +`concurrency * (2*response_bytes + 6*256KiB + 512KiB + fill_bytes)`, +where `fill_bytes = unit.len()*repeat`. At concurrency 1 the non-fill +allowance is 2,228,224 bytes: + +| Approximate target | `" the"` repeats | Fill bytes | Required wave allowance | +|---|---:|---:|---:| +| 128k (shipped) | 131,072 | 524,288 | 2,752,512 bytes | +| 256k (sizing example only) | 262,144 | 1,048,576 | 3,276,800 bytes | + +The declared 4,194,304-byte wave budget covers these allowances. The hypothetical +256k row is not a shipped or live-qualified case. Fill bytes exclude the rendered +header/footer, template controls and JSON envelope. Each complete serialized +request must independently fit 2,097,152 bytes; increasing `response_bytes` +does not increase that request cap. + +The space/letter fill needs no JSON escaping, but arbitrary fill does: a control +byte can expand to a six-byte `\uXXXX` escape. A 524,288-byte control-character +fill can therefore require 3,145,728 bytes before the envelope and fail the +request cap. Reservation receipts embed request bodies as JSON strings, escaping +quotes and backslashes again. Admission also bounds those escaped strings times +concurrency within the reservation allowance described in the +[contract](CONTRACT.md#workload-admission). +Rendered messages, encoded requests and escaped reservation copies coexist +during preparation; the wave formula counts fill only once and is not a peak +RSS guarantee. Serialized reservation buffers are released before dispatch. +Neither the response cap nor the wave allowance proves server context capacity. + For a smaller compatibility probe, use [`recipe-smoke.json`](../../crates/grill-perf/examples/recipe-smoke.json): concurrency 1 and 2, a 64-token cap, and nine requests per run.