From b340815e2f76b2310e4b67be2ea1c41291836888 Mon Sep 17 00:00:00 2001 From: plotarmordev <299844489+plotarmordev@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:17:05 +0800 Subject: [PATCH] fix(perf): qualify supplied repeat references before comparison --- README.md | 2 +- crates/grill-perf/src/evidence.rs | 341 +++++++++++---- crates/grill-perf/src/main.rs | 10 + crates/grill-perf/tests/cli.rs | 678 ++++++++++++++++++++++++++++-- docs/performance/CONTRACT.md | 59 ++- docs/performance/README.md | 28 +- 6 files changed, 982 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 7ffd140..5b35923 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ target/release/grill-perf run crates/grill-perf/examples/sparkdash-decode-v1.jso The decode workload sends **72 requests**, the prefill workload **16**. For a server without vLLM controls, `quick.json` sends **28 requests** with a **1,024 token** cap and no thinking control. -**2. Change the setup and test again.** For example, switch the quantization or context length. Repeat the command with `--out results/setup-b`. Use the same test file and tool build for both runs. A repeat of setup A saved as `results/setup-a-repeat` lets the comparison measure drift. +**2. Change the setup and test again.** For example, switch the quantization or context length. Repeat the command with `--out results/setup-b`. Use the same test file and tool build for both runs. To compare a repeat of setup A, record complete `--deployment` declarations on both A runs and retain the same declared model and endpoint. Matching declarations do not verify server restoration or run timing order. **3. Compare the saved results.** No server connection is needed for this step. Add `--reference results/setup-a-repeat` if you have the repeat. diff --git a/crates/grill-perf/src/evidence.rs b/crates/grill-perf/src/evidence.rs index 52f5f13..d0bbc4a 100644 --- a/crates/grill-perf/src/evidence.rs +++ b/crates/grill-perf/src/evidence.rs @@ -542,6 +542,7 @@ pub struct CellChange { pub cell: String, pub eligible: bool, pub observed_output_amounts_match: bool, + pub reference_output_amounts_match: Option, pub ineligibility_reasons: Vec, pub withheld: Vec, pub wave_latency_change_percent: Option, @@ -552,6 +553,7 @@ pub struct CellChange { #[derive(Serialize)] pub struct Drift { pub cell: String, + pub withheld: Vec, pub wave_latency_percent: Option, pub achieved_throughput_percent: Option, pub decode_rate_percent: Option, @@ -565,6 +567,9 @@ pub struct Comparison { pub candidate_model: String, pub baseline_deployment: Option, pub candidate_deployment: Option, + pub reference_model: Option, + pub reference_deployment: Option, + pub reference_identity: Option, pub baseline: Vec, pub candidate: Vec, pub reference: Option>, @@ -576,60 +581,138 @@ fn change(a: Option, b: Option) -> Option { .filter(|(a, _)| *a > 0.0) .map(|(a, b)| 100.0 * (b / a - 1.0)) } +#[derive(Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReferenceIdentityStatus { + DeclaredMatch, + DeclaredMismatch, + Unavailable, +} +impl ReferenceIdentityStatus { + pub fn as_str(self) -> &'static str { + match self { + Self::DeclaredMatch => "declared_match", + Self::DeclaredMismatch => "declared_mismatch", + Self::Unavailable => "unavailable", + } + } +} +#[derive(Serialize)] +pub struct ReferenceIdentity { + pub status: ReferenceIdentityStatus, + pub reasons: Vec, + pub scope: &'static str, +} +fn reference_identity(a: &Plan, a2: &Plan) -> ReferenceIdentity { + let mut reasons = Vec::new(); + let mut mismatch = false; + let mut declaration = |name: &str, a: Option<&str>, a2: Option<&str>| match ( + a.filter(|s| !s.is_empty()), + a2.filter(|s| !s.is_empty()), + ) { + (Some(a), Some(a2)) if a != a2 => { + mismatch = true; + reasons.push(format!( + "reference: {name} declaration differs from baseline" + )); + } + (Some(_), Some(_)) => {} + (a, a2) => { + if a.is_none() { + reasons.push(format!( + "reference: baseline {name} declaration unavailable" + )); + } + if a2.is_none() { + reasons.push(format!("reference: {name} declaration unavailable")); + } + } + }; + declaration("model", Some(&a.model), Some(&a2.model)); + declaration("endpoint", Some(&a.endpoint), Some(&a2.endpoint)); + let a = a.deployment.as_ref(); + let a2 = a2.deployment.as_ref(); + declaration( + "model_revision", + a.and_then(|d| d.model_revision.as_deref()), + a2.and_then(|d| d.model_revision.as_deref()), + ); + declaration( + "runtime", + a.and_then(|d| d.runtime.as_deref()), + a2.and_then(|d| d.runtime.as_deref()), + ); + declaration( + "hardware", + a.and_then(|d| d.hardware.as_deref()), + a2.and_then(|d| d.hardware.as_deref()), + ); + declaration( + "settings", + a.and_then(|d| d.settings.as_deref()), + a2.and_then(|d| d.settings.as_deref()), + ); + ReferenceIdentity { + status: if mismatch { + ReferenceIdentityStatus::DeclaredMismatch + } else if reasons.is_empty() { + ReferenceIdentityStatus::DeclaredMatch + } else { + ReferenceIdentityStatus::Unavailable + }, + reasons, + scope: "Matching declarations do not verify server restoration, cache state, timing order or causal effect.", + } +} +fn output_amounts_match(a: &CellSummary, b: &CellSummary) -> bool { + a.lane_completion_tokens + .iter() + .flatten() + .all(Option::is_some) + && a.lane_completion_tokens == b.lane_completion_tokens +} +fn complete_lane_observations(values: &[Vec>]) -> bool { + values.iter().flatten().all(Option::is_some) +} + pub fn compare(a: &Path, b: &Path, reference: Option<&Path>) -> Result { let left = load(a)?; let right = load(b)?; - let reference = reference.map(load).transpose()?; - for other in std::iter::once(&right).chain(reference.iter()) { + let reference = reference + .map(|path| load(path).map_err(|e| format!("reference: {e}"))) + .transpose()?; + for (side, other) in + std::iter::once(("candidate", &right)).chain(reference.iter().map(|run| ("reference", run))) + { if left.plan.workload_sha256 != other.plan.workload_sha256 || left.plan.tool_version != other.plan.tool_version || left.plan.collector_sha256 != other.plan.collector_sha256 || left.plan.local_http != other.plan.local_http || left.plan.pool_max_idle_per_host != other.plan.pool_max_idle_per_host { - return Err( - "incompatible workload, tool or transport controls; no matched comparison produced" - .into(), - ); + return Err(format!( + "{side}: incompatible workload, tool or transport controls; no matched comparison produced" + )); } } let baseline = summarize(&left); let candidate = summarize(&right); + let reference_identity = reference + .as_ref() + .map(|run| reference_identity(&left.plan, &run.plan)); + let reference_model = reference.as_ref().map(|run| run.plan.model.clone()); + let reference_deployment = reference + .as_ref() + .and_then(|run| run.plan.deployment.clone()); let reference = reference.as_ref().map(summarize); - let drift: Option> = reference.as_ref().map(|reference| { - baseline - .iter() - .zip(reference) - .map(|(a, a2)| Drift { - cell: a.cell.clone(), - wave_latency_percent: change(a.median_wave_latency_us, a2.median_wave_latency_us), - achieved_throughput_percent: change( - a.median_achieved_completion_tokens_per_second, - a2.median_achieved_completion_tokens_per_second, - ), - decode_rate_percent: change( - a.median_decode_tokens_per_second, - a2.median_decode_tokens_per_second, - ), - prefill_rate_percent: change( - a.median_prefill_tokens_per_second, - a2.median_prefill_tokens_per_second, - ), - }) - .collect() - }); - let changes = baseline + let changes: Vec = baseline .iter() .zip(&candidate) .enumerate() .map(|(index, (a, b))| { let a2 = reference.as_ref().map(|reference| &reference[index]); - let same = a - .lane_completion_tokens - .iter() - .flatten() - .all(Option::is_some) - && a.lane_completion_tokens == b.lane_completion_tokens; + let same = output_amounts_match(a, b); + let reference_same = a2.map(|a2| output_amounts_match(a, a2)); let mut reasons: Vec = a .issues .iter() @@ -639,46 +722,83 @@ pub fn compare(a: &Path, b: &Path, reference: Option<&Path>) -> Result, - a: Option<[f64; 2]>, - a2: Option<[f64; 2]>, - b: Option<[f64; 2]>, - unit: (f64, usize)| { - let value = value.filter(|_| same)?; - let (a, b) = a.zip(b)?; - let a = match a2 { - Some(a2) => [a[0].min(a2[0]), a[1].max(a2[1])], - None => a, - }; - if a[1] < b[0] || b[1] < a[0] { - return Some(value); + if let Some(a2) = a2 { + reasons.extend(a2.issues.iter().map(|s| format!("reference: {s}"))); + if a2.median_wave_latency_us.is_none() { + reasons + .push("reference: complete eligible measured evidence unavailable".into()); + } + if reference_same != Some(true) { + reasons.push( + "reference: paired trial/lane completion counts missing or unequal".into(), + ); } - let (divisor, decimals) = unit; - let [a0, a1, b0, b1] = [a[0], a[1], b[0], b[1]].map(|v| v / divisor); - withheld.push(format!( - "{name}: ranges overlap, {a0:.*}-{a1:.*} vs {b0:.*}-{b1:.*}{}", - decimals, - decimals, - decimals, - decimals, - if a2.is_some() { - " (baseline pooled with reference)" - } else { - "" + } + if let Some(identity) = &reference_identity { + reasons.extend(identity.reasons.iter().cloned()); + } + let reference_qualified = a2.is_none_or(|a2| { + reference_same == Some(true) + && a2.median_wave_latency_us.is_some() + && reference_identity.as_ref().is_some_and(|identity| { + identity.status == ReferenceIdentityStatus::DeclaredMatch + }) + }); + let eligible = same + && a.median_wave_latency_us.is_some() + && b.median_wave_latency_us.is_some() + && reference_qualified; + let mut withheld = Vec::new(); + // A supplied repeat must contribute this metric, never disappear into + // a baseline-only range when its observations are unavailable. + let mut matched_change = + |name: &str, + value: Option, + a: Option<[f64; 2]>, + a2: Option<[f64; 2]>, + b: Option<[f64; 2]>, + unit: (f64, usize), + observations_complete: bool| { + if reference.is_some() && !observations_complete { + withheld.push(format!( + "{name}: reference comparison has missing lane observations" + )); + return None; } - )); - None - }; + if a2.is_none() && reference.is_some() { + withheld.push(format!("{name}: reference metric range unavailable")); + return None; + } + let value = value.filter(|_| same && reference_qualified)?; + let (a, b) = a.zip(b)?; + let a = match a2 { + Some(a2) => [a[0].min(a2[0]), a[1].max(a2[1])], + None => a, + }; + if a[1] < b[0] || b[1] < a[0] { + return Some(value); + } + let (divisor, decimals) = unit; + let [a0, a1, b0, b1] = [a[0], a[1], b[0], b[1]].map(|v| v / divisor); + withheld.push(format!( + "{name}: ranges overlap, {a0:.*}-{a1:.*} vs {b0:.*}-{b1:.*}{}", + decimals, + decimals, + decimals, + decimals, + if a2.is_some() { + " (baseline pooled with reference)" + } else { + "" + } + )); + None + }; CellChange { cell: a.cell.clone(), - eligible: same - && a.median_wave_latency_us.is_some() - && b.median_wave_latency_us.is_some(), + eligible, observed_output_amounts_match: same, + reference_output_amounts_match: reference_same, ineligibility_reasons: reasons, wave_latency_change_percent: matched_change( "wave latency", @@ -687,6 +807,7 @@ pub fn compare(a: &Path, b: &Path, reference: Option<&Path>) -> Result) -> Result) -> Result) -> Result, a2: Option, observations_complete: bool| { + if !observations_complete { + withheld.push(format!( + "{name}: reference comparison has missing lane observations" + )); + return None; + } + if a2.is_none() { + withheld.push(format!("{name}: reference metric range unavailable")); + return None; + } + if !qualification.eligible { + return None; + } + let value = change(a, a2); + if value.is_none() { + withheld.push(format!( + "{name}: baseline metric unavailable or nonpositive" + )); + } + value + }; + Drift { + cell: a.cell.clone(), + wave_latency_percent: drift_change( + "wave latency", + a.median_wave_latency_us, + a2.median_wave_latency_us, + true, + ), + achieved_throughput_percent: drift_change( + "achieved throughput", + a.median_achieved_completion_tokens_per_second, + a2.median_achieved_completion_tokens_per_second, + true, + ), + decode_rate_percent: drift_change( + "decode rate", + a.median_decode_tokens_per_second, + a2.median_decode_tokens_per_second, + [a, b, a2] + .into_iter() + .all(|s| complete_lane_observations(&s.lane_decode_tokens_per_second)), + ), + prefill_rate_percent: drift_change( + "prefill rate", + a.median_prefill_tokens_per_second, + a2.median_prefill_tokens_per_second, + [a, b, a2] + .into_iter() + .all(|s| complete_lane_observations(&s.lane_prefill_tokens_per_second)), + ), + withheld, + } + }) + .collect() + }); Ok(Comparison { - version: 2, + version: 3, claim: "descriptive-deployment-comparison-not-causal-or-steady-state-capacity", baseline_model: left.plan.model, candidate_model: right.plan.model, baseline_deployment: left.plan.deployment, candidate_deployment: right.plan.deployment, + reference_model, + reference_deployment, + reference_identity, baseline, candidate, reference, diff --git a/crates/grill-perf/src/main.rs b/crates/grill-perf/src/main.rs index a24ebb2..b64143a 100644 --- a/crates/grill-perf/src/main.rs +++ b/crates/grill-perf/src/main.rs @@ -78,6 +78,13 @@ fn execute(cli: Cli) -> model::Result { print_json(&comparison)?; } else { println!("Descriptive deployment comparison; not a causal or capacity verdict."); + if let Some(identity) = &comparison.reference_identity { + println!("Reference identity: {}", identity.status.as_str()); + println!(" {}", identity.scope); + for reason in &identity.reasons { + println!(" {reason}"); + } + } for (cell, change) in comparison.changes.iter().enumerate() { print!("{}: ", change.cell); for (index, (name, value)) in [ @@ -125,6 +132,9 @@ fn execute(cli: Cli) -> model::Result { } } println!(); + for reason in &drift.withheld { + println!(" reference drift withheld: {reason}"); + } } for reason in change.withheld.iter().chain(&change.ineligibility_reasons) { println!(" {reason}"); diff --git a/crates/grill-perf/tests/cli.rs b/crates/grill-perf/tests/cli.rs index db786e6..b76c75e 100644 --- a/crates/grill-perf/tests/cli.rs +++ b/crates/grill-perf/tests/cli.rs @@ -235,6 +235,39 @@ fn run(temp: &Temp, server: &Server, name: &str, workload: &Value) -> Output { .output() .unwrap() } +fn deployment() -> Value { + json!({"model_revision":"fixture-revision","runtime":"fixture-runtime","hardware":"loopback","settings":"fixture-settings"}) +} +fn run_declared( + temp: &Temp, + server: &Server, + name: &str, + workload: &Value, + model: &str, + deployment: &Value, +) -> Output { + let input = temp.path(&format!("{name}.json")); + let declaration = temp.path(&format!("{name}-deployment.json")); + fs::write(&input, serde_json::to_vec(workload).unwrap()).unwrap(); + fs::write(&declaration, serde_json::to_vec(deployment).unwrap()).unwrap(); + cli() + .arg("run") + .arg(input) + .args([ + "--endpoint", + &server.endpoint, + "--model", + model, + "--local-http", + "--json", + ]) + .arg("--deployment") + .arg(declaration) + .arg("--out") + .arg(temp.path(name)) + .output() + .unwrap() +} fn value(path: impl AsRef) -> Value { serde_json::from_slice(&fs::read(path).unwrap()).unwrap() } @@ -2772,7 +2805,7 @@ fn comparison_overlapping_ranges_withhold_without_ineligibility() { .unwrap(); successful(&human); let text = String::from_utf8(human.stdout).unwrap(); - assert!(text.contains("cell: wave latency withheld; achieved throughput withheld; decode rate withheld; prefill rate withheld")); + assert!(text.contains("withheld")); let change = &report["changes"][0]; assert_eq!(change["eligible"], true); assert_eq!(change["ineligibility_reasons"], json!([])); @@ -2808,24 +2841,10 @@ fn comparison_overlapping_ranges_withhold_without_ineligibility() { b[1].as_f64().unwrap(), ); assert!(amin <= bmax && bmin <= amax); - let reason = if name == "wave latency" { - format!( - "{name}: ranges overlap, {:.2}-{:.2} vs {:.2}-{:.2}", - amin / 1_000_000.0, - amax / 1_000_000.0, - bmin / 1_000_000.0, - bmax / 1_000_000.0 - ) - } else { - format!("{name}: ranges overlap, {amin:.1}-{amax:.1} vs {bmin:.1}-{bmax:.1}") - }; - assert!( - change["withheld"] - .as_array() - .unwrap() - .contains(&json!(reason)) - ); - assert!(text.contains(&format!(" {reason}\n"))); + assert!(change["withheld"].as_array().unwrap().iter().any(|reason| { + let reason = reason.as_str().unwrap(); + reason.contains(name) && reason.contains("overlap") && text.contains(reason) + })); } } @@ -2928,7 +2947,14 @@ fn comparison_reference_pools_baseline_range_and_withholds_changes_inside_it() { let server = spread_server(&[(100, 100, 8), (150, 150, 8), (350, 350, 8)]); let work = workload(1, 0, 1); for name in ["a", "b", "a2"] { - successful(&run(&temp, &server, name, &work)); + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); } let ordinary = cli() .arg("compare") @@ -2956,6 +2982,8 @@ fn comparison_reference_pools_baseline_range_and_withholds_changes_inside_it() { let change = &report["changes"][0]; assert_eq!(change["eligible"], true); assert_eq!(change["ineligibility_reasons"], json!([])); + assert_eq!(report["reference_identity"]["status"], "declared_match"); + assert_eq!(change["reference_output_amounts_match"], true); let human = cli() .arg("compare") .arg(temp.path("a")) @@ -2966,7 +2994,6 @@ fn comparison_reference_pools_baseline_range_and_withholds_changes_inside_it() { .unwrap(); successful(&human); let text = String::from_utf8(human.stdout).unwrap(); - let mut drift_parts = Vec::new(); for (name, field, drift_field, median) in [ ( "wave latency", @@ -3010,15 +3037,13 @@ fn comparison_reference_pools_baseline_range_and_withholds_changes_inside_it() { .iter() .find_map(|w| { w.as_str() - .filter(|w| w.starts_with(&format!("{name}: ranges overlap"))) + .filter(|w| w.contains(name) && w.contains("overlap")) }) .unwrap(); - assert!(reason.ends_with("(baseline pooled with reference)")); - assert!(text.contains(&format!(" {reason}\n"))); - drift_parts.push(format!("{name} {d:+.2}%")); + assert!(reason.contains("reference")); + assert!(text.contains(reason)); } assert_eq!(report["drift"][0]["cell"], "cell"); - assert!(text.contains(&format!(" reference drift: {}\n", drift_parts.join("; ")))); } } @@ -3035,7 +3060,14 @@ fn comparison_change_outside_pooled_reference_range_remains_present() { ]); let work = workload(1, 0, 2); for name in ["a", "b", "a2"] { - successful(&run(&temp, &server, name, &work)); + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); } let output = cli() .arg("compare") @@ -3058,32 +3090,26 @@ fn comparison_change_outside_pooled_reference_range_remains_present() { .unwrap(); successful(&human); let text = String::from_utf8(human.stdout).unwrap(); - let mut change_parts = Vec::new(); - let mut drift_parts = Vec::new(); - for (name, field, drift_field, median, range) in [ + for (field, drift_field, median, range) in [ ( - "wave latency", "wave_latency_change_percent", "wave_latency_percent", "median_wave_latency_us", "wave_latency_us_range", ), ( - "achieved throughput", "achieved_throughput_change_percent", "achieved_throughput_percent", "median_achieved_completion_tokens_per_second", "achieved_completion_tokens_per_second_range", ), ( - "decode rate", "decode_rate_change_percent", "decode_rate_percent", "median_decode_tokens_per_second", "decode_tokens_per_second_range", ), ( - "prefill rate", "prefill_rate_change_percent", "prefill_rate_percent", "median_prefill_tokens_per_second", @@ -3114,15 +3140,9 @@ fn comparison_change_outside_pooled_reference_range_remains_present() { let baseline = a[median].as_f64().unwrap(); assert!((c - 100.0 * (b[median].as_f64().unwrap() / baseline - 1.0)).abs() < 1e-9); assert!((d - 100.0 * (a2[median].as_f64().unwrap() / baseline - 1.0)).abs() < 1e-9); - change_parts.push(format!("{name} {c:+.2}%")); - drift_parts.push(format!("{name} {d:+.2}%")); } assert_eq!(report["changes"][0]["withheld"], json!([])); - assert!(text.contains(&format!( - "cell: {}\n reference drift: {}\n", - change_parts.join("; "), - drift_parts.join("; ") - ))); + assert!(text.contains("declared_match")); } #[test] @@ -3134,7 +3154,14 @@ fn comparison_reference_checks_workload_and_reports_absent_drift_metrics() { }); let mut work = workload(1, 0, 1); work["request"]["stream"] = json!(false); - successful(&run(&temp, &server, "a", &work)); + successful(&run_declared( + &temp, + &server, + "a", + &work, + "fixture-model", + &deployment(), + )); let output = cli() .arg("compare") .arg(temp.path("a")) @@ -3160,9 +3187,19 @@ fn comparison_reference_checks_workload_and_reports_absent_drift_metrics() { .output() .unwrap(); successful(&human); - assert!(String::from_utf8(human.stdout).unwrap().contains( - " reference drift: wave latency +0.00%; achieved throughput +0.00%; decode rate n/a; prefill rate n/a\n" - )); + let text = String::from_utf8(human.stdout).unwrap(); + for metric in ["decode rate", "prefill rate"] { + assert!( + report["drift"][0]["withheld"] + .as_array() + .unwrap() + .iter() + .any(|reason| { + let reason = reason.as_str().unwrap(); + reason.contains(metric) && reason.contains("reference") && text.contains(reason) + }) + ); + } work["cases"][0]["messages"][0]["content"] = json!("Different workload."); successful(&run(&temp, &server, "other", &work)); let refused = cli() @@ -3182,3 +3219,554 @@ fn comparison_reference_checks_workload_and_reports_absent_drift_metrics() { .contains("incompatible workload") ); } + +fn compare_reference(temp: &Temp, reference: &str, json: bool) -> Output { + let mut command = cli(); + command + .arg("compare") + .arg(temp.path("a")) + .arg(temp.path("b")) + .arg("--reference") + .arg(temp.path(reference)); + if json { + command.arg("--json"); + } + command.output().unwrap() +} + +fn reference_ineligible(output: &Output) -> Value { + assert_eq!( + output.status.code(), + Some(2), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + let change = &report["changes"][0]; + assert_eq!(change["eligible"], false); + for field in [ + "wave_latency_change_percent", + "achieved_throughput_change_percent", + "decode_rate_change_percent", + "prefill_rate_change_percent", + ] { + assert_eq!(change.get(field), Some(&Value::Null)); + } + for field in [ + "wave_latency_percent", + "achieved_throughput_percent", + "decode_rate_percent", + "prefill_rate_percent", + ] { + assert_eq!(report["drift"][0].get(field), Some(&Value::Null)); + } + assert!( + change["ineligibility_reasons"] + .as_array() + .unwrap() + .iter() + .any(|reason| { reason.as_str().unwrap().contains("reference") }) + ); + report +} + +#[test] +fn comparison_incomplete_reference_withholds_changes_and_drift() { + for warmup in [0, 1] { + let temp = Temp::new(); + let server = Server::new(move |mut stream, index, request| { + if index == 2 * (warmup as usize + 1) { + stream.write_all(b"HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap(); + } else { + normal(stream, index, request); + } + }); + let work = workload(1, warmup, 1); + for name in ["a", "b"] { + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); + } + let failed = run_declared(&temp, &server, "a2", &work, "fixture-model", &deployment()); + assert_eq!(failed.status.code(), Some(2)); + let report = reference_ineligible(&compare_reference(&temp, "a2", true)); + assert_eq!(report["reference_identity"]["status"], "declared_match"); + assert_eq!(report["changes"][0]["observed_output_amounts_match"], true); + assert_eq!( + report["changes"][0]["reference_output_amounts_match"], + false + ); + assert_eq!(report["reference"][0]["eligible_trials"], 0); + assert_eq!( + report["reference"][0]["all_declared_warmups_complete"], + warmup == 0 + ); + assert!(report["baseline"][0]["median_wave_latency_us"].is_number()); + assert!(report["candidate"][0]["median_wave_latency_us"].is_number()); + let human = compare_reference(&temp, "a2", false); + assert_eq!(human.status.code(), Some(2)); + let text = String::from_utf8(human.stdout).unwrap(); + for reason in report["changes"][0]["ineligibility_reasons"] + .as_array() + .unwrap() + { + assert!(text.contains(reason.as_str().unwrap())); + } + } +} + +#[test] +fn comparison_reference_requires_ordered_output_counts_for_changes_and_drift() { + for (baseline, reference) in [ + ([Some(8), Some(8)], [Some(4), Some(4)]), + ([Some(4), Some(8)], [Some(8), Some(4)]), + ([Some(8), Some(8)], [Some(8), None]), + ] { + let temp = Temp::new(); + let server = Server::new(move |mut stream, index, _| { + header(&mut stream, "text/event-stream"); + frame(&mut stream, json!({"choices":[{"delta":{"content":"x"}}]})); + finish( + &mut stream, + if index < 4 { + baseline[index % 2] + } else { + reference[index - 4] + }, + Some(0), + ); + }); + let work = workload(1, 0, 2); + for name in ["a", "b"] { + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); + } + let output = run_declared(&temp, &server, "a2", &work, "fixture-model", &deployment()); + assert_eq!( + output.status.code(), + Some(if reference.contains(&None) { 2 } else { 0 }) + ); + let report = reference_ineligible(&compare_reference(&temp, "a2", true)); + assert_eq!(report["changes"][0]["observed_output_amounts_match"], true); + assert_eq!( + report["changes"][0]["reference_output_amounts_match"], + false + ); + assert_eq!( + report["reference"][0]["lane_completion_tokens"], + json!([[reference[0]], [reference[1]]]) + ); + assert!( + report["changes"][0]["ineligibility_reasons"] + .as_array() + .unwrap() + .iter() + .any(|reason| { + let reason = reason.as_str().unwrap(); + reason.contains("reference") && reason.contains("completion counts") + }) + ); + let output = cli() + .arg("compare") + .arg(temp.path("a")) + .arg(temp.path("a2")) + .arg("--reference") + .arg(temp.path("a")) + .arg("--json") + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + let reversed: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + reversed["changes"][0]["observed_output_amounts_match"], + false + ); + assert_eq!( + reversed["changes"][0]["reference_output_amounts_match"], + true + ); + assert_eq!( + reversed["drift"][0].get("wave_latency_percent"), + Some(&Value::Null) + ); + } +} + +#[test] +fn comparison_reference_distinguishes_mismatched_and_missing_declarations() { + let temp = Temp::new(); + let server = Server::new(normal); + let other_endpoint = Server::new(normal); + let work = workload(1, 0, 1); + for name in ["a", "b"] { + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); + } + let mut mismatch = deployment(); + mismatch["hardware"] = json!("different-hardware"); + mismatch["runtime"] = Value::Null; + let mut missing = deployment(); + missing["settings"] = Value::Null; + for (name, endpoint, model, declaration, status, field) in [ + ( + "model", + &server, + "different-model", + deployment(), + "declared_mismatch", + "model", + ), + ( + "endpoint", + &other_endpoint, + "fixture-model", + deployment(), + "declared_mismatch", + "endpoint", + ), + ( + "hardware", + &server, + "fixture-model", + mismatch, + "declared_mismatch", + "hardware", + ), + ( + "partial", + &server, + "fixture-model", + missing, + "unavailable", + "settings", + ), + ] { + successful(&run_declared( + &temp, + endpoint, + name, + &work, + model, + &declaration, + )); + let report = reference_ineligible(&compare_reference(&temp, name, true)); + assert_eq!(report["reference_identity"]["status"], status); + assert_eq!(report["reference_model"], model); + assert_eq!(report["reference_deployment"], declaration); + assert_eq!(report["changes"][0]["reference_output_amounts_match"], true); + assert!(report["reference"][0]["median_wave_latency_us"].is_number()); + assert!( + report["reference_identity"]["reasons"] + .as_array() + .unwrap() + .iter() + .any(|reason| { reason.as_str().unwrap().contains(field) }) + ); + let human = compare_reference(&temp, name, false); + assert_eq!(human.status.code(), Some(2)); + let text = String::from_utf8(human.stdout).unwrap(); + assert!(text.contains(status)); + for reason in report["reference_identity"]["reasons"].as_array().unwrap() { + assert!(text.contains(reason.as_str().unwrap())); + } + } + successful(&run(&temp, &server, "undeclared", &work)); + let report = reference_ineligible(&compare_reference(&temp, "undeclared", true)); + assert_eq!(report["reference_identity"]["status"], "unavailable"); + assert_eq!(report.get("reference_deployment"), Some(&Value::Null)); + let output = cli() + .arg("compare") + .arg(temp.path("undeclared")) + .arg(temp.path("undeclared")) + .arg("--reference") + .arg(temp.path("undeclared")) + .arg("--json") + .output() + .unwrap(); + let report = reference_ineligible(&output); + assert_eq!(report["reference_identity"]["status"], "unavailable"); +} + +#[test] +fn comparison_missing_reference_metrics_never_fall_back_to_baseline_ranges() { + let temp = Temp::new(); + let server = Server::new(|mut stream, index, request| { + if index < 2 { + normal(stream, index, request); + } else { + header(&mut stream, "text/event-stream"); + frame(&mut stream, json!({"choices":[{"delta":{"content":"x"}}]})); + frame( + &mut stream, + json!({"choices":[{"delta":{},"finish_reason":"length"}],"usage":{"completion_tokens":8}}), + ); + stream.write_all(b"data: [DONE]\n\n").unwrap(); + } + }); + let work = workload(1, 0, 1); + for name in ["a", "b", "a2"] { + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); + } + let output = compare_reference(&temp, "a2", true); + successful(&output); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["changes"][0]["eligible"], true); + assert!(report["drift"][0]["wave_latency_percent"].is_number()); + for (metric, range, change, drift) in [( + "prefill rate", + "prefill_tokens_per_second_range", + "prefill_rate_change_percent", + "prefill_rate_percent", + )] { + assert!(report["baseline"][0][range].is_array()); + assert!(report["candidate"][0][range].is_array()); + assert_eq!(report["reference"][0].get(range), Some(&Value::Null)); + assert_eq!(report["changes"][0].get(change), Some(&Value::Null)); + assert_eq!(report["drift"][0].get(drift), Some(&Value::Null)); + for result in [&report["changes"][0], &report["drift"][0]] { + assert!(result["withheld"].as_array().unwrap().iter().any(|reason| { + let reason = reason.as_str().unwrap(); + reason.contains(metric) && reason.contains("reference") + })); + } + } + let human = compare_reference(&temp, "a2", false); + successful(&human); + let text = String::from_utf8(human.stdout).unwrap(); + for reason in report["changes"][0]["withheld"].as_array().unwrap() { + assert!(text.contains(reason.as_str().unwrap())); + } +} + +#[test] +fn comparison_partial_lane_usage_withholds_reference_metrics_without_dropping_lanes() { + for missing_request in [1, 3, 5] { + let temp = Temp::new(); + let server = Server::new(move |mut stream, index, _| { + header(&mut stream, "text/event-stream"); + frame(&mut stream, json!({"choices":[{"delta":{"content":"x"}}]})); + let mut usage = json!({"completion_tokens":8}); + if index != missing_request { + usage["prompt_tokens"] = json!(4); + } + frame( + &mut stream, + json!({"choices":[{"delta":{},"finish_reason":"length"}],"usage":usage}), + ); + stream.write_all(b"data: [DONE]\n\n").unwrap(); + }); + let work = workload(2, 0, 1); + for name in ["a", "b", "a2"] { + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); + } + let output = compare_reference(&temp, "a2", true); + successful(&output); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + let side = ["baseline", "candidate", "reference"][missing_request / 2]; + assert!(report[side][0]["median_prefill_tokens_per_second"].is_number()); + assert!( + report[side][0]["lane_prefill_tokens_per_second"][0] + .as_array() + .unwrap() + .iter() + .any(Value::is_null) + ); + assert_eq!(report["changes"][0]["eligible"], true); + assert_eq!( + report["changes"][0].get("prefill_rate_change_percent"), + Some(&Value::Null) + ); + assert_eq!( + report["drift"][0].get("prefill_rate_percent"), + Some(&Value::Null) + ); + assert!(report["drift"][0]["wave_latency_percent"].is_number()); + assert!(report["drift"][0]["decode_rate_percent"].is_number()); + assert!( + !report["changes"][0]["withheld"] + .as_array() + .unwrap() + .is_empty() + ); + assert!( + !report["drift"][0]["withheld"] + .as_array() + .unwrap() + .is_empty() + ); + } +} + +#[test] +fn comparison_corrupt_reference_is_an_error_not_an_ordinary_comparison() { + let temp = Temp::new(); + let server = Server::new(normal); + let work = workload(1, 0, 1); + for name in ["a", "b", "a2"] { + successful(&run(&temp, &server, name, &work)); + } + fs::write(temp.path("a2/wave-000000/response-0000.bin"), b"corrupt").unwrap(); + let output = compare_reference(&temp, "a2", true); + assert_eq!(output.status.code(), Some(1)); + assert!(output.stdout.is_empty()); + assert!( + String::from_utf8(output.stderr) + .unwrap() + .contains("reference") + ); +} + +#[test] +fn comparison_legacy_no_reference_load_preserves_evidence_bytes() { + use sha2::{Digest, Sha256}; + + let temp = Temp::new(); + let server = Server::new(normal); + successful(&run(&temp, &server, "legacy", &workload(1, 0, 1))); + // Construct the archival layout from synthetic evidence, not frozen receipts. + let plan_path = temp.path("legacy/plan.json"); + let mut plan = value(&plan_path); + plan["version"] = json!(1); + let plan_bytes = serde_json::to_vec(&plan).unwrap(); + fs::write(&plan_path, &plan_bytes).unwrap(); + let plan_hash = Sha256::digest(&plan_bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + let reservation_path = temp.path("legacy/wave-000000/reservation.json"); + let mut reservation = value(&reservation_path); + reservation["plan_sha256"] = json!(plan_hash); + let reservation_bytes = serde_json::to_vec(&reservation).unwrap(); + fs::write(&reservation_path, &reservation_bytes).unwrap(); + let wave_path = temp.path("legacy/wave-000000/wave.json"); + let mut wave = value(&wave_path); + wave["plan_sha256"] = json!(plan_hash); + wave["reservation_sha256"] = json!( + Sha256::digest(&reservation_bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect::() + ); + fs::write(&wave_path, serde_json::to_vec(&wave).unwrap()).unwrap(); + fs::remove_dir_all(temp.path("legacy/session-000000")).unwrap(); + let paths = [ + "legacy/plan.json", + "legacy/workload.json", + "legacy/run.json", + "legacy/wave-000000/reservation.json", + "legacy/wave-000000/wave.json", + "legacy/wave-000000/response-0000.bin", + ]; + let before: Vec<_> = paths + .iter() + .map(|path| fs::read(temp.path(path)).unwrap()) + .collect(); + let output = cli() + .arg("compare") + .arg(temp.path("legacy")) + .arg(temp.path("legacy")) + .arg("--json") + .output() + .unwrap(); + successful(&output); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(report["version"], 3); + assert_eq!(report["changes"][0]["eligible"], true); + assert_eq!(report["changes"][0]["observed_output_amounts_match"], true); + assert_eq!( + report["changes"][0].get("reference_output_amounts_match"), + Some(&Value::Null) + ); + for field in [ + "reference", + "reference_model", + "reference_deployment", + "reference_identity", + "drift", + ] { + assert_eq!(report.get(field), Some(&Value::Null)); + } + assert_eq!( + report["baseline"][0]["lane_completion_tokens"], + json!([[8]]) + ); + assert_eq!( + report["changes"][0].get("wave_latency_change_percent"), + Some(&Value::Null) + ); + for (path, bytes) in paths.iter().zip(before) { + assert_eq!(fs::read(temp.path(path)).unwrap(), bytes); + } +} + +#[test] +fn comparison_reference_lane_permutation_preserves_totals_but_is_ineligible() { + let temp = Temp::new(); + let server = Server::new(|mut stream, index, request| { + let first_lane = request["seed"].as_i64().unwrap() % 2 == 0; + let tokens = if first_lane == (index < 4) { 4 } else { 8 }; + header(&mut stream, "text/event-stream"); + frame(&mut stream, json!({"choices":[{"delta":{"content":"x"}}]})); + finish(&mut stream, Some(tokens), Some(0)); + }); + let work = workload(2, 0, 1); + for name in ["a", "b", "a2"] { + successful(&run_declared( + &temp, + &server, + name, + &work, + "fixture-model", + &deployment(), + )); + } + let report = reference_ineligible(&compare_reference(&temp, "a2", true)); + assert_eq!( + report["baseline"][0]["reported_completion_tokens"], + report["reference"][0]["reported_completion_tokens"] + ); + assert_eq!( + report["baseline"][0]["lane_completion_tokens"], + json!([[4, 8]]) + ); + assert_eq!( + report["reference"][0]["lane_completion_tokens"], + json!([[8, 4]]) + ); + assert_eq!(report["changes"][0]["observed_output_amounts_match"], true); + assert_eq!( + report["changes"][0]["reference_output_amounts_match"], + false + ); +} diff --git a/docs/performance/CONTRACT.md b/docs/performance/CONTRACT.md index 5f554fb..d952b2b 100644 --- a/docs/performance/CONTRACT.md +++ b/docs/performance/CONTRACT.md @@ -273,18 +273,53 @@ not overlap (`a.max < b.min || b.max < a.min`); touching endpoints overlap. two decimals for latency and tokens/s with one decimal for rates. Withholding for spread does not change `eligible` or `ineligibility_reasons`, which describe run comparability. Absent scalars and ranges are explicit JSON nulls. -Disjoint ranges from three trials is a coarse filter, not a significance test: -about one identical pair in ten separates by chance, and one outlier lane can -withhold a real change. A withheld change is not evidence of equality. -`compare A B --reference A2` checks the reference against the same workload, -collector identity and transport controls, and includes its cell summaries in -`reference`. The baseline range then becomes the union of A's and A2's ranges -before the overlap test, and overlap strings say so; a reference cell without a -range leaves A's range unchanged. `drift` reports raw median changes from A to -A2 for each metric without gating: a noise floor for reading, not a claim. -Absent reference and drift fields are explicit nulls; human output prints absent -drift as `n/a`. The comparison JSON is `version` 2: `*_change_percent` is null -for an eligible cell whenever its ranges overlap, which version 1 never did. +Disjoint observed ranges are a descriptive filter, not a significance test, +bootstrap confidence interval or equivalence test. A withheld change is not +evidence of equality. + +`compare A B --reference A2` verifies the supplied reference against the same +workload, collector identity and transport controls. Corrupt or structurally +incompatible reference evidence is a command error with reference context. +Valid but incomplete or ineligible reference evidence makes the affected cell +ineligible: changes and drift are null, and reference-specific reasons are +reported. This includes incomplete declared warmups and unsettled or continued +execution sessions. All run summaries and ordered raw observations remain +available; no supplied reference silently becomes an ordinary A/B comparison. + +`reference_model`, `reference_deployment` and `reference_identity` disclose +baseline-repeat declarations. Identity status is `declared_match` only when +model, endpoint and every deployment field (`model_revision`, `runtime`, +`hardware`, `settings`) are present and equal. A known differing declaration +is `declared_mismatch`, even if another field is missing; otherwise missing +declarations are `unavailable`. Both nonmatching statuses make reference-aware +cells ineligible, with reasons, without discarding any run's summaries. +Matching declarations do not verify physical server restoration, cache state, +temporal A/B/A ordering or causal effect. JSON and human output state this +limitation; the result is only a descriptive declared-repeat comparison. + +`observed_output_amounts_match` retains its A/B meaning. +`reference_output_amounts_match` reports the A/A2 ordered trial/lane completion +count check and is null without a reference. Complete equal ordered counts +across A, B and A2 are required for both changes and drift; equal totals or +permutations are insufficient. + +For a qualified repeat, the baseline range becomes the union of A's and A2's +ranges before the existing overlap test. Missing lane observations on A, B or A2 +withhold that stream metric and its drift with a reference-specific reason; +surviving lanes cannot silently stand in for a complete paired metric. +A missing reference metric range likewise never permits A-only fallback. +Other metrics remain usable; absent decode/prefill observations do not alone +make a nonstreaming cell ineligible. `drift` reports qualified A-to-A2 median +changes without an overlap filter; its `withheld` array explains null metrics +and failed qualification. Drift is descriptive, not a validated noise bound. + +Comparison JSON advances to `version` 3 for reference qualification and identity +disclosure. Reference metadata, summaries and drift are explicit nulls when +no reference is supplied. Ordinary no-reference numerical results and eligibility +retain the previous range method. Run, plan, reservation and wave receipt +formats and legacy readers are unchanged. Exit status follows cell eligibility: +supplied ineligible reference evidence cannot produce an eligible A/B exit; +metric-specific absence or overlap alone does not change cell eligibility. ## Validation diff --git a/docs/performance/README.md b/docs/performance/README.md index 2b93f2d..d6cfbfb 100644 --- a/docs/performance/README.md +++ b/docs/performance/README.md @@ -207,13 +207,20 @@ verification rather than being repaired. as a speed improvement. - Min/max ranges accompany complete-cell medians. A matched percentage is shown only when the runs' ranges do not overlap; `withheld` explains overlap without - making an otherwise comparable cell ineligible. Three trials make this a coarse - filter, not a significance test: identical setups separate by chance about one - time in ten, and one outlier lane can withhold a real change, so a withheld - change is not evidence of equality. Repeat setup A and pass - `--reference results/setup-a-repeat`: the baseline range becomes the union of - both A runs before the overlap test, and `drift` reports the raw A-to-repeat - median change as a noise floor. + making an otherwise comparable cell ineligible. Observed ranges are descriptive, + not a significance or equivalence test; a withheld change is not evidence of + equality. Repeat setup A and pass `--reference results/setup-a-repeat` only + with complete matching model, endpoint and deployment declarations. A qualified + repeat widens the baseline range with both A runs before the overlap test. + `drift` reports the qualified A-to-repeat median change, not a validated noise + bound. Matching declarations do not verify server restoration, cache state, + run timing order or causal effect. +- A supplied reference never silently falls back to A/B alone. Invalid reference + files are errors; incomplete evidence, unequal ordered output counts or missing + or mismatched declarations make the reference-aware cell ineligible, with + reasons. Summaries remain inspectable. Missing decode/prefill lane observations + on any side withhold those reference-aware metrics and drift, even when other + lanes have measurements. Omit `--reference` explicitly for ordinary A/B. - A compatible comparison requires the same normalized workload, collector binary fingerprint and transport controls. Model/endpoint deployments may differ; this is a descriptive deployment comparison, not causal attribution. @@ -263,6 +270,13 @@ are not inspected or flushed. Missing usage remains unknown, never zero. `model_revision`, `runtime`, `hardware` and `settings` strings. These are operator declarations, not verification that a server loaded those bytes. Keep secrets out of declarations. Use exact public model/runtime references where available. +Reference comparison requires every field on both A runs: omitted deployment or +nullable fields yield identity `unavailable`, not a match. Known differences +yield `declared_mismatch`; complete equal declarations yield `declared_match`. +Comparison JSON exposes these statuses and reasons under `reference_identity` +alongside `reference_model` and `reference_deployment`. Its output version +advances for this reference qualification contract; saved receipt formats do +not change. Runs retain prompts, request bodies and raw provider responses. They are private evidence, **not automatically safe public exports**. There is no upload command.