diff --git a/README.md b/README.md index 5b35923..29c79cc 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,16 @@ target/release/grill-perf compare results/setup-a results/setup-b --json [Full speed benchmark guide](docs/performance/README.md) +For separately qualified DeepSeek and GLM studies, use the +[shared recipe workflow](docs/performance/SHARED-RECIPES.md): externally pinned +source installation, offline `bundle verify`, an explicit precollection policy, +and independent baseline/candidate/repeat acquisitions. `decide` evaluates the +captured observed-envelope policy; successful `compare` output is not PASS. +The shared workflow does not establish live qualification or cross-recipe +equivalence. Publish only a +[manually reviewed report](docs/performance/SHARED-REPORT-TEMPLATE.md), not private +run evidence. + ## Quality evaluation (WIP) Use **`grill`** to collect model answers and check them against task rules. diff --git a/crates/grill-perf/examples/glm-decode-v1.json b/crates/grill-perf/examples/glm-decode-v1.json new file mode 100644 index 0000000..0ff1151 --- /dev/null +++ b/crates/grill-perf/examples/glm-decode-v1.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "name": "glm-decode-v1", + "request": { + "profile": "vllm-fixed-v1", + "stream": true, + "output": { "tokens": 400, "mode": "exact" }, + "cache": "observe", + "temperature_milli": 0, + "top_p_milli": 1000, + "seed": null, + "thinking_control": { "kind": "vllm-enable-thinking-v1", "enabled": false } + }, + "limits": { + "total_ms": 360000, + "idle_ms": 60000, + "response_bytes": 1048576, + "wave_buffer_bytes": 67108864 + }, + "cases": [ + { + "id": "structured", + "messages": [ + { "role": "user", "content": "Count from 1 to 200. Output only the numbers, separated by spaces. No other text." } + ] + }, + { + "id": "prose", + "messages": [ + { "role": "user", "content": "Write a detailed step-by-step explanation of how a hash map works, including collision handling, resizing, and time complexity. Be thorough." } + ] + }, + { + "id": "code", + "messages": [ + { "role": "user", "content": "Output only Python source code. No comments, no docstrings, no markdown fences. Write functions clamp_00 through clamp_49. Each function is exactly:\ndef clamp_NN(x, lo=0, hi=1):\n if x < lo:\n return lo\n if x > hi:\n return hi\n return x\nChange only the function name suffix (00, 01, … 49). One blank line between functions. No other text." } + ] + }, + { + "id": "json", + "messages": [ + { "role": "user", "content": "Emit only a JSON array of fake GPU metrics rows. Each object needs host, gpuIndex, utilPct, tempC, powerW, memUsedMb. Invent many rows. No markdown. Keep expanding the array." } + ] + } + ], + "cells": [ + { "id": "structured-1", "case": "structured", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "structured-2", "case": "structured", "concurrency": 2, "warmup_trials": 1, "trials": 3 }, + { "id": "structured-4", "case": "structured", "concurrency": 4, "warmup_trials": 1, "trials": 3 }, + { "id": "structured-8", "case": "structured", "concurrency": 8, "warmup_trials": 1, "trials": 3 }, + { "id": "prose-1", "case": "prose", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "code-1", "case": "code", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "json-1", "case": "json", "concurrency": 1, "warmup_trials": 1, "trials": 3 } + ] +} diff --git a/crates/grill-perf/examples/glm-prefill-v1.json b/crates/grill-perf/examples/glm-prefill-v1.json new file mode 100644 index 0000000..17171fd --- /dev/null +++ b/crates/grill-perf/examples/glm-prefill-v1.json @@ -0,0 +1,56 @@ +{ + "version": 1, + "name": "glm-prefill-v1", + "request": { + "profile": "vllm-fixed-v1", + "stream": true, + "output": { "tokens": 8, "mode": "exact" }, + "cache": "observe", + "temperature_milli": 0, + "top_p_milli": 1000, + "seed": null, + "thinking_control": { "kind": "vllm-enable-thinking-v1", "enabled": false } + }, + "limits": { + "total_ms": 600000, + "idle_ms": 600000, + "response_bytes": 65536, + "wave_buffer_bytes": 4194304 + }, + "cases": [ + { + "id": "prefill-4k", + "messages": [ + { "role": "user", "content": "[prefill-bench {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 4070 } + }, + { + "id": "prefill-8k", + "messages": [ + { "role": "user", "content": "[prefill-bench {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 8166 } + }, + { + "id": "prefill-16k", + "messages": [ + { "role": "user", "content": "[prefill-bench {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 16358 } + }, + { + "id": "prefill-32k", + "messages": [ + { "role": "user", "content": "[prefill-bench {salt}]\nIgnore the filler below. Reply with the single word OK.\n{fill}\nReply OK." } + ], + "fill": { "unit": " the", "repeat": 32742 } + } + ], + "cells": [ + { "id": "prefill-4k-1", "case": "prefill-4k", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "prefill-8k-1", "case": "prefill-8k", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "prefill-16k-1", "case": "prefill-16k", "concurrency": 1, "warmup_trials": 1, "trials": 3 }, + { "id": "prefill-32k-1", "case": "prefill-32k", "concurrency": 1, "warmup_trials": 1, "trials": 3 } + ] +} diff --git a/crates/grill-perf/examples/recipes-v1.json b/crates/grill-perf/examples/recipes-v1.json new file mode 100644 index 0000000..3da7e9f --- /dev/null +++ b/crates/grill-perf/examples/recipes-v1.json @@ -0,0 +1,41 @@ +{ + "version": 1, + "entries": [ + { + "id": "sparkdash-decode-v1", + "file": "sparkdash-decode-v1.json", + "source_sha256": "d661f6a6ff4718b8999b9796844345e9023b55b971b51d6fea1307e177eecdee", + "workload_sha256": "89dd5320c4f000cc7a15b1d352c721f1e26ddaf9911cccafe7818761d3acffef" + }, + { + "id": "sparkdash-prefill-v1", + "file": "sparkdash-prefill-v1.json", + "source_sha256": "edf255fb30c20e934060ff08af1cfc2c680e6d1b4a2bf9417ce5d96389380468", + "workload_sha256": "549cf323c7117fd31097dc80ffdc267e6b748eab98f13d0a80b69f00b7274f3c" + }, + { + "id": "glm-decode-v1", + "file": "glm-decode-v1.json", + "source_sha256": "508791d3e04a7cb7bc82f06084a5cac718834387123c826c7f378f43f912cb98", + "workload_sha256": "2129bcb5239d58d900a112ca5c2d511136292d54ab9d0c21dce6eaf9e37155a9", + "base": "sparkdash-decode-v1" + }, + { + "id": "glm-prefill-v1", + "file": "glm-prefill-v1.json", + "source_sha256": "ffe1956899f4fc704f047fcaeb85733ebd100beff71676ca59b9bb7235ac16a7", + "workload_sha256": "31312a9a8ad58b7fb6d66bc927d79ce789a5bb6f31737ceb36d8e8d8d8d391f5", + "base": "sparkdash-prefill-v1" + } + ], + "recipes": { + "deepseek": { + "decode": "sparkdash-decode-v1", + "prefill": "sparkdash-prefill-v1" + }, + "glm": { + "decode": "glm-decode-v1", + "prefill": "glm-prefill-v1" + } + } +} diff --git a/crates/grill-perf/src/bundle.rs b/crates/grill-perf/src/bundle.rs new file mode 100644 index 0000000..c5d99ba --- /dev/null +++ b/crates/grill-perf/src/bundle.rs @@ -0,0 +1,254 @@ +use crate::{evidence, model::*}; +use serde::{Deserialize, Serialize}; +use std::path::{Component, Path, PathBuf}; + +const MANIFEST_CAP: usize = 64 * 1024; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +enum EntryId { + SparkdashDecodeV1, + SparkdashPrefillV1, + GlmDecodeV1, + GlmPrefillV1, +} +impl EntryId { + fn name(self) -> &'static str { + match self { + Self::SparkdashDecodeV1 => "sparkdash-decode-v1", + Self::SparkdashPrefillV1 => "sparkdash-prefill-v1", + Self::GlmDecodeV1 => "glm-decode-v1", + Self::GlmPrefillV1 => "glm-prefill-v1", + } + } + fn filename(self) -> &'static str { + match self { + Self::SparkdashDecodeV1 => "sparkdash-decode-v1.json", + Self::SparkdashPrefillV1 => "sparkdash-prefill-v1.json", + Self::GlmDecodeV1 => "glm-decode-v1.json", + Self::GlmPrefillV1 => "glm-prefill-v1.json", + } + } + fn base(self) -> Option { + match self { + Self::SparkdashDecodeV1 | Self::SparkdashPrefillV1 => None, + Self::GlmDecodeV1 => Some(Self::SparkdashDecodeV1), + Self::GlmPrefillV1 => Some(Self::SparkdashPrefillV1), + } + } +} +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Entry { + id: EntryId, + file: String, + source_sha256: String, + workload_sha256: String, + #[serde(skip_serializing_if = "Option::is_none")] + base: Option, +} +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Recipe { + decode: EntryId, + prefill: EntryId, +} +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct Recipes { + deepseek: Recipe, + glm: Recipe, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + version: u32, + entries: [Entry; 4], + recipes: Recipes, +} +#[derive(Serialize)] +struct Budgets { + warmup_requests: u64, + measured_requests: u64, + total_requests: u64, + total_output_token_ceiling: u64, + request_bytes_cap: usize, + limits: Limits, +} +#[derive(Serialize)] +struct VerifiedEntry { + #[serde(flatten)] + identity: Entry, + name: String, + request: RequestSettings, + budgets: Budgets, +} +#[derive(Serialize)] +pub struct Verification { + version: u32, + claim: &'static str, + manifest_sha256: String, + entries: Vec, + recipes: Recipes, +} +fn sha256(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} +fn root(manifest: &Path) -> Result { + let parent = manifest + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + // Check components before resolution so a symlink followed by /.. cannot hide. + let mut checked = PathBuf::new(); + for component in parent.components() { + checked.push(component.as_os_str()); + evidence::directory(&checked)?; + } + evidence::directory(parent)?; + Ok(parent.to_path_buf()) +} +pub fn verify(path: &Path) -> Result { + let root = root(path)?; + let bytes = evidence::read(path, MANIFEST_CAP)?; + let manifest: Manifest = + serde_json::from_slice(&bytes).map_err(|e| format!("invalid recipe manifest: {e}"))?; + if manifest.version != 1 { + return Err("unsupported recipe manifest version".into()); + } + let recipes = &manifest.recipes; + if recipes.deepseek.decode != EntryId::SparkdashDecodeV1 + || recipes.deepseek.prefill != EntryId::SparkdashPrefillV1 + || recipes.glm.decode != EntryId::GlmDecodeV1 + || recipes.glm.prefill != EntryId::GlmPrefillV1 + { + return Err("recipe mappings must select their explicit decode and prefill entries".into()); + } + let mut workloads = Vec::with_capacity(manifest.entries.len()); + for (index, entry) in manifest.entries.iter().enumerate() { + if manifest.entries[..index] + .iter() + .any(|other| other.id == entry.id) + || entry.base != entry.id.base() + { + return Err("duplicate entry or incorrect base relationship".into()); + } + let mut components = Path::new(&entry.file).components(); + if !matches!(components.next(), Some(Component::Normal(_))) + || components.next().is_some() + || entry.file != entry.id.filename() + { + return Err("entry filename must be the declared identity's leaf filename".into()); + } + if !sha256(&entry.source_sha256) || !sha256(&entry.workload_sha256) { + return Err("entry digests must be lowercase SHA256 hex".into()); + } + let source = evidence::read(&root.join(&entry.file), FILE_CAP)?; + if evidence::digest(&source) != entry.source_sha256 { + return Err("workload source digest mismatch".into()); + } + let workload: Workload = + serde_json::from_slice(&source).map_err(|e| format!("invalid workload: {e}"))?; + workload.validate()?; + let normalized = serde_json::to_vec(&workload).map_err(|e| e.to_string())?; + if evidence::digest(&normalized) != entry.workload_sha256 + || workload.name != entry.id.name() + { + return Err("workload normalized digest or name mismatch".into()); + } + workloads.push(workload); + } + for (entry, variant) in manifest.entries.iter().zip(&workloads) { + if let Some(base_id) = entry.base { + let index = manifest + .entries + .iter() + .position(|other| other.id == base_id) + .ok_or("missing base workload")?; + let base = &workloads[index]; + if base.request.thinking != Some(false) || base.request.thinking_control.is_some() { + return Err("base workload must declare legacy thinking false".into()); + } + let mut request = base.request.clone(); + request.thinking = None; + request.thinking_control = + Some(ThinkingControl::VllmEnableThinkingV1 { enabled: false }); + let Workload { + version, + name: _, + request: variant_request, + limits, + cases, + cells, + } = variant; + if *version != base.version + || *variant_request != request + || *limits != base.limits + || *cases != base.cases + || *cells != base.cells + { + return Err("GLM variant differs beyond its name and thinking control".into()); + } + } + } + let entries = manifest + .entries + .into_iter() + .zip(workloads) + .map(|(identity, workload)| { + // Workload admission bounds trials, concurrency and output tokens. + let warmup_requests = workload + .cells + .iter() + .map(|cell| u64::from(cell.warmup_trials) * u64::from(cell.concurrency)) + .sum::(); + let measured_requests = workload + .cells + .iter() + .map(|cell| u64::from(cell.trials) * u64::from(cell.concurrency)) + .sum::(); + let total_requests = warmup_requests + measured_requests; + let total_output_token_ceiling = + total_requests * u64::from(workload.request.output.tokens); + VerifiedEntry { + identity, + name: workload.name, + request: workload.request, + budgets: Budgets { + warmup_requests, + measured_requests, + total_requests, + total_output_token_ceiling, + request_bytes_cap: REQUEST_CAP, + limits: workload.limits, + }, + } + }) + .collect(); + Ok(Verification { + version: 1, + claim: "verified-declared-bundle-not-live-qualification-or-cross-recipe-equivalence", + manifest_sha256: evidence::digest(&bytes), + entries, + recipes: manifest.recipes, + }) +} +pub fn show(verification: &Verification) { + println!("Verified offline bundle {}", verification.manifest_sha256); + for entry in &verification.entries { + println!( + "{}: source {}; workload {}; {} requests; {} output tokens ceiling; thinking {:?}; thinking_control {:?}", + entry.name, + entry.identity.source_sha256, + entry.identity.workload_sha256, + entry.budgets.total_requests, + entry.budgets.total_output_token_ceiling, + entry.request.thinking, + entry.request.thinking_control, + ); + } + println!("Declared controls only; no live qualification or cross-recipe equivalence."); +} diff --git a/crates/grill-perf/src/main.rs b/crates/grill-perf/src/main.rs index 68dafad..8637941 100644 --- a/crates/grill-perf/src/main.rs +++ b/crates/grill-perf/src/main.rs @@ -1,3 +1,4 @@ +mod bundle; mod evidence; mod lifecycle; mod metrics; @@ -50,6 +51,19 @@ enum Command { #[arg(long)] json: bool, }, + /// Verify the declared shared recipe bundle without network calls. + Bundle { + #[command(subcommand)] + command: BundleCommand, + }, +} +#[derive(Subcommand)] +enum BundleCommand { + Verify { + manifest: PathBuf, + #[arg(long)] + json: bool, + }, } fn print_json(value: &impl serde::Serialize) -> model::Result<()> { let encoded = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?; @@ -71,6 +85,17 @@ fn execute(cli: Cli) -> model::Result { match cli.command { Command::Run(options) => show_summary(run::execute(&options)?, options.json) .map(|complete| if complete { 0 } else { 2 }), + Command::Bundle { + command: BundleCommand::Verify { manifest, json }, + } => { + let verification = bundle::verify(&manifest)?; + if json { + print_json(&verification)?; + } else { + bundle::show(&verification); + } + Ok(0) + } Command::Pause { run } => { lifecycle::pause(&run)?; println!( diff --git a/crates/grill-perf/tests/cli.rs b/crates/grill-perf/tests/cli.rs index fc551a9..9352d71 100644 --- a/crates/grill-perf/tests/cli.rs +++ b/crates/grill-perf/tests/cli.rs @@ -3786,6 +3786,8 @@ fn comparison_reference_lane_permutation_preserves_totals_but_is_ineligible() { false ); } +#[path = "support/bundle.rs"] +mod bundle_tests; #[path = "support/metrics.rs"] mod metrics_tests; #[path = "support/policy.rs"] diff --git a/crates/grill-perf/tests/support/bundle.rs b/crates/grill-perf/tests/support/bundle.rs new file mode 100644 index 0000000..63728b7 --- /dev/null +++ b/crates/grill-perf/tests/support/bundle.rs @@ -0,0 +1,314 @@ +use super::*; +use sha2::{Digest, Sha256}; + +fn digest(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} +fn examples() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("examples") +} +fn fixture(temp: &Temp) -> Value { + let manifest = value(examples().join("recipes-v1.json")); + for entry in manifest["entries"].as_array().unwrap() { + let file = entry["file"].as_str().unwrap(); + fs::copy(examples().join(file), temp.path(file)).unwrap(); + } + save_manifest(temp, &manifest); + manifest +} +fn save_manifest(temp: &Temp, manifest: &Value) { + fs::write( + temp.path("recipes-v1.json"), + serde_json::to_vec(manifest).unwrap(), + ) + .unwrap(); +} +fn verify(path: &Path) -> Output { + cli() + .args(["bundle", "verify"]) + .arg(path) + .arg("--json") + .output() + .unwrap() +} +fn rejected(output: Output) { + assert_eq!(output.status.code(), Some(1)); + assert!(output.stdout.is_empty()); +} +// These fixtures retain the checked-in typed field order and literal strings. +// Removing only insignificant whitespace yields their Workload serialization. +fn compact(source: &str) -> Vec { + let mut quoted = false; + let mut escaped = false; + source + .bytes() + .filter(|&byte| { + let keep = quoted || !byte.is_ascii_whitespace(); + if escaped { + escaped = false; + } else if quoted && byte == b'\\' { + escaped = true; + } else if byte == b'"' { + quoted = !quoted; + } + keep + }) + .collect() +} +fn replace_source(temp: &Temp, manifest: &mut Value, index: usize, source: &str) { + let entry = &mut manifest["entries"][index]; + fs::write(temp.path(entry["file"].as_str().unwrap()), source).unwrap(); + entry["source_sha256"] = json!(digest(source.as_bytes())); + entry["workload_sha256"] = json!(digest(&compact(source))); + save_manifest(temp, manifest); +} + +#[test] +fn bundle_checked_in_data_verifies_from_another_cwd_without_network() { + let temp = Temp::new(); + let server = Server::new(normal); + let output = cli() + .current_dir(&temp.0) + .env("HTTP_PROXY", &server.endpoint) + .env("HTTPS_PROXY", &server.endpoint) + .env("ALL_PROXY", &server.endpoint) + .env_remove("MODEL_API_KEY") + .args(["bundle", "verify"]) + .arg(examples().join("recipes-v1.json")) + .arg("--json") + .output() + .unwrap(); + successful(&output); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + let manifest = value(examples().join("recipes-v1.json")); + assert_eq!( + report["manifest_sha256"], + digest(&fs::read(examples().join("recipes-v1.json")).unwrap()) + ); + assert_eq!(report["recipes"], manifest["recipes"]); + for entry in report["entries"].as_array().unwrap() { + let work = value(examples().join(entry["file"].as_str().unwrap())); + let requests: u64 = work["cells"] + .as_array() + .unwrap() + .iter() + .map(|cell| { + (cell["warmup_trials"].as_u64().unwrap() + cell["trials"].as_u64().unwrap()) + * cell["concurrency"].as_u64().unwrap() + }) + .sum(); + assert_eq!(entry["budgets"]["total_requests"], requests); + assert_eq!( + entry["budgets"]["total_output_token_ceiling"], + requests * work["request"]["output"]["tokens"].as_u64().unwrap() + ); + assert_ne!(entry["source_sha256"], entry["workload_sha256"]); + } + assert_eq!(server.count.load(Ordering::SeqCst), 0); +} + +#[test] +fn bundle_rejects_unknown_schema_versions_mappings_and_entry_counts() { + let temp = Temp::new(); + let original = fixture(&temp); + let mut invalid = Vec::new(); + let mut manifest = original.clone(); + manifest["version"] = json!(2); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["source_git"] = json!("not-a-manifest-field"); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["entries"][0]["unknown"] = json!(true); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["recipes"]["glm"]["unknown"] = json!(true); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["recipes"]["other"] = manifest["recipes"]["glm"].clone(); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["recipes"]["glm"]["decode"] = json!("sparkdash-decode-v1"); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["entries"].as_array_mut().unwrap().pop(); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["entries"] + .as_array_mut() + .unwrap() + .push(original["entries"][0].clone()); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["entries"][1] = manifest["entries"][0].clone(); + invalid.push(manifest); + let mut manifest = original.clone(); + manifest["entries"][2]["base"] = json!("sparkdash-prefill-v1"); + invalid.push(manifest); + for manifest in invalid { + save_manifest(&temp, &manifest); + rejected(verify(&temp.path("recipes-v1.json"))); + } + let source = serde_json::to_string(&original).unwrap(); + fs::write( + temp.path("recipes-v1.json"), + source.replacen("{", "{\"version\":1,", 1), + ) + .unwrap(); + rejected(verify(&temp.path("recipes-v1.json"))); +} + +#[test] +fn bundle_rejects_paths_symlinks_and_oversized_files() { + use std::os::unix::fs::symlink; + let temp = Temp::new(); + let original = fixture(&temp); + for file in [ + "../sparkdash-decode-v1.json", + "/sparkdash-decode-v1.json", + "nested/sparkdash-decode-v1.json", + "./sparkdash-decode-v1.json", + "sparkdash-prefill-v1.json", + ] { + let mut manifest = original.clone(); + manifest["entries"][0]["file"] = json!(file); + save_manifest(&temp, &manifest); + rejected(verify(&temp.path("recipes-v1.json"))); + } + save_manifest(&temp, &original); + let links = Temp::new(); + symlink(&temp.0, links.path("root")).unwrap(); + rejected(verify(&links.path("root/recipes-v1.json"))); + rejected(verify(&links.path("root/./recipes-v1.json"))); + symlink(temp.path("recipes-v1.json"), links.path("manifest.json")).unwrap(); + rejected(verify(&links.path("manifest.json"))); + let workload_path = temp.path("sparkdash-decode-v1.json"); + fs::remove_file(&workload_path).unwrap(); + symlink(examples().join("sparkdash-decode-v1.json"), &workload_path).unwrap(); + rejected(verify(&temp.path("recipes-v1.json"))); + fs::remove_file(&workload_path).unwrap(); + fs::write(&workload_path, vec![b' '; 4 * 1024 * 1024 + 1]).unwrap(); + rejected(verify(&temp.path("recipes-v1.json"))); + fs::write(temp.path("recipes-v1.json"), vec![b' '; 64 * 1024 + 1]).unwrap(); + rejected(verify(&temp.path("recipes-v1.json"))); +} + +#[test] +fn bundle_distinguishes_raw_and_normalized_hashes_and_rejects_bad_pins() { + let temp = Temp::new(); + let original = fixture(&temp); + for field in ["source_sha256", "workload_sha256"] { + for pin in ["0".repeat(64), "A".repeat(64), "a".repeat(63)] { + let mut manifest = original.clone(); + manifest["entries"][0][field] = json!(pin); + save_manifest(&temp, &manifest); + rejected(verify(&temp.path("recipes-v1.json"))); + } + } + save_manifest(&temp, &original); + let source = fs::read_to_string(temp.path("sparkdash-decode-v1.json")).unwrap() + "\n"; + fs::write(temp.path("sparkdash-decode-v1.json"), &source).unwrap(); + rejected(verify(&temp.path("recipes-v1.json"))); + let mut manifest = original.clone(); + manifest["entries"][0]["source_sha256"] = json!(digest(source.as_bytes())); + save_manifest(&temp, &manifest); + let output = verify(&temp.path("recipes-v1.json")); + successful(&output); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!( + report["entries"][0]["workload_sha256"], + original["entries"][0]["workload_sha256"] + ); + assert_ne!( + report["entries"][0]["source_sha256"], + original["entries"][0]["source_sha256"] + ); +} + +#[test] +fn bundle_rejects_semantic_variant_drift_even_with_recomputed_hashes() { + let temp = Temp::new(); + let original = fixture(&temp); + for (index, from, to) in [ + (2, "\"temperature_milli\": 0", "\"temperature_milli\": 1"), + (2, "\"enabled\": false", "\"enabled\": true"), + (2, "\"total_ms\": 360000", "\"total_ms\": 360001"), + (2, "\"trials\": 3", "\"trials\": 4"), + (2, "Count from 1 to 200.", "Count from 1 to 201."), + (2, "\"glm-decode-v1\"", "\"other-decode-v1\""), + (3, "\"repeat\": 4070", "\"repeat\": 4071"), + (3, "\"version\": 1", "\"version\": 2"), + ] { + fixture(&temp); + let mut manifest = original.clone(); + let source = + fs::read_to_string(temp.path(manifest["entries"][index]["file"].as_str().unwrap())) + .unwrap(); + replace_source(&temp, &mut manifest, index, &source.replace(from, to)); + rejected(verify(&temp.path("recipes-v1.json"))); + } +} + +#[test] +fn bundle_recipe_paths_collect_both_declared_control_mappings() { + let temp = Temp::new(); + let output = verify(&examples().join("recipes-v1.json")); + successful(&output); + let report: Value = serde_json::from_slice(&output.stdout).unwrap(); + for recipe in ["deepseek", "glm"] { + for kind in ["decode", "prefill"] { + let id = &report["recipes"][recipe][kind]; + let entry = report["entries"] + .as_array() + .unwrap() + .iter() + .find(|entry| entry["id"] == *id) + .unwrap(); + let server = Server::new(|mut stream, _, request| { + header(&mut stream, "text/event-stream"); + frame( + &mut stream, + json!({"id":"fixture","choices":[{"index":0,"delta":{"content":"synthetic answer"}}]}), + ); + finish(&mut stream, request["max_tokens"].as_u64(), Some(0)); + }); + let output = cli() + .arg("run") + .arg(examples().join(entry["file"].as_str().unwrap())) + .args([ + "--endpoint", + &server.endpoint, + "--model", + "fixture-model", + "--local-http", + "--json", + "--out", + ]) + .arg(temp.path(&format!("{recipe}-{kind}"))) + .output() + .unwrap(); + successful(&output); + let requests: Vec<_> = server.seen.try_iter().collect(); + assert_eq!( + requests.len() as u64, + entry["budgets"]["total_requests"].as_u64().unwrap() + ); + for request in requests { + assert_eq!( + request["chat_template_kwargs"], + if recipe == "deepseek" { + json!({"thinking":false}) + } else { + json!({"enable_thinking":false}) + } + ); + assert_eq!(request["max_tokens"], entry["request"]["output"]["tokens"]); + assert_eq!(request["min_tokens"], entry["request"]["output"]["tokens"]); + assert_eq!(request["ignore_eos"], true); + } + } + } +} diff --git a/docs/performance/SHARED-RECIPES.md b/docs/performance/SHARED-RECIPES.md new file mode 100644 index 0000000..008969d --- /dev/null +++ b/docs/performance/SHARED-RECIPES.md @@ -0,0 +1,307 @@ +# Shared recipe workflow + +The checked-in [recipe manifest](../../crates/grill-perf/examples/recipes-v1.json) +selects the frozen sparkDash decode/prefill workloads for DeepSeek and materialized +GLM variants. The GLM variants change only the workload name and the explicit +thinking-key declaration. `bundle verify` checks that relation offline; it does +not resolve, rewrite or collect workloads. Different thinking declarations have +different normalized workload identities. Do not compare across recipes as if +the workloads were equivalent. + +These command blocks are the recipe entrypoints. There is no second collector, +shell orchestration tool, model discovery, forward creation, health check, +deployment change, cache flush, weight download, retry or upload step. Operators +manage serving state separately and obtain approval before model collection. +Unsupported request controls must fail qualification; never strip a field to +make a server accept the workload. + +## Install from an externally pinned source revision + +Use a clean **native Linux ARM64** machine with Git, the workspace-required +Rust/Cargo toolchain, a C/C++ compiler and CMake for rustls/AWS-LC. Check the +workspace `rust-version` and the public build instructions at the chosen revision. +Do not substitute an emulated or cross-compiled installation for the native +installation smoke. Installation and bundle verification require no model service. + +The initial CPU-qualified source pin is +`a6729f9ab5410587bb9da1adb7b34944a9cfc436`. +It identifies the collector and bundle, not a live recipe qualification. +Set `SOURCE` to an absolute, new checkout path. For a later release, obtain +another reviewed full immutable SHA; a branch or moving tag is not a pin. +Stop if a command fails, and verify the checkout SHA before building. + +```sh +SOURCE_GIT_SHA=a6729f9ab5410587bb9da1adb7b34944a9cfc436 +git clone https://github.com/plotarmordev/thegrill.git "${SOURCE:?absolute new checkout path required}" +git -C "$SOURCE" checkout --detach "${SOURCE_GIT_SHA:?reviewed full published commit SHA required}" +test "$(git -C "$SOURCE" rev-parse HEAD)" = "$SOURCE_GIT_SHA" +cargo build --manifest-path "$SOURCE/Cargo.toml" -p grill-perf --release --locked +GRILL_PERF="$SOURCE/target/release/grill-perf" +"$GRILL_PERF" --version +"$GRILL_PERF" --help +"$GRILL_PERF" bundle verify "$SOURCE/crates/grill-perf/examples/recipes-v1.json" --json +sha256sum "$GRILL_PERF" "$SOURCE/Cargo.lock" +rustc -vV +cargo --version +uname -srm +``` + +Check the reported checkout commit against the external pin before building. +Retain the checkout as the authoritative workload location. Select the executable +by this absolute path in every command; do not trust an unrelated `grill-perf` +on `PATH`. Record the actual binary SHA, package version, Cargo.lock SHA, build +command/profile, Rust/Cargo versions, target and native OS/architecture context. +Record deliberate build flags if used; do not collect a whole-environment dump. +The Git source revision is not `source_sha256`: that field hashes the exact +workload file bytes. The normalized `workload_sha256` is the existing typed +Workload serialization digest, not a generic JSON canonicalization. + +At this pin, a clean-source native Linux ARM64 build, offline bundle verification +and the full four-entry CLI workflow were exercised with synthetic loopback +responses. This is CPU installation/protocol evidence, not model qualification. +Repeat the installation smoke on the target host before reporting it as verified. + +## Verify and select the exact workload + +```sh +MANIFEST="$SOURCE/crates/grill-perf/examples/recipes-v1.json" +"$GRILL_PERF" bundle verify "$MANIFEST" --json +``` + +The verifier admits a closed, bounded manifest with exactly four entries and the +explicit `deepseek`/`glm` decode/prefill mappings. Each entry declares its fixed +leaf filename, raw and normalized digests, and the GLM entries declare their +corresponding `base`. The manifest has no self-hash or containing Git revision. +Verification rejects unsupported versions, unknown fields, incorrect mappings, +unsafe paths, symlink files or roots, hash mismatches and semantic variant drift. +Its report contains the manifest digest, entry identities, declared controls and +request/token ceilings. Keep the source directory immutable during verification +and collection; verification is not a filesystem snapshot or signature. + +| Setting | Decode, either recipe | Prefill, either recipe | +|---|---|---| +| Request profile | `vllm-fixed-v1`, streaming | `vllm-fixed-v1`, streaming | +| Output | exact 400 tokens | exact 8 tokens | +| Sampling | temperature zero, top_p one, no seed | temperature zero, top_p one, no seed | +| Cache | `observe` | `observe`; salted filled prompts | +| Requests including warmup | 72: 18 warmup, 54 measured | 16: 4 warmup, 12 measured | +| Output token ceiling | 28,800 | 128 | +| Total / idle deadline, milliseconds | 360000 / 60000 | 600000 / 600000 | +| Response / wave allowance, bytes | 1048576 / 67108864 | 65536 / 4194304 | + +The workloads retain their exact case/cell schedules and declared warmups and +trials. Prompt size names are not tokenizer measurements. Exact output requests +use `min_tokens`, `max_tokens` and `ignore_eos`; successful length stops are not +automatically censored failures. Eligibility and provider usage remain decisive. +`observe` does not establish a cold cache. Review reported generated channels and +reasoning-token evidence: a declared thinking key does not prove the template +honored it. + +DeepSeek sends `chat_template_kwargs: {"thinking": false}`; GLM sends +`chat_template_kwargs: {"enable_thinking": false}`. Neither mapping automatically +qualifies the other. Preserve the frozen sparkDash files and attribution to +[MiaAI-Lab's sparkDash](https://github.com/MiaAI-Lab/sparkDash). + +## Create and approve the policy before collection + +Choose the recipe and one workload first. A decode study and a prefill study are +separate acquisitions with separate source pins and policy scopes. Use the +actual binary and exact selected workload to obtain the pins: + +```sh +sha256sum "$GRILL_PERF" "$WORKLOAD" +``` + +Create `POLICY` as an absolute filename outside the checkout, using a JSON editor. +Copy the first digest into `collector_sha256` and the second into +`workload_source_sha256`. Do not use the Git SHA, Cargo.lock digest, manifest digest +or normalized workload digest for those fields. Choose an identifier, required +metrics and practical regression/reference-spread tolerances with the study +reviewer, independently of the collected outcomes. Approve the complete policy +before the first run; retain exactly the same policy bytes for A, B and A2. +`run --policy FILE` validates and persists those bytes before dispatch. It does +not prove independent preregistration or physical server restoration. + +The following policies are **deliberately invalid illustrations, not runnable +policies**. Every `REPLACE_...` string must be replaced. Hashes must become actual +lowercase SHA256 strings; each tolerance must become an operator-approved JSON +integer, not a string. No tolerance below is a recommendation or implicit +default. These examples choose one metric per cell; the operator may explicitly +add other required metrics to every relevant cell before approval. Do not remove +cells. Decode requires the complete decode scope shown here for either recipe: + +```json +{ + "version": 1, + "method": "observed-envelope-v1", + "id": "REPLACE_APPROVED_POLICY_ID", + "collector_sha256": "REPLACE_ACTUAL_BINARY_SHA256", + "workload_source_sha256": "REPLACE_SELECTED_DECODE_FILE_SHA256", + "min_trials": 3, + "cells": [ + {"cell":"structured-1","metrics":[{"metric":"decode_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"structured-2","metrics":[{"metric":"decode_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"structured-4","metrics":[{"metric":"decode_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"structured-8","metrics":[{"metric":"decode_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"prose-1","metrics":[{"metric":"decode_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"code-1","metrics":[{"metric":"decode_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"json-1","metrics":[{"metric":"decode_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]} + ] +} +``` + +For a prefill study, use this separate complete policy instead: + +```json +{ + "version": 1, + "method": "observed-envelope-v1", + "id": "REPLACE_APPROVED_POLICY_ID", + "collector_sha256": "REPLACE_ACTUAL_BINARY_SHA256", + "workload_source_sha256": "REPLACE_SELECTED_PREFILL_FILE_SHA256", + "min_trials": 3, + "cells": [ + {"cell":"prefill-4k-1","metrics":[{"metric":"prefill_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"prefill-8k-1","metrics":[{"metric":"prefill_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"prefill-16k-1","metrics":[{"metric":"prefill_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]}, + {"cell":"prefill-32k-1","metrics":[{"metric":"prefill_tokens_per_second","max_regression_bps":"REPLACE_APPROVED_INTEGER","max_reference_spread_bps":"REPLACE_APPROVED_INTEGER"}]} + ] +} +``` + +The metric enum also permits `wave_latency_us` and +`achieved_completion_tokens_per_second`; direction is intrinsic to the metric. +Every workload cell must appear exactly once with nonempty unique metrics. +The declared trial minimum must fit the workload, and every cell must declare +warmup. Basis-point schema bounds are engineering limits, not scientific advice. +The reference-spread bound is an observed variability gate, not a confidence bound. +See the [performance contract](CONTRACT.md) for policy admission and decision +arithmetic. No policy generator subcommand is provided. + +Prepare complete private deployment JSON files for A and B, containing nonempty +`model_revision`, `runtime`, `hardware` and `settings` declarations. Describe the +actual context limit, quantization, tensor parallelism, speculative configuration +and other relevant serving settings there; do not paste invented example values. +Reuse A's exact deployment declaration for the restored repeat. Keep the same +model selector and stable endpoint for A/A2; changing a forward port prevents the +declared reference identity from matching. Record B's actual declaration rather +than copying A's declaration when the configuration changed. + +## DeepSeek entrypoint: qualify first + +Set `WORKLOAD` to exactly one of these selections before creating its policy: + +```sh +WORKLOAD="$SOURCE/crates/grill-perf/examples/sparkdash-decode-v1.json" +``` + +or, for a separate prefill study: + +```sh +WORKLOAD="$SOURCE/crates/grill-perf/examples/sparkdash-prefill-v1.json" +``` + +Set `POLICY`, `DEPLOYMENT_A`, `DEPLOYMENT_B`, `A`, `B` and `A2` to absolute paths. +Each output directory must be new with an existing parent. Set `ENDPOINT_A`, +`ENDPOINT_B` and `MODEL` explicitly to approved serving selections. The following +blocks assume HTTPS; literal-loopback HTTP additionally needs `--local-http` on +each run. Add `--auth-env MODEL_API_KEY` only when an independently supplied +credential is required. Do not put credentials into the policy or workload. + +Collect A with the approved policy: + +```sh +"$GRILL_PERF" run "$WORKLOAD" --policy "$POLICY" --endpoint "$ENDPOINT_A" \ + --model "$MODEL" --deployment "$DEPLOYMENT_A" --out "$A" --json +``` + +After A has completed, obtain approval and change the serving setup separately. +Ensure collection does not overlap, then collect B: + +```sh +"$GRILL_PERF" run "$WORKLOAD" --policy "$POLICY" --endpoint "$ENDPOINT_B" \ + --model "$MODEL" --deployment "$DEPLOYMENT_B" --out "$B" --json +``` + +After B has completed, restore A separately and verify the restoration through +the operator's approved procedure. Collect a new, independent repeat; copying a +run directory is not a repeat: + +```sh +"$GRILL_PERF" run "$WORKLOAD" --policy "$POLICY" --endpoint "$ENDPOINT_A" \ + --model "$MODEL" --deployment "$DEPLOYMENT_A" --out "$A2" --json +"$GRILL_PERF" compare "$A" "$B" --reference "$A2" --json +"$GRILL_PERF" decide "$A" "$B" --reference "$A2" --json +``` + +## GLM entrypoint: qualify independently later + +Do not transfer DeepSeek's qualification to GLM. Obtain separate authorization +and review the actual GLM template/control support first. Select exactly one +GLM workload and create its own policy with its actual source digest: + +```sh +WORKLOAD="$SOURCE/crates/grill-perf/examples/glm-decode-v1.json" +``` + +or, for a separate prefill study: + +```sh +WORKLOAD="$SOURCE/crates/grill-perf/examples/glm-prefill-v1.json" +``` + +Use fresh GLM-specific paths, approved model/endpoint selections and complete +actual deployment declarations, under the same prerequisites as DeepSeek. +Collect the baseline: + +```sh +"$GRILL_PERF" run "$WORKLOAD" --policy "$POLICY" --endpoint "$ENDPOINT_A" \ + --model "$MODEL" --deployment "$DEPLOYMENT_A" --out "$A" --json +``` + +After completion and a separately authorized configuration change, collect B: + +```sh +"$GRILL_PERF" run "$WORKLOAD" --policy "$POLICY" --endpoint "$ENDPOINT_B" \ + --model "$MODEL" --deployment "$DEPLOYMENT_B" --out "$B" --json +``` + +After completion and separately verified restoration of A, collect A2: + +```sh +"$GRILL_PERF" run "$WORKLOAD" --policy "$POLICY" --endpoint "$ENDPOINT_A" \ + --model "$MODEL" --deployment "$DEPLOYMENT_A" --out "$A2" --json +"$GRILL_PERF" compare "$A" "$B" --reference "$A2" --json +"$GRILL_PERF" decide "$A" "$B" --reference "$A2" --json +``` + +## Read decisions without broadening the claim + +`compare` remains descriptive: successful comparison is eligibility, never PASS. +Only a successfully printed versioned `decide` envelope constitutes a decision. +Do not infer a verdict from an exit code or missing output; parsing/output errors +can share exit values with decision outcomes. Inspect `decision`, `eligibility`, +all scoped gates, coverage and bounded reason codes. The policy evaluates the +observed envelope, not statistical significance, equivalence, intelligence, +causality or universal no-regression. + +A, B and A2 must all capture the same exact policy before dispatch. Missing +bindings are INCONCLUSIVE; conflicting present bindings or corrupt evidence are +ERROR. Insufficient observations, unqualified references, copied acquisitions, +out-of-order declared starts or missing metrics cannot silently become PASS. +Declared start order and matching reference declarations do not authenticate +nonoverlap or physical restoration. Paused/resumed sessions do not qualify as +uninterrupted performance acquisitions. Preserve every cell and metric; do not +select only favorable gates or treat a withheld comparison percentage as zero. + +Maintain separate source-reviewed, loopback-tested and live-qualified statuses +for each recipe and workload. Both explicit control mappings and all four +workload entries were exercised through the CLI against synthetic responses. +Live qualification remains pending: DeepSeek first under coordination, GLM +separately later. Earlier model smoke receipts do not qualify this new +bundle/policy workflow. + +Use the [manual reviewed report template](SHARED-REPORT-TEMPLATE.md) only after +privacy review. It is not an exporter or replayable evidence package. No upstream +recipe files are changed by these commands; maintainers may separately adopt +these public command blocks without copying private deployment code or evidence. diff --git a/docs/performance/SHARED-REPORT-TEMPLATE.md b/docs/performance/SHARED-REPORT-TEMPLATE.md new file mode 100644 index 0000000..d4f3675 --- /dev/null +++ b/docs/performance/SHARED-REPORT-TEMPLATE.md @@ -0,0 +1,85 @@ +# Manually reviewed shared recipe report + +Copy this template into a separately reviewed public report. Replace bracketed +fields only with explicitly approved public values. Omit a field or mark it +`withheld` when publication is not approved; never paste private text to explain +why it was withheld. This summary is not raw replay evidence, an authenticated +execution record or an automatic safe-to-publish export. The CLI has no exporter +or uploader. + +## Publication gate + +- Public pins, change description and scope approved: [yes / no] +- Privacy review completed: [yes / no] +- Entire report restricted to the approved fields below: [yes / no] + +Do not publish while any gate is `no`. Do not attach private deployment JSON, +local paths, endpoints, model/provider selectors, credentials, raw prompts, +responses, raw timing/usage metrics, metrics labels, stderr or loader diagnostic +text. Do not paste complete run/compare output. Review policy identifiers and +all other operator-supplied identifiers before publishing them. Hashes also need +publication approval; a digest is not an authorization to disclose its source. + +## Public provenance + +| Field | Approved public value | +|---|---| +| Externally pinned full source Git SHA | [SHA or withheld] | +| Actual collector binary SHA256 | [digest or withheld] | +| Evaluator binary SHA256 | [digest or withheld] | +| Package version | [version or withheld] | +| Cargo.lock SHA256 | [digest or withheld] | +| Build profile, command and deliberate flags | [reviewed build context or withheld] | +| Rust/Cargo versions, target, OS/architecture | [reviewed context or withheld] | +| Clean native Linux ARM64 install smoke | [verified / not run / failed] | +| Manifest SHA256 | [digest or withheld] | +| Recipe | [deepseek / glm] | +| Workload | [decode / prefill] | +| Exact workload source SHA256 | [digest or withheld] | +| Normalized workload SHA256 | [digest or withheld] | +| Captured policy SHA256 and approved identifier | [pins or withheld] | +| Role-labelled A, B, A2 evidence fingerprints | [digests or withheld] | +| Source review status for this recipe/workload | [reviewed / pending] | +| Loopback fixture status for this recipe/workload | [passed / not run / failed] | +| Live qualification status for this recipe/workload | [qualified within stated scope / pending / failed] | + +Source Git revision, workload source bytes and normalized workload identity are +separate pins. A binary rebuilt from the same source need not have the same +binary digest. DeepSeek qualification does not qualify GLM, or vice versa. + +## Approved change and evaluated scope + +- Public change identifier: [approved public issue/commit identifier or withheld] +- Public change description: [independently reviewed public description or withheld] +- Declared policy approval before collection: [yes / no / unverified] +- Same captured policy in all roles: [yes / no / unknown] +- Distinct ordered acquisitions: [qualified / unqualified / unknown] +- Declared A/A2 reference identity: [declared_match / unqualified / unknown] +- Operator-reviewed nonoverlap and restoration: [reviewed / unverified] + +Repeat a row for **every** policy cell/metric, including missing or unfavorable +gates. Use only public bundled cell IDs, metric enum values, numeric policy +thresholds and coverage counts from reviewed decision output. Do not include raw +sample values, private labels or copied diagnostic prose. + +| Cell | Metric | Regression tolerance (bps) | Reference spread limit (bps) | Expected / observed coverage by A, B, A2 | Gate decision | Bounded reason codes | +|---|---|---|---|---|---|---| +| [public cell] | [metric enum] | [approved integer] | [approved integer] | [reviewed coverage counts] | [PASS / REGRESSION / INCONCLUSIVE / ERROR] | [codes only] | + +## Decision and limitations + +- Successfully parsed versioned decision envelope: [yes / no] +- Overall decision: [PASS / REGRESSION / INCONCLUSIVE / ERROR / no decision] +- Eligibility: [qualified / unqualified / unknown] +- Aggregate bounded reason codes: [codes only] +- Complete scope retained, including missing observations: [yes / no] + +The decision concerns the declared observed-envelope policy and this workload, +binary, scope and acquisition set only. It is not a significance test, confidence +interval, causal attribution, equivalence test, universal no-regression result, +intelligence score or maximum-capacity claim. `compare` success does not mean +PASS. A missing decision envelope is not a decision. Matching declarations do +not independently prove model identity, nonoverlap or physical restoration. +Thinking controls are declarations, and `observe` does not prove a cold cache. +This reviewed summary omits private replay evidence; readers cannot reconstruct +the underlying observations from it alone.