diff --git a/Cargo.lock b/Cargo.lock index 93fd6b417..3c289ff08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -282,10 +282,13 @@ dependencies = [ "env_logger", "executor", "lambda-vm-prover", + "log", "rkyv", + "serde_cbor", "stark", "tempfile", "tikv-jemalloc-ctl", + "tikv-jemalloc-sys", "tikv-jemallocator", ] diff --git a/Makefile b/Makefile index cf794e081..529e1b7da 100644 --- a/Makefile +++ b/Makefile @@ -73,8 +73,15 @@ RECURSION_VERIFIER_PRESETS := min blowup2 blowup4 blowup8 # `continuation` feature: verify a multi-epoch ContinuationProof bundle instead # of a monolithic VmProof. Only the presets the benchmarks actually measure. RECURSION_CONT_PRESETS := min blowup2 blowup4 +# `batched` feature: verify a BatchedProof (one FRI per domain height) instead +# of a monolithic VmProof. +RECURSION_BATCHED_PRESETS := min blowup2 +# `owned` feature: the monolithic proof, deserialized instead of read in place. +RECURSION_OWNED_PRESETS := blowup2 RECURSION_VERIFIER_ARTIFACTS := $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-, $(addsuffix .elf, $(RECURSION_VERIFIER_PRESETS))) \ - $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-cont-, $(addsuffix .elf, $(RECURSION_CONT_PRESETS))) + $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-cont-, $(addsuffix .elf, $(RECURSION_CONT_PRESETS))) \ + $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-batched-, $(addsuffix .elf, $(RECURSION_BATCHED_PRESETS))) \ + $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-owned-, $(addsuffix .elf, $(RECURSION_OWNED_PRESETS))) # Override with: make ... SYSROOT_DIR=$HOME/.lambda-vm-sysroot # to install the sysroot in a user-writable location and avoid sudo. @@ -248,6 +255,22 @@ $(RECURSION_ARTIFACTS_DIR)/recursion-cont-$(1).elf: FORCE | prepare-sysroot $(RE endef $(foreach preset,$(RECURSION_CONT_PRESETS),$(eval $(call recursion_cont_verifier_rule,$(preset)))) +# Batched variants: same crate, `batched` feature on top of the preset feature +# -> recursion-batched--bench -> recursion-batched-.elf. +define recursion_batched_verifier_rule +$(RECURSION_ARTIFACTS_DIR)/recursion-batched-$(1).elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR) + $$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-batched-$(1)-bench,--features "batched $(1)") +endef +$(foreach preset,$(RECURSION_BATCHED_PRESETS),$(eval $(call recursion_batched_verifier_rule,$(preset)))) + +# Owned variants: same crate and the same blob as the default guest, read via +# one rkyv deserialize instead of in place. +define recursion_owned_verifier_rule +$(RECURSION_ARTIFACTS_DIR)/recursion-owned-$(1).elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR) + $$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-owned-$(1)-bench,--features "owned $(1)") +endef +$(foreach preset,$(RECURSION_OWNED_PRESETS),$(eval $(call recursion_owned_verifier_rule,$(preset)))) + clean-asm: -rm -rf $(ASM_ARTIFACTS_DIR) diff --git a/bench_vs/lambda/recursion/Cargo.toml b/bench_vs/lambda/recursion/Cargo.toml index cc4d00a70..cf14deaef 100644 --- a/bench_vs/lambda/recursion/Cargo.toml +++ b/bench_vs/lambda/recursion/Cargo.toml @@ -21,6 +21,15 @@ blowup8 = [] # memory-bounded inner prove) instead of a monolithic VmProof. Selects the # `recursion-cont--bench` bins below. continuation = [] +# Orthogonal to the presets: verify a BatchedProof (one FRI per domain height +# instead of one per table) instead of a monolithic VmProof. Selects the +# `recursion-batched--bench` bins below. Mutually exclusive with +# `continuation` — main.rs guards it. +batched = [] +# Orthogonal to the presets: verify the same monolithic VmProof as the default +# guest, but deserialized into owned values instead of read in place. Exists to +# price zero-copy against an owned copy in guest cycles. +owned = [] # One distinctly named binary per preset (selected by its feature) so a parallel # `make -j` builds them to different filenames — structurally race-free, no cp @@ -60,6 +69,21 @@ name = "recursion-cont-blowup4-bench" path = "src/main.rs" required-features = ["continuation", "blowup4"] +[[bin]] +name = "recursion-batched-min-bench" +path = "src/main.rs" +required-features = ["batched", "min"] + +[[bin]] +name = "recursion-batched-blowup2-bench" +path = "src/main.rs" +required-features = ["batched", "blowup2"] + +[[bin]] +name = "recursion-owned-blowup2-bench" +path = "src/main.rs" +required-features = ["owned", "blowup2"] + [dependencies] lambda-vm-prover = { path = "../../../prover", default-features = false, features = [ "profile-markers", diff --git a/bench_vs/lambda/recursion/src/main.rs b/bench_vs/lambda/recursion/src/main.rs index 1a846109b..5cae557c1 100644 --- a/bench_vs/lambda/recursion/src/main.rs +++ b/bench_vs/lambda/recursion/src/main.rs @@ -53,6 +53,12 @@ compile_error!("select exactly one of the `min`/`blowup2`/`blowup4`/`blowup8` fe all(feature = "blowup4", feature = "blowup8"), ))] compile_error!("select exactly one of the `min`/`blowup2`/`blowup4`/`blowup8` features"); +#[cfg(any( + all(feature = "continuation", feature = "batched"), + all(feature = "continuation", feature = "owned"), + all(feature = "batched", feature = "owned"), +))] +compile_error!("`continuation`, `batched` and `owned` are mutually exclusive proof layouts"); /// The build preset fixing the inner `ProofOptions` (see the module docs). #[cfg(feature = "min")] @@ -88,7 +94,7 @@ pub fn main() -> ! { // not self-enforcing here. let options = PRESET.options(); - #[cfg(not(feature = "continuation"))] + #[cfg(not(any(feature = "continuation", feature = "batched", feature = "owned")))] let attestation = lambda_vm_prover::recursion::verify_and_attest_blob(blob, &options) .expect("verify errored") .expect("inner proof failed verification"); @@ -98,6 +104,16 @@ pub fn main() -> ! { .expect("verify errored") .expect("inner continuation proof failed verification"); + #[cfg(feature = "batched")] + let attestation = lambda_vm_prover::recursion::verify_batched_and_attest(blob, &options) + .expect("verify errored") + .expect("inner batched proof failed verification"); + + #[cfg(feature = "owned")] + let attestation = lambda_vm_prover::recursion::verify_owned_and_attest(blob, &options) + .expect("verify errored") + .expect("inner proof failed verification"); + lambda_vm_syscalls::syscalls::commit(&attestation); lambda_vm_syscalls::syscalls::sys_halt(); } diff --git a/bin/cli/Cargo.toml b/bin/cli/Cargo.toml index b9140e34c..5047b2edf 100644 --- a/bin/cli/Cargo.toml +++ b/bin/cli/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" license.workspace = true [dependencies] +serde_cbor = "0.11" executor = { path = "../../executor" } prover = { path = "../../prover", package = "lambda-vm-prover" } stark = { path = "../../crypto/stark" } @@ -13,11 +14,19 @@ clap = { version = "4.3.10", features = ["derive"] } rkyv = { version = "0.8.10", default-features = false, features = ["alloc", "bytecheck", "aligned", "pointer_width_64"] } tempfile = "3" tikv-jemallocator = "0.6" -tikv-jemalloc-ctl = { version = "0.6", features = ["stats"], optional = true } +# No `stats` by default: that feature propagates to tikv-jemalloc-sys and builds +# jemalloc with `--enable-stats`, i.e. counters on the malloc fast path, for every +# binary. `keep_large_buffers_warm` needs only `raw`/`mallctl`; the heap tracker is +# what needs the counters, and it is behind `jemalloc-stats`. +tikv-jemalloc-ctl = { version = "0.6" } +tikv-jemalloc-sys = "0.6" env_logger = "0.11" +# Used by `keep_large_buffers_warm`, whose body is Linux-only — so a macOS build +# will not catch its absence. +log = "0.4" [features] -jemalloc-stats = ["dep:tikv-jemalloc-ctl"] +jemalloc-stats = ["tikv-jemalloc-ctl/stats"] disk-spill = ["prover/disk-spill"] instruments = ["prover/instruments", "stark/instruments"] # GPU profiling build (Nsight): CUDA prover + instruments spans + NVTX ranges. diff --git a/bin/cli/examples/cmp_proofs.rs b/bin/cli/examples/cmp_proofs.rs new file mode 100644 index 000000000..41cb5f4f1 --- /dev/null +++ b/bin/cli/examples/cmp_proofs.rs @@ -0,0 +1,102 @@ +//! Compare two `VmProof` files table by table: `cmp_proofs A.proof B.proof`. +//! +//! A diagnostic for the streaming prover: which table, and which part of it, +//! first departs from the monolithic prover's proof of the same execution. + +use std::os::unix::fs::FileExt; + +use prover::VmProof; + +fn read(path: &str) -> VmProof { + let file = std::fs::File::open(path).expect("open"); + let len = file.metadata().expect("metadata").len() as usize; + let mut buf = rkyv::util::AlignedVec::<16>::with_capacity(len); + buf.resize(len, 0); + file.read_exact_at(&mut buf, 0).expect("read"); + rkyv::from_bytes::(&buf).expect("deserialize") +} + +fn main() { + let args: Vec = std::env::args().collect(); + let (a, b) = (read(&args[1]), read(&args[2])); + println!( + "tables: {} vs {}", + a.proof.proofs.len(), + b.proof.proofs.len() + ); + println!( + "table_counts equal: {}", + format!("{:?}", a.table_counts) == format!("{:?}", b.table_counts) + ); + println!("counts A: {:?}", a.table_counts); + println!("counts B: {:?}", b.table_counts); + println!( + "runtime_page_ranges equal: {}", + format!("{:?}", a.runtime_page_ranges) == format!("{:?}", b.runtime_page_ranges) + ); + println!( + "num_private_input_pages: {} vs {}", + a.num_private_input_pages, b.num_private_input_pages + ); + println!( + "public_output equal: {}", + a.public_output == b.public_output + ); + let mut shown = 0; + for (i, (x, y)) in a.proof.proofs.iter().zip(b.proof.proofs.iter()).enumerate() { + let mut diffs = Vec::new(); + if x.trace_length != y.trace_length { + diffs.push(format!( + "trace_length {} vs {}", + x.trace_length, y.trace_length + )); + } + if x.lde_trace_main_merkle_root != y.lde_trace_main_merkle_root { + diffs.push("main root".into()); + } + if x.lde_trace_precomputed_merkle_root != y.lde_trace_precomputed_merkle_root { + diffs.push("precomputed root".into()); + } + if x.lde_trace_aux_merkle_root != y.lde_trace_aux_merkle_root { + diffs.push("aux root".into()); + } + if format!("{:?}", x.trace_ood_evaluations) != format!("{:?}", y.trace_ood_evaluations) { + diffs.push("ood".into()); + } + if x.composition_poly_root != y.composition_poly_root { + diffs.push("composition root".into()); + } + if x.fri_layers_merkle_roots != y.fri_layers_merkle_roots { + diffs.push("fri roots".into()); + } + if x.fri_final_poly_coeffs != y.fri_final_poly_coeffs { + diffs.push("fri final".into()); + } + if x.query_list.len() != y.query_list.len() { + diffs.push(format!( + "queries {} vs {}", + x.query_list.len(), + y.query_list.len() + )); + } + if !diffs.is_empty() && shown < 40 { + println!("table {i}: {}", diffs.join(", ")); + shown += 1; + } + } + println!("(showing at most 40 differing tables)"); + let main_diff: Vec = a + .proof + .proofs + .iter() + .zip(b.proof.proofs.iter()) + .enumerate() + .filter(|(_, (x, y))| x.lde_trace_main_merkle_root != y.lde_trace_main_merkle_root) + .map(|(i, _)| i) + .collect(); + println!( + "tables whose MAIN root differs ({}): {:?}", + main_diff.len(), + main_diff + ); +} diff --git a/bin/cli/src/main.rs b/bin/cli/src/main.rs index a04e920db..d24acccda 100644 --- a/bin/cli/src/main.rs +++ b/bin/cli/src/main.rs @@ -10,6 +10,79 @@ use clap::{Parser, Subcommand, ValueHint}; #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +/// jemalloc serves allocations of 8 MiB and up from one shared arena and purges each of +/// them the moment it is freed, whatever the decay says, unless decay is disabled for that +/// arena (`extent_may_force_decay`). A prover that allocates and drops one trace-sized +/// buffer after another then refaults and re-zeroes the same pages for every chunk. +/// Disable the arena's decay and purge it on our own clock instead: hot buffers are +/// reused across threads, cold ones still go back to the OS. +/// +/// The index is `opt.narenas`: jemalloc 5 reserves the slot immediately after the +/// automatic arenas for the oversize arena (`arena_init_huge`, `huge_arena_ind = +/// narenas_total_get()`), and `opt.oversize_threshold` defaults to the same 8 MiB. +/// +/// Linux only: elsewhere jemalloc is built without background threads and the decay +/// mallctl traps. +/// +/// Called only from the paths that allocate trace-sized buffers, not from `main`: +/// it disables an arena's decay for the life of the process and leaves a purge +/// thread behind, which is not something `cli verify` or `--help` should pay, and +/// it would otherwise change the allocator under every baseline measured with +/// this binary. +fn keep_large_buffers_warm() { + #[cfg(target_os = "linux")] + { + use std::ffi::CString; + use std::ptr::null_mut; + use std::time::Duration; + use tikv_jemalloc_ctl::raw; + + const PURGE_EVERY: Duration = Duration::from_secs(10); + + // The arena only exists after the first large allocation. + std::hint::black_box(vec![0u8; 16 << 20]); + // SAFETY: `opt.narenas` is `unsigned`, the decay knob is `ssize_t`, and `purge` + // takes no value. + unsafe { + let huge_arena = match raw::read::(b"opt.narenas\0") { + Ok(n) => n, + Err(e) => { + // Not fatal, but the run is then indistinguishable from one + // where the knob worked — which is exactly what makes a + // memory measurement unreadable. Say so. + log::warn!( + "keep_large_buffers_warm: cannot read opt.narenas ({e}); \ + the oversize arena keeps jemalloc's default decay" + ); + return; + } + }; + let decay = format!("arena.{huge_arena}.dirty_decay_ms\0"); + if let Err(e) = raw::write(decay.as_bytes(), -1i64) { + log::warn!( + "keep_large_buffers_warm: cannot disable decay on arena \ + {huge_arena} ({e}); the oversize arena keeps jemalloc's default" + ); + return; + } + log::debug!("keep_large_buffers_warm: decay disabled on arena {huge_arena}"); + let purge = CString::new(format!("arena.{huge_arena}.purge")).unwrap(); + std::thread::spawn(move || { + loop { + std::thread::sleep(PURGE_EVERY); + tikv_jemalloc_sys::mallctl( + purge.as_ptr(), + null_mut(), + null_mut(), + null_mut(), + 0, + ); + } + }); + } + } +} use executor::vm::instruction::decoding::Instruction; use executor::vm::instruction::execution::{Accelerator, SyscallNumbers}; use executor::{elf::Elf, flamegraph::FlamegraphGenerator, vm::execution::Executor}; @@ -37,6 +110,14 @@ fn read_aligned_file(path: &Path) -> std::io::Result> /// Polls jemalloc `stats.allocated` every 10ms from a background thread, /// tracking the high-water mark. Near-zero overhead because jemalloc uses /// thread-local caches — `epoch::advance()` just merges cached counters. +/// +/// `stats.allocated` is live bytes, not resident pages, so the mark is real +/// simultaneous residency rather than an allocator watermark that freed memory +/// keeps propping up. What it does not say on its own is *when* the mark was +/// set, which is what makes a peak actionable — a peak inside one table's work +/// and a peak spread across every table call for opposite fixes. The tracker +/// therefore also records how far into the run the mark was set, to be read +/// against the phase timeline. #[cfg(feature = "jemalloc-stats")] mod heap_tracker { use std::sync::Arc; @@ -49,6 +130,8 @@ mod heap_tracker { pub struct HeapTracker { stop: Arc, peak: Arc, + /// Milliseconds from `start()` to the sample that set `peak`. + peak_at_ms: Arc, handle: Option>, } @@ -56,35 +139,47 @@ mod heap_tracker { pub fn start() -> Self { let stop = Arc::new(AtomicBool::new(false)); let peak = Arc::new(AtomicUsize::new(0)); + let peak_at_ms = Arc::new(AtomicUsize::new(0)); let stop_clone = stop.clone(); let peak_clone = peak.clone(); + let peak_at_clone = peak_at_ms.clone(); + let started = std::time::Instant::now(); let handle = thread::spawn(move || { - while !stop_clone.load(Ordering::Relaxed) { - // Refresh jemalloc's cached stats + // Records the elapsed time of the sample that raised the mark, + // so a peak can be placed against the phase timeline instead of + // being a number with no location. + let sample = |peak: &AtomicUsize, at: &AtomicUsize| { epoch::advance().ok(); - if let Ok(allocated) = stats::allocated::read() { - peak_clone.fetch_max(allocated, Ordering::Relaxed); + if let Ok(allocated) = stats::allocated::read() + && allocated > peak.fetch_max(allocated, Ordering::Relaxed) + { + at.store(started.elapsed().as_millis() as usize, Ordering::Relaxed); } + }; + while !stop_clone.load(Ordering::Relaxed) { + sample(&peak_clone, &peak_at_clone); thread::sleep(Duration::from_millis(10)); } // One final sample after stop signal - epoch::advance().ok(); - if let Ok(allocated) = stats::allocated::read() { - peak_clone.fetch_max(allocated, Ordering::Relaxed); - } + sample(&peak_clone, &peak_at_clone); }); Self { stop, peak, + peak_at_ms, handle: Some(handle), } } - pub fn stop(mut self) -> usize { + /// `(peak bytes, milliseconds into the run when it was set)`. + pub fn stop(mut self) -> (usize, usize) { self.shutdown(); - self.peak.load(Ordering::Relaxed) + ( + self.peak.load(Ordering::Relaxed), + self.peak_at_ms.load(Ordering::Relaxed), + ) } fn shutdown(&mut self) { @@ -228,6 +323,61 @@ enum Commands { #[arg(long, value_hint = ValueHint::FilePath)] private_input: Option, }, + + /// Build the traces without proving, to compare what each production path + /// holds. Peak heap is per process, so each path is measured on its own run. + TraceBuild { + /// Path to the ELF file + #[arg(value_parser, value_hint = ValueHint::FilePath)] + elf: PathBuf, + + /// Path to the private input file + #[arg(long, value_hint = ValueHint::FilePath)] + private_input: Option, + + /// Prove-and-retire (the spec's Approach 1): walk the execution, + /// committing and retiring each table as it fills, instead of building + /// every trace first. + #[arg(long)] + prove_and_retire: bool, + + /// How far down Approach 1's pipeline to run. Only meaningful with + /// --prove-and-retire; each stage includes the ones before it. + #[arg( + long, + value_enum, + default_value = "logup", + requires = "prove_and_retire" + )] + through: Stage, + + /// Assemble the per-table proof the LogUp stage leaves and run the + /// ordinary verifier on it, after the timings are reported. + #[arg(long, requires = "prove_and_retire")] + verify: bool, + + /// Write the assembled per-table proof here (implies the assembly, not + /// the verification). + #[arg(short, long, requires = "prove_and_retire", value_hint = ValueHint::FilePath)] + output: Option, + }, +} + +/// Approach 1's passes, in order. +#[derive(Copy, Clone, PartialEq, Eq, clap::ValueEnum)] +enum Stage { + /// Walk and commit every chunk's main trace. + Commit, + /// Also commit the tables that stay, and sample the shared challenge. + Challenge, + /// Also walk again to build and commit the auxiliary columns. + Logup, + /// Instead of one FRI per table, fold them by domain, open at the group's + /// indices, and assemble the batched proof. + Batched, + /// Only the walk: replay the execution and rebuild every table, proving + /// nothing. The floor each pass pays. + Walk, } fn main() -> ExitCode { @@ -293,6 +443,21 @@ fn main() -> ExitCode { } } Commands::CountElements { elf, private_input } => cmd_count_elements(elf, private_input), + Commands::TraceBuild { + elf, + private_input, + prove_and_retire, + through, + verify, + output, + } => cmd_trace_build( + elf, + private_input, + prove_and_retire, + through, + verify, + output, + ), } } @@ -669,7 +834,12 @@ fn cmd_prove( #[cfg(feature = "jemalloc-stats")] { let peak_bytes = tracker.stop(); - println!("Peak heap: {} MB", peak_bytes / (1024 * 1024)); + let (peak_bytes, peak_at_ms) = peak_bytes; + println!( + "Peak heap: {} MB (at {:.1}s)", + peak_bytes / (1024 * 1024), + peak_at_ms as f64 / 1000.0 + ); } ExitCode::SUCCESS } @@ -843,7 +1013,12 @@ fn cmd_prove_continuation( #[cfg(feature = "jemalloc-stats")] { let peak_bytes = tracker.stop(); - println!("Peak heap: {} MB", peak_bytes / (1024 * 1024)); + let (peak_bytes, peak_at_ms) = peak_bytes; + println!( + "Peak heap: {} MB (at {:.1}s)", + peak_bytes / (1024 * 1024), + peak_at_ms as f64 / 1000.0 + ); } ExitCode::SUCCESS } @@ -965,6 +1140,370 @@ fn parse_epoch_size_log2(value: &str) -> Result { Ok(epoch_size_log2) } +/// Approach 1's pipeline, as far as `through`. +/// +/// Each stage is measured in its own process because peak heap is per process, +/// and reported as the number of tables it accounted for — chunks for the +/// Commit phase, every table in AIR order once the later passes have run. +fn run_approach_1( + elf: &Elf, + elf_bytes: &[u8], + private_inputs: &[u8], + max_rows: &prover::tables::MaxRowsConfig, + options: &stark::proof::options::ProofOptions, + through: Stage, + verify: bool, +) -> Result<(usize, Option), String> { + #[cfg(feature = "instruments")] + stark::instruments::reset_timeline(); + let t0 = std::time::Instant::now(); + if through == Stage::Walk { + let resident = prover::logup_phase::walk_only(elf, private_inputs, max_rows) + .map_err(|e| format!("{e:?}"))?; + println!(" walk only {:>8.2}s", t0.elapsed().as_secs_f64()); + return Ok((resident.pages.len(), None)); + } + let committed = prover::commit_phase::run_to_end(elf, private_inputs, max_rows, options) + .map_err(|e| format!("{e:?}"))?; + let t_commit = t0.elapsed(); + if through == Stage::Commit { + println!(" pass 1 (commit) {:>8.2}s", t_commit.as_secs_f64()); + return Ok((committed.chunks.len(), None)); + } + let t1 = std::time::Instant::now(); + let challenge = prover::challenge_phase::run(&committed, elf, elf_bytes, options) + .map_err(|e| format!("{e:?}"))?; + // The mains are committed; nothing downstream reads their traces again. + drop(committed); + let t_challenge = t1.elapsed(); + if through == Stage::Challenge { + println!(" pass 1 (commit) {:>8.2}s", t_commit.as_secs_f64()); + println!(" pass 2 (challenge) {:>8.2}s", t_challenge.as_secs_f64()); + return Ok((challenge.roots.len(), None)); + } + // The batched path replaces the per-table prove; running both would measure + // neither. + if through == Stage::Batched { + let t3 = std::time::Instant::now(); + let batched = + prover::logup_phase::run_batched(elf, private_inputs, max_rows, options, &challenge) + .map_err(|e| format!("{e:?}"))?; + let t_fold = t3.elapsed(); + let t4 = std::time::Instant::now(); + let opened = prover::logup_phase::run_open( + elf, + private_inputs, + max_rows, + options, + &challenge, + &batched, + ) + .map_err(|e| format!("{e:?}"))?; + let tables = batched.tables.len(); + let groups = batched.groups.len(); + let t_open = t4.elapsed(); + let proof = prover::logup_phase::assemble_batched_proof(batched, opened) + .map_err(|e| format!("{e:?}"))?; + println!(" pass 1 (commit) {:>8.2}s", t_commit.as_secs_f64()); + println!(" pass 2 (challenge) {:>8.2}s", t_challenge.as_secs_f64()); + println!(" pass 3-4 (deep+fold) {:>7.2}s", t_fold.as_secs_f64()); + println!(" pass 5 (open) {:>8.2}s", t_open.as_secs_f64()); + report_span_totals(); + report_batched_size(&proof, tables, groups); + if verify { + let started = std::time::Instant::now(); + match prover::batched_verifier::verify(&proof, elf_bytes, options) { + Ok(true) => println!( + "Batched proof verifies: {tables} tables in {groups} groups, {:.3}s", + started.elapsed().as_secs_f64() + ), + Ok(false) => return Err("batched proof REJECTED by the verifier".into()), + Err(e) => return Err(format!("batched proof verification error: {e}")), + } + } + return Ok((tables, None)); + } + + let t2 = std::time::Instant::now(); + let logup = prover::logup_phase::run(elf, private_inputs, max_rows, options, &challenge) + .map_err(|e| format!("{e:?}"))?; + let t_prove = t2.elapsed(); + println!(" pass 1 (commit) {:>8.2}s", t_commit.as_secs_f64()); + println!(" pass 2 (challenge) {:>8.2}s", t_challenge.as_secs_f64()); + println!(" pass 3 (prove) {:>8.2}s", t_prove.as_secs_f64()); + report_span_totals(); + report_fri_shape(&logup.tables); + let tables = logup.tables.len(); + let proof = verify.then(|| prover::logup_phase::assemble_vm_proof(logup, &challenge)); + Ok((tables, proof)) +} + +/// What the batched proof weighs, against what the per-table one weighs. +/// +/// The prize was priced before any of this was built: the per-table FRI data +/// was 57.9% of the proof. This is the same measurement on the other side — +/// what a table still carries once the layers, the final polynomial, the +/// queries and the nonce belong to its group. +fn report_batched_size(proof: &prover::logup_phase::BatchedProof, tables: usize, groups: usize) { + let mut per_table = 0usize; + for (t, o) in proof.tables.iter().zip(proof.openings.iter()) { + per_table += serde_cbor::to_vec(&t.trace_ood) + .map(|v| v.len()) + .unwrap_or(0) + + serde_cbor::to_vec(&t.trace_ood_next) + .map(|v| v.len()) + .unwrap_or(0) + + serde_cbor::to_vec(&t.parts_ood) + .map(|v| v.len()) + .unwrap_or(0) + + serde_cbor::to_vec(o).map(|v| v.len()).unwrap_or(0) + + 32 * 4; + } + let mut per_group = 0usize; + for (_, fri) in proof.groups.iter() { + per_group += serde_cbor::to_vec(&fri.layer_roots) + .map(|v| v.len()) + .unwrap_or(0) + + serde_cbor::to_vec(&fri.final_poly_coeffs) + .map(|v| v.len()) + .unwrap_or(0) + + serde_cbor::to_vec(&fri.query_list) + .map(|v| v.len()) + .unwrap_or(0); + } + let total = per_table + per_group; + println!( + "Batched: {tables} tables over {groups} groups; {} MB per table + {} MB per group = {} MB", + per_table / (1024 * 1024), + per_group / (1024 * 1024), + total / (1024 * 1024), + ); +} + +/// Where the time went, summed per span label. +/// +/// The prover's own spans are per table and there are hundreds of them, so the raw +/// timeline is unreadable; what answers "where is the time" is the total per +/// label. Sums exceed wall time, because tables run concurrently — the ratios +/// between labels are the point, not the absolute figures. +fn report_span_totals() { + #[cfg(feature = "instruments")] + { + use std::collections::BTreeMap; + let spans = stark::instruments::take_timeline(); + let mut by_label: BTreeMap<&str, (std::time::Duration, usize)> = BTreeMap::new(); + for s in &spans { + let e = by_label.entry(s.label).or_default(); + e.0 += s.wall; + e.1 += 1; + } + let mut rows: Vec<_> = by_label.into_iter().collect(); + rows.sort_by_key(|(_, (d, _))| std::cmp::Reverse(*d)); + println!(" --- summed over tables (concurrent, so > wall) ---"); + for (label, (d, n)) in rows.into_iter().take(12) { + println!(" {label:<28} {:>8.2}s x{n}", d.as_secs_f64()); + } + } +} + +/// What one FRI per table costs, and what batching by height would collapse. +/// +/// Step 0 of the batched-FRI analysis: the prize is the per-table FRI data, and +/// batching can only merge tables that share a domain exactly — Lambda's fold +/// squares the coset offset each layer, so a short table over `offset·` does +/// not line up with a tall fold over `offset²·`. So the number worth knowing +/// is how many tables collapse into how many distinct heights, against the +/// bytes that would be saved. +fn report_fri_shape( + proofs: &[stark::proof::stark::StarkProof< + prover::tables::types::GoldilocksField, + prover::tables::types::GoldilocksExtension, + (), + >], +) { + use std::collections::BTreeMap; + + let mut by_height: BTreeMap = BTreeMap::new(); + let (mut fri_bytes, mut total_bytes) = (0usize, 0usize); + for p in proofs { + *by_height.entry(p.trace_length).or_default() += 1; + fri_bytes += serde_cbor::to_vec(&p.fri_layers_merkle_roots) + .map(|v| v.len()) + .unwrap_or(0) + + serde_cbor::to_vec(&p.fri_final_poly_coeffs) + .map(|v| v.len()) + .unwrap_or(0) + + serde_cbor::to_vec(&p.query_list) + .map(|v| v.len()) + .unwrap_or(0); + total_bytes += serde_cbor::to_vec(p).map(|v| v.len()).unwrap_or(0); + } + println!( + "FRI: {} tables over {} distinct heights; per-table FRI data {} MB of {} MB ({:.1}%)", + proofs.len(), + by_height.len(), + fri_bytes / (1024 * 1024), + total_bytes / (1024 * 1024), + 100.0 * fri_bytes as f64 / total_bytes.max(1) as f64, + ); + for (rows, tables) in by_height.iter().rev() { + println!(" {rows:>9} rows x{tables}"); + } +} + +/// Build the traces one way or the other, so the two production paths can be +/// compared on what they hold. Nothing is proved: this measures the side of the +/// prover that Approach 1's Commit phase replaces. +fn cmd_trace_build( + elf_path: PathBuf, + private_input_path: Option, + prove_and_retire: bool, + through: Stage, + verify: bool, + output: Option, +) -> ExitCode { + // Only the per-table proof is serializable today, so `--output` with any + // other stage would walk the whole execution and then write nothing. Say so + // before spending the walk rather than exiting 0 in silence. + if output.is_some() && prove_and_retire && through != Stage::Logup { + eprintln!( + "--output writes the per-table proof, which only `--through logup` assembles; \ + `--through {}` has no serializable proof yet (see docs/prove_and_retire_design.md \u{00A7}10).", + match through { + Stage::Walk => "walk", + Stage::Commit => "commit", + Stage::Challenge => "challenge", + Stage::Batched => "batched", + Stage::Logup => unreachable!(), + } + ); + return ExitCode::FAILURE; + } + let elf_data = match std::fs::read(&elf_path) { + Ok(data) => data, + Err(e) => { + eprintln!("Failed to read ELF file: {e}"); + return ExitCode::FAILURE; + } + }; + let private_inputs = match read_private_input(private_input_path.as_ref()) { + Ok(inputs) => inputs, + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + }; + let elf = match executor::elf::Elf::load(&elf_data) { + Ok(elf) => elf, + Err(e) => { + eprintln!("Failed to load ELF: {e}"); + return ExitCode::FAILURE; + } + }; + + #[cfg(feature = "jemalloc-stats")] + let tracker = heap_tracker::HeapTracker::start(); + let started = std::time::Instant::now(); + + let max_rows = prover::tables::MaxRowsConfig::default(); + let options = match stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) { + Ok(o) => o, + Err(e) => { + eprintln!("bad proof options: {e:?}"); + return ExitCode::FAILURE; + } + }; + let outcome = if prove_and_retire { + // The walk allocates and drops one trace-sized buffer after another, + // which is the pattern this works around. + keep_large_buffers_warm(); + run_approach_1( + &elf, + &elf_data, + &private_inputs, + &max_rows, + &options, + through, + verify || output.is_some(), + ) + } else { + prover::commit_phase::build_resident(&elf, &private_inputs, &max_rows) + .map(|t| (t.cpus.len(), None)) + .map_err(|e| format!("{e:?}")) + }; + + let elapsed = started.elapsed(); + let proof = match outcome { + Ok((n, proof)) => { + println!( + "Trace build ({}): {n} tables, {:.3}s", + if prove_and_retire { + "prove-and-retire" + } else { + "resident" + }, + elapsed.as_secs_f64() + ); + proof + } + Err(e) => { + eprintln!("trace build failed: {e}"); + return ExitCode::FAILURE; + } + }; + + #[cfg(feature = "jemalloc-stats")] + { + let (peak_bytes, peak_at_ms) = tracker.stop(); + println!( + "Peak heap: {} MB (at {:.1}s)", + peak_bytes / (1024 * 1024), + peak_at_ms as f64 / 1000.0 + ); + } + + let Some(proof) = proof else { + return ExitCode::SUCCESS; + }; + if let Some(path) = output { + let bytes = match rkyv::to_bytes::(&proof) { + Ok(b) => b, + Err(e) => { + eprintln!("Failed to serialize the A1 proof: {e}"); + return ExitCode::FAILURE; + } + }; + if let Err(e) = std::fs::write(&path, &bytes) { + eprintln!("Failed to write {}: {e}", path.display()); + return ExitCode::FAILURE; + } + println!( + "A1 proof written: {} ({} bytes)", + path.display(), + bytes.len() + ); + } + if verify { + let started = std::time::Instant::now(); + match prover::verify_with_options(&proof, &elf_data, &options, None, None) { + Ok(true) => println!( + "A1 proof verifies: {} tables, {:.3}s", + proof.proof.proofs.len(), + started.elapsed().as_secs_f64() + ), + Ok(false) => { + eprintln!("A1 proof REJECTED by the verifier"); + return ExitCode::FAILURE; + } + Err(e) => { + eprintln!("A1 proof verification error: {e}"); + return ExitCode::FAILURE; + } + } + } + ExitCode::SUCCESS +} + #[cfg(test)] mod tests { use super::*; diff --git a/crypto/crypto/src/merkle_tree/merkle.rs b/crypto/crypto/src/merkle_tree/merkle.rs index 447654907..01e489b0a 100644 --- a/crypto/crypto/src/merkle_tree/merkle.rs +++ b/crypto/crypto/src/merkle_tree/merkle.rs @@ -282,6 +282,73 @@ where self.create_proof(merkle_path) } + /// Free the leaf half of the node buffer, keeping the inner nodes + /// (`nodes[0..leaves_len - 1]`, root at index 0). Roughly halves the tree's + /// footprint. + /// + /// Every node an opening needs is retained except one: the leaf-level + /// sibling, which the caller regenerates and hands to + /// [`get_proof_by_pos_with_leaf_sibling`](Self::get_proof_by_pos_with_leaf_sibling). + /// + /// `leaves_len` is checked against the buffer rather than trusted, so this + /// is a no-op — returning `false` — on a tree that was already dropped, on a + /// single-leaf or root-only tree, on disk-spill mmap backing, and on a wrong + /// `leaves_len`. Truncating twice would silently eat inner nodes. + pub fn drop_leaves(&mut self, leaves_len: usize) -> bool { + if leaves_len <= 1 || self.is_root_only() { + return false; + } + #[cfg(feature = "disk-spill")] + if self.mmap_backing.is_some() { + return false; + } + // A full tree, and only a full tree, has exactly `2 * leaves_len - 1` + // nodes. Anything else means this is not the shape we were told. + if self.nodes.len() != 2 * leaves_len - 1 { + return false; + } + self.nodes.truncate(leaves_len - 1); + self.nodes.shrink_to_fit(); + true + } + + /// Leaf index whose hash must be regenerated to open position `pos`. + pub fn sibling_leaf_position(pos: usize) -> usize { + pos ^ 1 + } + + /// Opening for `pos` on a tree whose leaves were dropped, with the + /// leaf-level sibling supplied by the caller (see + /// [`sibling_leaf_position`](Self::sibling_leaf_position)). + /// + /// Byte-identical to what [`get_proof_by_pos`](Self::get_proof_by_pos) would + /// return on the full tree: same bottom node, and every node above it read + /// from the retained inner nodes at the same indices. + pub fn get_proof_by_pos_with_leaf_sibling( + &self, + pos: usize, + leaves_len: usize, + sibling_leaf: B::Node, + ) -> Option> { + if leaves_len <= 1 || pos >= leaves_len || self.is_root_only() { + return None; + } + let mut merkle_path = Vec::with_capacity(leaves_len.trailing_zeros() as usize); + merkle_path.push(sibling_leaf); + + let mut node = parent_index(pos + leaves_len - 1); + while node != ROOT { + // `node_get`, not `self.nodes` directly: every other read in this + // file goes through it for the disk-spill mmap indirection. The two + // are mutually exclusive today — `drop_leaves` refuses an mmap-backed + // tree — but a direct read would silently yield `None` here if that + // ever stopped holding, and the prover's opening path unwraps this. + merkle_path.push(self.node_get(sibling_index(node))?.clone()); + node = parent_index(node); + } + self.create_proof(merkle_path) + } + /// Creates a proof from a Merkle pasth fn create_proof(&self, merkle_path: Vec) -> Option> { Some(Proof { merkle_path }) diff --git a/crypto/crypto/src/tests/merkle_tests.rs b/crypto/crypto/src/tests/merkle_tests.rs index a4be838b1..8b43a26f7 100644 --- a/crypto/crypto/src/tests/merkle_tests.rs +++ b/crypto/crypto/src/tests/merkle_tests.rs @@ -172,3 +172,79 @@ mod disk_spill_serde_tests { assert_eq!(restored.root, unspilled.root); } } + +/// A leaf-dropped opening must be byte-identical to the full-tree one. +/// +/// Dropping the leaves changes only *where* the bottom node of the path comes +/// from — regenerated by the caller instead of read out of the buffer — never +/// what it is. If these ever diverge, the streaming prover emits openings the +/// verifier rejects. +#[test] +fn leaf_dropped_opening_matches_the_full_tree() { + const MODULUS: u64 = 13; + type U64PF = U64Field; + type FE = FieldElement; + + let leaves_len = 8; + let values: Vec = (1..=leaves_len as u64).map(FE::new).collect(); + let full = MerkleTree::>::build(&values).unwrap(); + + // The leaf hashes, as the tree stores them: what the prover regenerates. + let leaf_hashes: Vec = full.nodes()[leaves_len - 1..].to_vec(); + + let mut dropped = MerkleTree::>::build(&values).unwrap(); + assert!( + dropped.drop_leaves(leaves_len), + "a full power-of-two tree must drop its leaves" + ); + assert_eq!(dropped.root, full.root, "dropping leaves moved the root"); + assert_eq!( + dropped.nodes().len(), + leaves_len - 1, + "only the inner nodes should remain" + ); + + for pos in 0..leaves_len { + let expected = full.get_proof_by_pos(pos).expect("full-tree opening"); + let sibling = MerkleTree::>::sibling_leaf_position(pos); + let actual = dropped + .get_proof_by_pos_with_leaf_sibling(pos, leaves_len, leaf_hashes[sibling]) + .expect("leaf-dropped opening"); + assert_eq!( + expected.merkle_path, actual.merkle_path, + "opening for leaf {pos} differs from the full-tree one" + ); + } +} + +/// Dropping twice, or with the wrong shape, must not eat inner nodes. +#[test] +fn drop_leaves_refuses_anything_but_a_full_tree() { + const MODULUS: u64 = 13; + type U64PF = U64Field; + type FE = FieldElement; + + let leaves_len = 8; + let values: Vec = (1..=leaves_len as u64).map(FE::new).collect(); + let mut tree = MerkleTree::>::build(&values).unwrap(); + + assert!(tree.drop_leaves(leaves_len)); + let after_first = tree.nodes().len(); + + assert!( + !tree.drop_leaves(leaves_len), + "a second drop must be refused, not applied" + ); + assert_eq!(tree.nodes().len(), after_first, "a refused drop truncated"); + + let mut other = MerkleTree::>::build(&values).unwrap(); + assert!( + !other.drop_leaves(leaves_len * 2), + "a wrong leaves_len must be refused" + ); + assert_eq!( + other.nodes().len(), + 2 * leaves_len - 1, + "a refused drop truncated" + ); +} diff --git a/crypto/stark/src/batched_verifier.rs b/crypto/stark/src/batched_verifier.rs new file mode 100644 index 000000000..363c8fceb --- /dev/null +++ b/crypto/stark/src/batched_verifier.rs @@ -0,0 +1,373 @@ +//! Verification of a batched proof: every table's rounds after round 1, and one +//! FRI per height group over the fold of their DEEP codewords. +//! +//! The per-table steps are the ordinary verifier's, run on a view of each +//! table's data with the FRI left empty; the fold coefficients and the group +//! FRI challenges are replayed from the shared seed the prover used, and the +//! group's first FRI layer is the coefficient-weighted sum of the tables' DEEP +//! evaluations at the group's query indices. + +use crate::config::Commitment; +use crate::domain::{VerifierDomain, new_verifier_domain}; +use crate::proof::stark::StarkProof; +use crate::proof::view::StarkProofView; +use crate::prover::GroupFri; +use crate::table::Table; +use crate::traits::AIR; +use crate::verifier::{Challenges, IsStarkVerifier, RoundsChallenges, Verifier}; +use crypto::fiat_shamir::is_transcript::IsStarkTranscript; +use log::error; +use math::field::element::FieldElement; +use math::field::traits::{IsFFTField, IsField, IsSubFieldOf}; +use math::traits::AsBytes; + +/// One height group as the verifier sees it. +pub struct BatchedGroup<'a, E: IsField> { + /// Rows of every member's trace: they share the domain or they could not + /// have been folded together. + pub trace_rows: usize, + pub fri: &'a GroupFri, +} + +struct GroupReplay { + domain: VerifierDomain, + layout: crate::fri::terminal::FriFoldLayout, + zetas: Vec>, + iotas: Vec, + acc: Vec>, + acc_sym: Vec>, + /// The first table of the group, whose AIR supplied the domain. Kept from + /// the scan that already found it rather than looked up again: the second + /// scan is what forced an `expect` into verifier code. + member: usize, +} + +/// Verify the rounds after round 1 of every table and each group's FRI. +/// +/// `transcript` is the shared transcript right after the LogUp challenges: the +/// state every table's fork and the fold seed start from. `tables[i]` carries +/// table `i`'s round-1 roots, round-3 data and openings; its FRI fields are +/// ignored. `fold_order` is the order the prover drew the coefficients in and +/// `groups` the order it ran the FRIs in. +#[allow(clippy::too_many_arguments)] +pub fn verify_batched( + airs: &[&dyn AIR], + public_inputs: &[PI], + tables: &[StarkProof], + group_of: &[usize], + groups: &[BatchedGroup<'_, FieldExtension>], + fold_order: &[usize], + transcript: &(impl IsStarkTranscript + Clone), + rap_challenges: &[FieldElement], +) -> bool +where + Field: IsSubFieldOf + IsFFTField + Send + Sync, + FieldExtension: IsField + Send + Sync, + Field::BaseType: math::field::element::NativeArchived, + FieldExtension::BaseType: math::field::element::NativeArchived, + PI: rkyv::Archive + Clone, + ::Archived: rkyv::Deserialize, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, +{ + type V = Verifier; + let n = airs.len(); + if tables.len() != n || public_inputs.len() != n || group_of.len() != n || fold_order.len() != n + { + error!("batched: {} AIRs against {} tables", n, tables.len()); + return false; + } + if groups.is_empty() || group_of.iter().any(|&g| g >= groups.len()) { + error!("batched: a table names a group the proof does not have"); + return false; + } + let mut seen = vec![false; n]; + for &idx in fold_order { + if idx >= n || std::mem::replace(&mut seen[idx], true) { + error!("batched: fold order is not a permutation of the tables"); + return false; + } + } + let Some(first_air) = airs.first() else { + return false; + }; + let num_queries = first_air.options().fri_number_of_queries; + let grinding_factor = first_air.context().proof_options.grinding_factor; + + // Every table's domain is its group's, and every block and opening has the + // shape its AIR declares. + // + // Both run before the fold seed below reads a single out-of-domain value. + // That seed indexes the blocks at the dimensions the *proof* advertises, so + // a proof whose advertised dimensions disagree with its data length has to + // be rejected here rather than panic there — the rule `ood_blocks_well_formed` + // documents, and the one the ordinary verifier keeps by running it in round 1. + for (idx, ((air, table), &g)) in airs.iter().zip(tables).zip(group_of).enumerate() { + if table.trace_length == 0 || table.trace_length != groups[g].trace_rows { + error!("batched: table {idx} does not live on its group's domain"); + return false; + } + let view = StarkProofView::Owned(table); + // `trace_length` is non-zero by the check above, so the division is safe. + if table.composition_poly_parts_ood_evaluation.len() + != air.composition_poly_degree_bound(table.trace_length) / table.trace_length + || !V::::ood_blocks_well_formed(*air, view) + || !V::::trace_opening_widths_well_formed( + *air, + view, + num_queries, + ) + { + error!("batched: table {idx}'s blocks or openings are malformed"); + return false; + } + } + + // The fold coefficients, from the seed, in the prover's order. + let mut seed = transcript.clone(); + let mut coefficients = vec![FieldElement::::zero(); n]; + for &idx in fold_order { + let t = &tables[idx]; + if let Some(ref bpi) = t.bus_public_inputs { + seed.append_field_element(&bpi.table_contribution); + } + seed.append_bytes(&t.composition_poly_root); + for ood in [&t.trace_ood_evaluations, &t.trace_ood_next_evaluations] { + for col_idx in 0..ood.width { + for row_idx in 0..ood.height { + seed.append_field_element(&ood.get_row(row_idx)[col_idx]); + } + } + } + for elem in t.composition_poly_parts_ood_evaluation.iter() { + seed.append_field_element(elem); + } + coefficients[idx] = seed.sample_field_element(); + } + + // Each group's FRI challenges, from the same seed, in the prover's order. + let mut replays: Vec> = Vec::with_capacity(groups.len()); + for (g, group) in groups.iter().enumerate() { + let Some(member) = group_of.iter().position(|&h| h == g) else { + error!("batched: group {g} has no member"); + return false; + }; + let domain = new_verifier_domain(airs[member], group.trace_rows); + let layout = V::::fri_termination_params(airs[member], &domain); + let fri = group.fri; + if fri.layer_roots.len() != layout.num_committed + || fri.final_poly_coeffs.len() != (1usize << layout.effective_k) + || fri.query_list.len() != num_queries + || fri.iotas.len() != num_queries + || fri.query_list.iter().any(|q| { + q.layers_auth_paths.len() != layout.num_committed + || q.layers_evaluations_sym.len() != layout.num_committed + }) + { + error!("batched: group {g}'s FRI has the wrong shape"); + return false; + } + let mut zetas: Vec> = fri + .layer_roots + .iter() + .map(|root| { + let zeta = seed.sample_field_element(); + seed.append_bytes(root); + zeta + }) + .collect(); + if layout.total_folds > 0 { + zetas.push(seed.sample_field_element()); + } + for c in fri.final_poly_coeffs.iter() { + seed.append_field_element(c); + } + if grinding_factor > 0 { + let grinding_seed = seed.state(); + let Some(nonce) = fri.nonce else { + error!("batched: group {g} has no grinding nonce"); + return false; + }; + if !crate::grinding::is_valid_nonce(&grinding_seed, nonce, grinding_factor) { + error!("batched: group {g}'s grinding nonce is not valid"); + return false; + } + seed.append_bytes(&nonce.to_be_bytes()); + } + let iotas = + V::::sample_query_indexes(num_queries, &domain, &mut seed); + if iotas != fri.iotas { + error!("batched: group {g}'s query indices are not the transcript's"); + return false; + } + replays.push(GroupReplay { + domain, + layout, + zetas, + iotas, + acc: vec![FieldElement::zero(); num_queries], + acc_sym: vec![FieldElement::zero(); num_queries], + member, + }); + } + + // Every table: rounds 2 and 3 against its fork, its openings at the group's + // indices, and its DEEP evaluations there, folded into the group's. + for (idx, ((air, table), &g)) in airs.iter().zip(tables).zip(group_of).enumerate() { + let view = StarkProofView::Owned(table); + let mut fork = transcript.clone(); + if n > 1 { + fork.append_bytes(&(idx as u64).to_le_bytes()); + } + if let Some(ref root) = table.lde_trace_aux_merkle_root { + fork.append_bytes(root); + } + if let Some(ref bpi) = table.bus_public_inputs { + fork.append_field_element(&bpi.table_contribution); + } + // Shapes were pinned to the AIR in the pre-pass above, before the fold + // seed read any of this table's blocks. + let domain = new_verifier_domain(*air, table.trace_length); + let layout = V::::ood_layout(*air); + let RoundsChallenges { + z, + boundary_coeffs, + transition_coeffs, + trace_term_coeffs, + gammas, + } = V::::replay_rounds_2_to_4( + *air, + view, + &public_inputs[idx], + &domain, + &mut fork, + rap_challenges, + &layout, + ); + let replay = &replays[g]; + let challenges = Challenges { + z, + boundary_coeffs, + transition_coeffs, + trace_term_coeffs, + gammas, + zetas: replay.zetas.clone(), + iotas: replay.iotas.clone(), + rap_challenges: rap_challenges.to_vec(), + grinding_seed: [0u8; 32], + }; + let ood_current = view.trace_ood_evaluations(); + let ood_next = view.trace_ood_next_evaluations(); + let ood_full = layout.reconstruct_full( + ood_current.row_major_data(), + ood_current.width(), + ood_next.row_major_data(), + ); + if !V::::step_2_verify_claimed_composition_polynomial( + *air, + view, + &public_inputs[idx], + &domain, + &challenges, + &ood_full, + layout.step_size(), + ) { + error!("batched: table {idx} fails the out-of-domain consistency check"); + return false; + } + if !V::::step_4_verify_trace_and_composition_openings( + view, + &challenges, + ) { + error!("batched: table {idx}'s openings do not authenticate"); + return false; + } + let Some((evals, evals_sym)) = + V::::reconstruct_deep_composition_poly_evaluations_for_all_queries( + &challenges, + &domain, + view, + &ood_full, + layout.next_row_cols(), + layout.step_size(), + ) + else { + error!("batched: table {idx}'s DEEP evaluations cannot be reconstructed"); + return false; + }; + let coefficient = &coefficients[idx]; + let replay = &mut replays[g]; + for (acc, eval) in replay.acc.iter_mut().zip(evals.iter()) { + *acc = &*acc + coefficient * eval; + } + for (acc, eval) in replay.acc_sym.iter_mut().zip(evals_sym.iter()) { + *acc = &*acc + coefficient * eval; + } + } + + // Each group's FRI, from the folded first layer down to the final polynomial. + for (g, (group, replay)) in groups.iter().zip(replays.iter()).enumerate() { + let member = replay.member; + let fri = group.fri; + let synthetic = StarkProof:: { + trace_length: group.trace_rows, + lde_trace_main_merkle_root: Commitment::default(), + lde_trace_aux_merkle_root: None, + lde_trace_precomputed_merkle_root: None, + trace_ood_evaluations: Table::new(Vec::new(), 0), + trace_ood_next_evaluations: Table::new(Vec::new(), 0), + composition_poly_root: Commitment::default(), + composition_poly_parts_ood_evaluation: Vec::new(), + fri_layers_merkle_roots: fri.layer_roots.clone(), + fri_final_poly_coeffs: fri.final_poly_coeffs.clone(), + query_list: fri.query_list.clone(), + deep_poly_openings: Vec::new(), + nonce: fri.nonce, + bus_public_inputs: None, + public_inputs: public_inputs[member].clone(), + }; + let view = StarkProofView::Owned(&synthetic); + let terminal_offset = replay + .domain + .coset_offset + .pow(1u64 << replay.layout.total_folds); + let terminal_codeword = + crate::fri::terminal::terminal_codeword_from_coeffs::( + &fri.final_poly_coeffs, + &terminal_offset, + replay.layout.terminal_len, + ); + let mut inverses: Vec> = replay + .iotas + .iter() + .map(|&iota| { + V::::query_challenge_to_evaluation_point( + iota, + false, + &replay.domain, + ) + }) + .collect(); + if FieldElement::inplace_batch_inverse(&mut inverses).is_err() { + error!("batched: group {g} has a query at a zero point"); + return false; + } + let ok = (0..num_queries).zip(inverses).all(|(i, inv)| { + V::::verify_query_and_sym_openings( + view, + &replay.zetas, + replay.iotas[i], + view.query(i), + inv, + &replay.acc[i], + &replay.acc_sym[i], + &terminal_codeword, + ) + }); + if !ok { + error!("batched: group {g}'s FRI does not verify"); + return false; + } + } + true +} diff --git a/crypto/stark/src/instruments.rs b/crypto/stark/src/instruments.rs index 0f68059f4..e756dd16f 100644 --- a/crypto/stark/src/instruments.rs +++ b/crypto/stark/src/instruments.rs @@ -290,6 +290,43 @@ pub struct MultiProveTiming { pub heap_snapshots: Vec, } +/// Residency accounting for the retire-LDE / retire-traces modes. +/// +/// The honest budget, not an estimate: how many times a main LDE was actually +/// materialized, how many retired-chunk traces were actually built, and how +/// many shapes were answered without building one. A mode that trades time for +/// memory has to be able to say what the trade cost, or the next change to it +/// is guesswork. +static MAIN_LDE_EXPANSIONS: AtomicU64 = AtomicU64::new(0); +static RETIRED_TRACE_BUILDS: AtomicU64 = AtomicU64::new(0); +static RETIRED_SHAPE_QUERIES: AtomicU64 = AtomicU64::new(0); + +/// A main LDE was materialized from a trace — the Round 1 commit, or a rebuild +/// after it was retired. One per table is the floor; anything above it is what +/// the barriers cost. +pub fn count_main_lde_expansion() { + MAIN_LDE_EXPANSIONS.fetch_add(1, Ordering::Relaxed); +} + +/// A retired chunk's trace was built from its routed ops. +pub fn count_retired_trace_build() { + RETIRED_TRACE_BUILDS.fetch_add(1, Ordering::Relaxed); +} + +/// A retired chunk's shape was derived from op counts, with no trace built. +pub fn count_retired_shape_query() { + RETIRED_SHAPE_QUERIES.fetch_add(1, Ordering::Relaxed); +} + +/// `(main LDE expansions, retired trace builds, shapes answered build-free)`. +pub fn residency_counts() -> (u64, u64, u64) { + ( + MAIN_LDE_EXPANSIONS.load(Ordering::Relaxed), + RETIRED_TRACE_BUILDS.load(Ordering::Relaxed), + RETIRED_SHAPE_QUERIES.load(Ordering::Relaxed), + ) +} + /// Round 1 sub-timings: atomics so parallel rayon workers can accumulate safely. static R1_MAIN_LDE_US: AtomicU64 = AtomicU64::new(0); static R1_MAIN_MERKLE_US: AtomicU64 = AtomicU64::new(0); @@ -365,6 +402,9 @@ pub fn take_r1_sub() -> Round1SubOps { /// In practice this is safe because store/take pairs always execute within the /// same rayon task closure. pub fn reset_all() { + MAIN_LDE_EXPANSIONS.store(0, Ordering::Relaxed); + RETIRED_TRACE_BUILDS.store(0, Ordering::Relaxed); + RETIRED_SHAPE_QUERIES.store(0, Ordering::Relaxed); R1_MAIN_LDE_US.store(0, Ordering::Relaxed); R1_MAIN_MERKLE_US.store(0, Ordering::Relaxed); R1_AUX_LDE_US.store(0, Ordering::Relaxed); diff --git a/crypto/stark/src/lib.rs b/crypto/stark/src/lib.rs index 6f8e7c82e..b7b6b2605 100644 --- a/crypto/stark/src/lib.rs +++ b/crypto/stark/src/lib.rs @@ -3,6 +3,7 @@ #[cfg(all(target_arch = "wasm32", feature = "disk-spill"))] compile_error!("the `disk-spill` feature requires memmap2, which does not compile on wasm32"); +pub mod batched_verifier; #[cfg(feature = "debug-checks")] pub mod bus_debug; pub mod commitment; diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index faf512a72..bc2672685 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -1,5 +1,6 @@ use std::any::Any; use std::marker::PhantomData; +use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; #[cfg(feature = "instruments")] use std::time::{Duration, Instant}; @@ -120,20 +121,66 @@ where pub(crate) precomputed_root: Option, /// Preprocessed tables only: number of precomputed columns. Zero otherwise. pub(crate) num_precomputed_cols: usize, + /// `Some(leaves_len)` when `tree`'s leaf half was freed after committing, so + /// the opening path knows to regenerate the one leaf-level sibling it needs. + /// `None` on a full tree. + pub(crate) leaves_dropped: Option, } impl TableCommit where FieldElement: AsBytes, { + /// Roots without a tree, for a pass that will not open against it. The + /// tree is the expensive half; a caller that already knows the roots and + /// only needs them to travel should not pay for one. + /// + /// Serves preprocessed and plain tables alike — `precomputed` is `Some` + /// exactly when the AIR is preprocessed. + fn known_roots(root: Commitment, precomputed: Option) -> Self { + Self { + tree: Arc::new(BatchedMerkleTree::from_root(root)), + root, + precomputed_tree: None, + precomputed_root: precomputed, + num_precomputed_cols: 0, + leaves_dropped: None, + } + } + /// Build a `TableCommit` for a plain (non-preprocessed) table. - fn plain(tree: BatchedMerkleTree, root: Commitment) -> Self { + fn plain(#[allow(unused_mut)] mut tree: BatchedMerkleTree, root: Commitment) -> Self { + let leaves_dropped = Self::retire_leaves(&mut tree); Self { tree: Arc::new(tree), root, precomputed_tree: None, precomputed_root: None, num_precomputed_cols: 0, + leaves_dropped, + } + } + + /// Free the tree's leaf half in streaming mode, returning the `leaves_len` + /// the opening path needs to rebuild a path without them. + /// + /// Halves a committed tree's footprint: every inner node is kept, so the + /// only node an opening has to regenerate is the leaf-level sibling, and the + /// paths stay byte-identical. Inert under `cuda`, where openings can come + /// off the device instead of the host tree. + #[allow(unused_variables)] + fn retire_leaves(tree: &mut BatchedMerkleTree) -> Option { + #[cfg(feature = "cuda")] + { + None + } + #[cfg(not(feature = "cuda"))] + { + if !streaming_retire_lde() { + return None; + } + let leaves_len = tree.nodes().len().div_ceil(2); + tree.drop_leaves(leaves_len).then_some(leaves_len) } } @@ -147,12 +194,16 @@ where precomputed_root: Commitment, num_precomputed_cols: usize, ) -> Self { + #[allow(unused_mut)] + let mut tree = tree; + let leaves_dropped = Self::retire_leaves(&mut tree); Self { tree: Arc::new(tree), root, precomputed_tree: Some(precomputed_tree), precomputed_root: Some(precomputed_root), num_precomputed_cols, + leaves_dropped, } } @@ -164,6 +215,7 @@ where precomputed_tree: self.precomputed_tree.as_ref().map(Arc::clone), precomputed_root: self.precomputed_root, num_precomputed_cols: self.num_precomputed_cols, + leaves_dropped: self.leaves_dropped, } } @@ -597,6 +649,172 @@ fn host_cores() -> usize { .unwrap_or(4) } +/// A table's Round 1 roots, in the order Fiat-Shamir absorbs them. +/// +/// A plain table contributes one root. A preprocessed one contributes two: its +/// precomputed columns commit separately from the multiplicities, and the +/// transcript takes the precomputed root first. Getting that order or that +/// count wrong yields different challenges from the same execution, which is +/// why the pair travels together instead of as a bare `Commitment`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MainRoots { + /// The precomputed columns' root — `Some` iff the AIR is preprocessed. + pub precomputed: Option, + /// The root of the columns that depend on the execution: the whole trace + /// for a plain AIR, the multiplicities for a preprocessed one. + pub main: Commitment, +} + +/// One height group's FRI: the instance every member of the group folds into. +#[derive( + Debug, + Clone, + serde::Serialize, + serde::Deserialize, + rkyv::Archive, + rkyv::Serialize, + rkyv::Deserialize, +)] +#[serde(bound = "")] +pub struct GroupFri { + pub layer_roots: Vec, + pub final_poly_coeffs: Vec>, + /// The query indices the whole group answers, and which each member's + /// openings are taken at. + pub iotas: Vec, + pub query_list: Vec>, + pub nonce: Option, +} + +/// One table's contribution to a batched FRI. +#[derive(Clone)] +pub struct TableDeep { + /// The domain the codeword lives on. Only tables that agree on this can be + /// folded together: the fold squares the coset offset each layer, so a + /// short codeword over `offset·` never lines up with a tall fold over + /// `offset²·`. + pub lde_size: usize, + /// Rows before the blowup. Carried rather than divided out of `lde_size`, + /// because the batch has to rebuild the very domain the codeword was + /// computed on and a wrong one gives wrong twiddles and a silently wrong + /// fold. + pub trace_rows: usize, + /// The DEEP composition codeword, `lde_size` long. + pub deep: Vec>, + /// What the batch's coefficient is drawn from: this table's public round-3 + /// data, in the order a transcript absorbs it. + /// + /// Not the fork's state, which was the first design and is unusable — the + /// verifier has no forks, only a proof. Everything here is carried in the + /// proof, so the verifier rebuilds the same seed from the same bytes. + pub bus_contribution: Option>, + /// The round 1 roots and the full bus inputs, which the proof carries even + /// though only the contribution goes into the seed. + pub main_roots: MainRoots, + pub aux_root: Option, + pub bus_public_inputs: Option>, + pub composition_poly_root: Commitment, + pub trace_ood: Table, + pub trace_ood_next: Table, + pub parts_ood: Vec>, + /// The composition parts over the LDE domain, kept for the Open pass when + /// the caller trades memory for not recomputing them. Round 2 is the + /// constraint evaluation, the costliest thing a rebuild does. + pub composition_lde: Option>>>, +} + +/// Source of truth for a table whose *trace* has been retired. +/// +/// The retire-LDE mode ([`streaming_retire_lde`]) drops a table's LDE and +/// rebuilds it from the still-resident trace. This goes one rung further down +/// the same ladder: drop the trace too, and rebuild it from the compact routed +/// op lists it was built from. The prover asks for a trace at the two points it +/// needs one — the Round 1 main commit, and the table's fused chain — and drops +/// it again after each. +/// +/// `build_main` MUST be deterministic: the trace rebuilt for the fused chain has +/// to be byte-identical to the one Round 1 committed, or the root will not match +/// what the verifier recomputes. +/// +/// Port of Approach 1 step C.2b (PR #647, commit a7eabd3c). +pub trait TraceProvider: Sync +where + Field: IsSubFieldOf + IsField, + FieldExtension: IsField, +{ + /// Whether table `idx` is retired (built on demand) rather than resident. + fn is_retired(&self, idx: usize) -> bool; + + /// Row count of table `idx`'s main trace. Cheap: the pre-pass sizes the LDE + /// domain with it, without materializing the trace. + fn num_rows(&self, idx: usize) -> usize; + + /// Main-column count of table `idx`. Cheap, like `num_rows`: the memory + /// estimates need the table's width before anything is materialized. + fn num_main_columns(&self, idx: usize) -> usize; + + /// Build the main-only trace (no auxiliary columns) for retired table `idx`. + fn build_main(&self, idx: usize) -> TraceTable; +} + +/// Retire each table's main LDE right after the Round 1 commit and rebuild it +/// on demand inside the table's fused chain, instead of holding all N of them +/// across the Round 1 barrier. +/// +/// Round 1's main commit is a phase-wide barrier (the shared LogUp challenges +/// need every root absorbed first), so all N tables' main LDEs are live at once +/// — `O(N x main_cols x lde_size)`, the largest single term in the prover's +/// peak. Retiring trades one extra LDE expansion (iFFT + coset + FFT) per table +/// for dropping that term to `O(k x ...)`. +/// +/// Driven by `LAMBDA_STREAM_LDE`: `1`/`true` forces it on, `0`/unset off, and +/// `auto` lets the caller decide from an estimate of the proof's peak RAM (the +/// prover crate resolves `auto` through [`set_retire_lde`] before proving, so +/// the estimate never costs anything on the default path). Inert under `cuda`, +/// where the LDE lives on the device and the host buffer is already empty on +/// the device-only path. +/// +/// Port of Approach 1 milestone M1 (PR #647, commit 6562c5f4). +pub fn streaming_retire_lde() -> bool { + match RETIRE_LDE_OVERRIDE.load(Ordering::Relaxed) { + RETIRE_OVERRIDE_ON => true, + RETIRE_OVERRIDE_OFF => false, + _ => matches!( + std::env::var("LAMBDA_STREAM_LDE").as_deref(), + Ok("1") | Ok("true") + ), + } +} + +/// Whether `LAMBDA_STREAM_LDE=auto` asked for the decision to be made from a +/// peak-RAM estimate. Only then does the caller pay for that estimate. +pub fn streaming_retire_lde_is_auto() -> bool { + matches!(std::env::var("LAMBDA_STREAM_LDE").as_deref(), Ok("auto")) +} + +/// Resolve `LAMBDA_STREAM_LDE=auto` to a decision for the rest of the process. +/// +/// Process-global, like the env var it resolves, and read once per table inside +/// `multi_prove` — so it must be set before proving starts and must not change +/// mid-proof, or a table would be rebuilt against a commitment it never +/// produced. Set it only from the resolution of `auto`: an explicit `0`/`1` +/// is the operator overriding the estimate, and this must not silently undo it. +pub fn set_retire_lde(on: bool) { + RETIRE_LDE_OVERRIDE.store( + if on { + RETIRE_OVERRIDE_ON + } else { + RETIRE_OVERRIDE_OFF + }, + Ordering::Relaxed, + ); +} + +const RETIRE_OVERRIDE_UNSET: u8 = 0; +const RETIRE_OVERRIDE_OFF: u8 = 1; +const RETIRE_OVERRIDE_ON: u8 = 2; +static RETIRE_LDE_OVERRIDE: AtomicU8 = AtomicU8::new(RETIRE_OVERRIDE_UNSET); + /// Number of tables `multi_prove` proves concurrently, out of `num_airs` of /// them. /// @@ -607,6 +825,9 @@ fn host_cores() -> usize { /// `1..=num_airs`. Without the `parallel` feature this is 1 and the env var is /// ignored. /// +/// Not to be confused with `prover::pass::table_parallelism`, which sizes the +/// prove-and-retire walk's batches and reads `A1_TABLE_PARALLELISM`. +/// /// # Why the `cuda` arm has no core term /// /// Measured over 881 runs on two RTX 5090 boxes (sweep record linked from @@ -1305,6 +1526,8 @@ pub trait IsStarkProver< &twiddles.two_half_fwd, ) .expect("row-major coset LDE expansion"); + #[cfg(feature = "instruments")] + crate::instruments::count_main_lde_expansion(); #[cfg(feature = "instruments")] let main_lde_dur = t_sub.elapsed(); @@ -1412,6 +1635,38 @@ pub trait IsStarkProver< Ok(()) } + /// Rebuild a table's row-major main LDE from its trace. + /// + /// Byte-identical to what the Round 1 main commit produced: it runs the same + /// production path (row-major copy + cache-blocked two-half coset LDE), not + /// the column-wise debug reconstruction. Used by the retire-LDE streaming + /// path, which drops this buffer after the commit and rebuilds it here. + fn rebuild_main_lde( + trace: &TraceTable, + domain: &Domain, + twiddles: &LdeTwiddles, + ) -> (Vec>, usize) { + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + let (trace_data, total_cols) = trace.main_data_row_major(); + + let mut main_data: Vec> = Vec::with_capacity(lde_size * total_cols); + main_data.extend_from_slice(trace_data); + + Polynomial::>::coset_lde_full_expand_row_major::( + &mut main_data, + total_cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .expect("row-major coset LDE expansion"); + #[cfg(feature = "instruments")] + crate::instruments::count_main_lde_expansion(); + + (main_data, total_cols) + } + /// Recompute Round1 from the trace, reusing the Merkle trees stored in commitments. /// /// Only used by `run_debug_checks` — the production path consumes the @@ -1490,6 +1745,732 @@ pub trait IsStarkProver< )) } + /// Commit one table and return its main-trace Merkle root. + /// + /// Approach 1's Commit phase closes a table mid-execution and commits it + /// right there, before the tables after it exist. This is that step alone: + /// the same row-major coset LDE and the same row-pair leaf layout Round 1 + /// uses, so a table committed during the walk carries the root Round 1 + /// would have given it. + /// + /// Only the root comes back. The tree is what the openings later read, and + /// a caller that just has to put a commitment in the transcript should not + /// pay to hold it. + fn commit_table_root( + air: &dyn AIR, + trace: &TraceTable, + ) -> Option + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + { + let (domain, twiddles) = domain_and_twiddles(air, trace.num_rows()); + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + let (data, cols) = trace.main_data_row_major(); + if cols == 0 || data.is_empty() { + return None; + } + let mut lde: Vec> = Vec::with_capacity(lde_size * cols); + lde.extend_from_slice(data); + Polynomial::>::coset_lde_full_expand_row_major::( + &mut lde, + cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .ok()?; + if !air.is_preprocessed() { + return Self::commit_rows_bit_reversed::(&lde, cols).map(|(_, root)| { + MainRoots { + precomputed: None, + main: root, + } + }); + } + // A preprocessed table commits as two trees, and the transcript absorbs + // both. The precomputed half is a constant of the AIR, so it is derived + // here only to be checked against that constant — the same check + // `commit_main_trace` makes, and the one that catches a table whose + // precomputed columns were built wrong. + let num_precomputed = air.num_precomputed_columns(); + let (_, precomputed_root) = + Self::commit_rows_bit_reversed_subset::(&lde, cols, 0, num_precomputed)?; + if precomputed_root != air.precomputed_commitment() { + return None; + } + let (_, main) = + Self::commit_rows_bit_reversed_subset::(&lde, cols, num_precomputed, cols)?; + Some(MainRoots { + precomputed: Some(precomputed_root), + main, + }) + } + + /// A table's auxiliary commitment, built against the shared challenges and + /// dropped with the call. + /// + /// [`Self::commit_table_root`]'s counterpart for the LogUp pass. The aux + /// columns are written into `trace`, expanded and committed, and everything + /// this allocated dies here — which is the point: a prover that holds one + /// table at a time cannot keep the aux LDE around for later rounds, so it + /// re-derives it when it needs it again. + /// + /// Returns the root together with the bus public inputs the aux build + /// produced, since the proof carries them. A table with no aux trace has + /// nothing to commit; callers ask `air.has_aux_trace()` rather than reading + /// that off a `None`, which also means a failed commit. + fn commit_aux_root( + air: &dyn AIR, + trace: &mut TraceTable, + challenges: &[FieldElement], + ) -> Option<(Commitment, Option>)> + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + { + let (domain, twiddles) = domain_and_twiddles(air, trace.num_rows()); + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + let bus_public_inputs = air.build_auxiliary_trace(trace, challenges); + + let (trace_data, total_cols) = trace.aux_data_row_major(); + if total_cols == 0 || trace_data.is_empty() { + return None; + } + let mut aux_data: Vec> = + Vec::with_capacity(lde_size * total_cols); + aux_data.extend_from_slice(trace_data); + Polynomial::>::coset_lde_full_expand_row_major::( + &mut aux_data, + total_cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .ok()?; + let (_, root) = Self::commit_rows_bit_reversed(&aux_data, total_cols)?; + Some((root, bus_public_inputs)) + } + + /// One table's whole proof, rebuilt from its trace and dropped with the + /// call. + /// + /// The composition polynomial needs both LDEs at once, so this is where a + /// pass that holds one table at a time pays its widest moment: main LDE, + /// auxiliary LDE and the composition parts, for one table. The ordinary + /// prover keeps all three for every table simultaneously. + /// + /// `transcript` must be the table's own fork — the shared state after the + /// LogUp challenges, domain-separated by AIR index — with nothing appended + /// yet. The auxiliary root and the table's bus contribution go in here, as + /// they do in the fused path, so every challenge below comes out identical + /// and the proof is the one the ordinary prover would have written. + fn prove_table_from_trace( + air: &dyn AIR, + pub_inputs: &PI, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut (impl IsStarkTranscript + Clone), + ) -> Result, ProvingError> + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + PI: Send + Sync + Clone, + { + let (domain, twiddles) = domain_and_twiddles(air, trace.num_rows()); + #[cfg(feature = "instruments")] + let __r1 = crate::instruments::span("a1_round_1"); + let mut round_1_result = + Self::round_1_from_trace(air, trace, challenges, transcript, None)?; + #[cfg(feature = "instruments")] + drop(__r1); + #[cfg(feature = "instruments")] + let __r24 = crate::instruments::span("a1_rounds_2_to_4"); + let out = Self::prove_rounds_2_to_4( + air, + pub_inputs, + &mut round_1_result, + transcript, + &domain, + &twiddles, + ); + #[cfg(feature = "instruments")] + drop(__r24); + out + } + + /// Round 1 for one table, rebuilt from its trace. + /// + /// Both LDEs and both commitments, and the two things the table's own fork + /// takes before round 2 samples anything: the auxiliary root, then the bus + /// contribution. Leaving the contribution out moves beta and everything + /// below it, while the main and auxiliary roots still match — a symptom + /// that points nowhere near the transcript. + fn round_1_from_trace( + air: &dyn AIR, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut (impl IsStarkTranscript + Clone), + known_main: Option, + ) -> Result, ProvingError> + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + { + let (domain, twiddles) = domain_and_twiddles(air, trace.num_rows()); + let lde_size = domain.interpolation_domain_size * domain.blowup_factor; + + let bus_public_inputs = if air.has_aux_trace() { + air.build_auxiliary_trace(trace, challenges) + } else { + None + }; + + let expand_main = |data: &[FieldElement], cols: usize| { + let mut out: Vec> = Vec::with_capacity(lde_size * cols); + out.extend_from_slice(data); + Polynomial::>::coset_lde_full_expand_row_major::( + &mut out, + cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .map(|_| out) + }; + + #[cfg(feature = "instruments")] + let __m = crate::instruments::span("a1_r1_main"); + let (main_src, num_main_cols) = trace.main_data_row_major(); + let main_data = + expand_main(main_src, num_main_cols).map_err(|_| ProvingError::EmptyCommitment)?; + // A caller that already has this table's main roots — the Commit phase + // computed every one of them — and that will not open against the tree + // can hand them over instead. Building the tree is the hottest thing in + // a prove (keccak is 17.6% of the profile), so not building one that + // nothing will ask a question of is the cheapest saving there is. + let main = match known_main { + Some(roots) => TableCommit::known_roots(roots.main, roots.precomputed), + None => Self::table_commit_for(air, &main_data, num_main_cols)?, + }; + + #[cfg(feature = "instruments")] + drop(__m); + #[cfg(feature = "instruments")] + let __a = crate::instruments::span("a1_r1_aux"); + let (aux_data, num_aux_cols, aux) = if air.has_aux_trace() { + let (aux_src, cols) = trace.aux_data_row_major(); + let mut out: Vec> = Vec::with_capacity(lde_size * cols); + out.extend_from_slice(aux_src); + Polynomial::>::coset_lde_full_expand_row_major::( + &mut out, + cols, + domain.blowup_factor, + &twiddles.coset_weights, + &twiddles.two_half_inv, + &twiddles.two_half_fwd, + ) + .map_err(|_| ProvingError::EmptyCommitment)?; + #[cfg(feature = "instruments")] + let __am = crate::instruments::span("a1_aux_merkle"); + let (tree, root) = + Self::commit_rows_bit_reversed(&out, cols).ok_or(ProvingError::EmptyCommitment)?; + #[cfg(feature = "instruments")] + drop(__am); + (out, cols, Some(TableCommit::plain(tree, root))) + } else { + (Vec::new(), 0, None) + }; + + #[cfg(feature = "instruments")] + drop(__a); + // The fork takes the auxiliary root, then the table's bus contribution, + // before round 2 samples anything. Both, in that order — the + // contribution is what ties this table's share of the LogUp bus into + // its own challenges, and leaving it out moves every one of them. + if let Some(ref c) = aux { + transcript.append_bytes(&c.root); + } + if let Some(ref bpi) = bus_public_inputs { + transcript.append_field_element(&bpi.table_contribution); + } + + let round_1_result = Round1 { + lde_trace: LDETraceTable::from_row_major( + main_data, + num_main_cols, + aux_data, + num_aux_cols, + air.step_size(), + domain.blowup_factor, + ), + main, + aux, + rap_challenges: challenges.to_vec(), + bus_public_inputs, + }; + + Ok(round_1_result) + } + + /// Rounds 2 and 3 for one table, against its own fork. + /// + /// Returns the out-of-domain point with them: rounds 4 and 5 open against + /// it, and re-deriving it would mean re-running round 2 to get the + /// composition root the transcript needs first. + #[allow(clippy::type_complexity)] + fn rounds_2_and_3( + air: &dyn AIR, + pub_inputs: &PI, + round_1_result: &mut Round1, + transcript: &mut (impl IsStarkTranscript + Clone), + domain: &Domain, + twiddles: &LdeTwiddles, + ) -> Result< + ( + Round2, + Round3, + FieldElement, + Table, + Table, + ), + ProvingError, + > + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + PI: Send + Sync + Clone, + { + let beta = transcript.sample_field_element(); + let num_boundary_constraints = air + .boundary_constraints( + pub_inputs, + &round_1_result.rap_challenges, + round_1_result.bus_public_inputs.as_ref(), + domain.interpolation_domain_size, + ) + .constraints + .len(); + let num_transition_constraints = air.context().num_transition_constraints; + let mut coefficients: Vec<_> = + core::iter::successors(Some(FieldElement::one()), |x| Some(x * &beta)) + .take(num_boundary_constraints + num_transition_constraints) + .collect(); + let transition_coefficients: Vec<_> = + coefficients.drain(..num_transition_constraints).collect(); + let boundary_coefficients = coefficients; + + let mut round_2_result = Self::round_2_compute_composition_polynomial( + air, + pub_inputs, + domain, + twiddles, + round_1_result, + &transition_coefficients, + &boundary_coefficients, + )?; + transcript.append_bytes(&round_2_result.composition_poly_root); + + let z = transcript.sample_z_ood( + &domain.lde_roots_of_unity_coset, + &domain.trace_roots_of_unity, + ); + let round_3_result = Self::round_3_evaluate_polynomials_in_out_of_domain_element( + air, + domain, + round_1_result, + &mut round_2_result, + &z, + ); + + // The fork is left standing where round 4 would pick it up: the two + // out-of-domain blocks and then the composition parts, in the order the + // verifier absorbs them. A pass that batches the FRI has to fold these + // states together, so it needs them advanced this far. + let (ood_block0, ood_block1) = + Self::ood_layout(air).split_full(&round_3_result.trace_ood_evaluations); + for block in [&ood_block0, &ood_block1] { + for col in block.columns().iter() { + for elem in col.iter() { + transcript.append_field_element(elem); + } + } + } + for element in round_3_result.composition_poly_parts_ood_evaluation.iter() { + transcript.append_field_element(element); + } + + Ok((round_2_result, round_3_result, z, ood_block0, ood_block1)) + } + + /// One table taken as far as a batched FRI lets it go on its own. + /// + /// Rounds 1 to 3, then the DEEP composition codeword — and there it stops, + /// because the next thing is a fold whose coefficient binds every table in + /// the batch and so cannot be known yet. + /// + /// The codeword is kept rather than the LDEs it came from. That is the + /// whole reason this split is affordable: on the ethrex block the trace and + /// composition LDEs of every table are tens of gigabytes, while their + /// DEEP codewords together are about 6.5 GB — one extension element per row + /// instead of every column. Holding them is what saves walking the + /// execution again just to recompute them once the coefficient is known. + fn deep_for_table( + air: &dyn AIR, + pub_inputs: &PI, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut (impl IsStarkTranscript + Clone), + known_main: Option, + keep_composition: bool, + ) -> Result, ProvingError> + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + PI: Send + Sync + Clone, + { + let (domain, twiddles) = domain_and_twiddles(air, trace.num_rows()); + #[cfg(feature = "instruments")] + let __f1 = crate::instruments::span("a1_fold_r1"); + let mut round_1_result = + Self::round_1_from_trace(air, trace, challenges, transcript, known_main)?; + #[cfg(feature = "instruments")] + drop(__f1); + #[cfg(feature = "instruments")] + let __f23 = crate::instruments::span("a1_fold_r23"); + let (mut round_2_result, round_3_result, z, trace_ood, trace_ood_next) = + Self::rounds_2_and_3( + air, + pub_inputs, + &mut round_1_result, + transcript, + &domain, + &twiddles, + )?; + + #[cfg(feature = "instruments")] + drop(__f23); + #[cfg(feature = "instruments")] + let __fd = crate::instruments::span("a1_fold_deep"); + // Round 4's opening move, up to the point where the batch takes over: + // gamma is this table's own, sampled from its own fork. + let gamma = transcript.sample_field_element(); + let n_terms_composition_poly = round_2_result.lde_composition_poly_evaluations.len(); + let layout = Self::ood_layout(air); + let num_terms_trace = layout.num_surviving(); + let mut coefficients: Vec<_> = + core::iter::successors(Some(FieldElement::one()), |x| Some(x * &gamma)) + .take(n_terms_composition_poly + num_terms_trace) + .collect(); + let trace_term_powers: Vec<_> = coefficients.drain(..num_terms_trace).collect(); + let trace_term_coeffs = layout.build_trace_term_coeffs(&trace_term_powers); + + let deep = Self::compute_deep_composition_poly_evaluations( + &mut round_1_result.lde_trace, + &mut round_2_result, + &round_3_result, + &z, + &domain, + &domain.trace_primitive_root, + &coefficients, + &trace_term_coeffs, + ); + + // Bit-reversed here rather than at fold time. FRI wants it that way, and + // the permutation depends only on the length — which every member of a + // group shares — so permuting each codeword before the fold gives the + // same result as permuting the sum, one pass earlier. + let mut deep = deep; + in_place_bit_reverse_permute(&mut deep); + + #[cfg(feature = "instruments")] + drop(__fd); + Ok(TableDeep { + lde_size: domain.interpolation_domain_size * domain.blowup_factor, + trace_rows: domain.interpolation_domain_size, + deep, + bus_contribution: round_1_result + .bus_public_inputs + .as_ref() + .map(|b| b.table_contribution.clone()), + main_roots: MainRoots { + precomputed: round_1_result.main.precomputed_root, + main: round_1_result.main.root, + }, + aux_root: round_1_result.aux.as_ref().map(|c| c.root), + bus_public_inputs: round_1_result.bus_public_inputs.clone(), + composition_poly_root: round_2_result.composition_poly_root, + trace_ood, + trace_ood_next, + parts_ood: round_3_result.composition_poly_parts_ood_evaluation.clone(), + composition_lde: keep_composition + .then(|| std::mem::take(&mut round_2_result.lde_composition_poly_evaluations)), + }) + } + + /// The batch's coefficient, drawn from every table's round-3 data. + /// + /// `pre_fork` is the shared transcript as it stood before the per-table + /// forks — after the LogUp challenges and nothing else. On top of it go, per + /// table in AIR order: the bus contribution when there is one, the + /// composition root, the two out-of-domain blocks column by column, and the + /// composition parts. That byte order is the protocol, and the verifier + /// walks it from the same fields the proof carries, which is why this reads + /// public data rather than the forks — the verifier has no forks. + /// + /// Drawing `alpha` from all of it is what makes the fold binding: a table + /// cannot be swapped after the fact without moving the coefficient that + /// folded it. + fn fold_coefficient( + seed: &mut (impl IsStarkTranscript + Clone), + table: &TableDeep, + ) -> FieldElement + where + FieldElement: AsBytes, + FieldElement: AsBytes, + { + if let Some(ref c) = table.bus_contribution { + seed.append_field_element(c); + } + seed.append_bytes(&table.composition_poly_root); + for block in [&table.trace_ood, &table.trace_ood_next] { + for col in block.columns().iter() { + for elem in col.iter() { + seed.append_field_element(elem); + } + } + } + for elem in table.parts_ood.iter() { + seed.append_field_element(elem); + } + seed.sample_field_element() + } + + /// Every table's coefficient, in the order they are folded. + /// + /// Only for reasoning about the sequence as a whole; the prover draws them + /// one at a time, as it folds. + fn fold_coefficients( + pre_fork: &(impl IsStarkTranscript + Clone), + tables: &[TableDeep], + ) -> Vec> + where + FieldElement: AsBytes, + FieldElement: AsBytes, + { + let mut seed = pre_fork.clone(); + tables + .iter() + .map(|t| Self::fold_coefficient(&mut seed, t)) + .collect() + } + + /// Add `coefficient * codeword` into a group's running accumulator. + /// + /// The accumulator is the batch polynomial the spec describes. A member is + /// added and dropped, so what is held is one codeword per distinct domain + /// rather than one per table — which on the ethrex block is 13 instead of + /// one per table, and about a gigabyte instead of eight and a half. + fn accumulate( + acc: &mut Vec>, + coefficient: &FieldElement, + codeword: &[FieldElement], + ) { + if acc.is_empty() { + acc.resize(codeword.len(), FieldElement::::zero()); + } + for (dst, src) in acc.iter_mut().zip(codeword.iter()) { + *dst = &*dst + coefficient * src; + } + } + + /// One table's openings, at the indices its group decided. + /// + /// The Open pass: the batched FRI fixed the query indices for a whole + /// group, and every member owes its rows at those indices. The table is + /// rebuilt to serve them — round 1 for the trace commitments and round 2 + /// for the composition ones — and dies with the call, which is the trade + /// the approach makes everywhere else too. + /// + /// `transcript` is the table's own fork again, in the same state round 1 + /// expects, because rebuilding walks the same rounds it walked before. + fn open_for_table( + air: &dyn AIR, + pub_inputs: &PI, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut (impl IsStarkTranscript + Clone), + iotas: &[usize], + ) -> Result, ProvingError> + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + PI: Send + Sync + Clone, + { + let (domain, twiddles) = domain_and_twiddles(air, trace.num_rows()); + #[cfg(feature = "instruments")] + let __o1 = crate::instruments::span("a1_open_r1"); + let mut round_1_result = + Self::round_1_from_trace(air, trace, challenges, transcript, None)?; + #[cfg(feature = "instruments")] + drop(__o1); + #[cfg(feature = "instruments")] + let __o23 = crate::instruments::span("a1_open_r23"); + let (round_2_result, _, _, _, _) = Self::rounds_2_and_3( + air, + pub_inputs, + &mut round_1_result, + transcript, + &domain, + &twiddles, + )?; + #[cfg(feature = "instruments")] + drop(__o23); + #[cfg(feature = "instruments")] + let __od = crate::instruments::span("a1_open_deep"); + let out = + Self::open_deep_composition_poly(&domain, &round_1_result, &round_2_result, iotas); + #[cfg(feature = "instruments")] + drop(__od); + Ok(out) + } + + /// [`open_for_table`](Self::open_for_table) with the composition parts the + /// fold pass kept: round 1 is rebuilt for the trace commitments, the + /// composition commitment is rebuilt from the evaluations, and rounds 2 and + /// 3 — the constraint evaluation and the out-of-domain values — are skipped. + fn open_for_table_kept( + air: &dyn AIR, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut (impl IsStarkTranscript + Clone), + iotas: &[usize], + composition_lde: Vec>>, + ) -> Result, ProvingError> + where + FieldElement: AsBytes + math::traits::ByteConversion, + FieldElement: AsBytes + math::traits::ByteConversion, + PI: Send + Sync + Clone, + { + let (domain, _twiddles) = domain_and_twiddles(air, trace.num_rows()); + #[cfg(feature = "instruments")] + let __o1 = crate::instruments::span("a1_open_r1"); + let round_1_result = Self::round_1_from_trace(air, trace, challenges, transcript, None)?; + #[cfg(feature = "instruments")] + drop(__o1); + #[cfg(feature = "instruments")] + let __o2 = crate::instruments::span("a1_open_kept_commit"); + let (composition_poly_merkle_tree, composition_poly_root) = + crate::commitment::commit_bit_reversed( + &composition_lde, + crate::commitment::ROWS_PER_LEAF, + ) + .ok_or(ProvingError::EmptyCommitment)?; + let round_2_result = Round2 { + lde_composition_poly_evaluations: composition_lde, + composition_poly_merkle_tree, + composition_poly_root, + #[cfg(feature = "cuda")] + gpu_composition_tree: None, + }; + #[cfg(feature = "instruments")] + drop(__o2); + #[cfg(feature = "instruments")] + let __od = crate::instruments::span("a1_open_deep"); + let out = + Self::open_deep_composition_poly(&domain, &round_1_result, &round_2_result, iotas); + #[cfg(feature = "instruments")] + drop(__od); + Ok(out) + } + + /// One FRI over a group's finished accumulator. + /// + /// The members were folded in as they were produced, so by here the group + /// is a single codeword and nothing of the tables remains. + fn batch_fri( + air: &dyn AIR, + acc: Vec>, + trace_rows: usize, + transcript: &mut (impl IsStarkTranscript + Clone), + ) -> Option> + where + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + { + if acc.is_empty() { + return None; + } + let (domain, _) = domain_and_twiddles(air, trace_rows); + let coset_offset = FieldElement::::from(air.context().proof_options.coset_offset); + let (final_poly_coeffs, layers) = fri::commit_phase_from_evaluations( + acc, + transcript, + &coset_offset, + domain.lde_roots_of_unity_coset.len(), + domain.blowup_factor.trailing_zeros(), + air.options().fri_final_poly_log_degree as u32, + domain.fri_inv_twiddles(), + ); + + let grinding_factor = air.context().proof_options.grinding_factor; + let mut nonce = None; + if grinding_factor > 0 { + let value = grinding::generate_nonce_maybe_gpu(&transcript.state(), grinding_factor)?; + transcript.append_bytes(&value.to_be_bytes()); + nonce = Some(value); + } + let iotas = + Self::sample_query_indexes(air.options().fri_number_of_queries, &domain, transcript); + let query_list = fri::query_phase(&layers, &iotas); + + Some(GroupFri { + layer_roots: layers.iter().map(|l| l.merkle_tree.root).collect(), + final_poly_coeffs, + iotas, + query_list, + nonce, + }) + } + + /// The main commitment of an already-expanded LDE, split when the AIR is + /// preprocessed. Shares [`Self::commit_table_root`]'s rule, but keeps the + /// trees, which rounds 2-4 need. + fn table_commit_for( + air: &dyn AIR, + lde: &[FieldElement], + cols: usize, + ) -> Result, ProvingError> + where + FieldElement: AsBytes + math::traits::ByteConversion, + { + if !air.is_preprocessed() { + let (tree, root) = + Self::commit_rows_bit_reversed(lde, cols).ok_or(ProvingError::EmptyCommitment)?; + return Ok(TableCommit::plain(tree, root)); + } + let num_precomputed = air.num_precomputed_columns(); + let (precomputed_tree, precomputed_root) = + Self::commit_rows_bit_reversed_subset(lde, cols, 0, num_precomputed) + .ok_or(ProvingError::EmptyCommitment)?; + if precomputed_root != air.precomputed_commitment() { + return Err(ProvingError::PrecomputedCommitmentMismatch); + } + let (mult_tree, mult_root) = + Self::commit_rows_bit_reversed_subset(lde, cols, num_precomputed, cols) + .ok_or(ProvingError::EmptyCommitment)?; + Ok(TableCommit::preprocessed( + mult_tree, + mult_root, + std::sync::Arc::new(precomputed_tree), + precomputed_root, + num_precomputed, + )) + } + /// Reconstruct Round1 for every table, print the bus balance report, and /// validate each trace. Called once after every table's aux commit, which /// under `debug-checks` means between the fused chain's two admitted @@ -2692,25 +3673,67 @@ pub trait IsStarkProver< tree: &BatchedMerkleTree, challenge: usize, gather: G, + leaves_dropped: Option, ) -> PolynomialOpenings where C: IsField, - FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, G: Fn(usize) -> Vec>, { let domain_size = domain.lde_roots_of_unity_coset.len() as u64; + let proof = match leaves_dropped { + None => tree + .get_proof_by_pos(challenge) + .expect("FRI query index in bounds"), + // Leaf-dropped tree: every node of the path is retained except the + // leaf-level sibling, which is rehashed here from the same rows, in + // the same order and byte layout, that the commit hashed. + Some(leaves_len) => { + let sibling = BatchedMerkleTree::::sibling_leaf_position(challenge); + let leaf = Self::hash_row_pair_leaf(&gather, sibling, domain_size); + tree.get_proof_by_pos_with_leaf_sibling(challenge, leaves_len, leaf) + .expect("FRI query index in bounds") + } + }; // Rows `2·challenge` and `2·challenge+1` are committed together as the // single leaf at position `challenge`; one Merkle path authenticates both // the queried row and its symmetric counterpart. PolynomialOpenings { - proof: tree - .get_proof_by_pos(challenge) - .expect("FRI query index in bounds"), + proof, evaluations: gather(reverse_index(challenge * 2, domain_size)), evaluations_sym: gather(reverse_index(challenge * 2 + 1, domain_size)), } } + /// Rehash one Merkle leaf from the LDE rows behind it. + /// + /// Must mirror `commit_rows_bit_reversed_subset`'s `hash_leaf` exactly: the + /// `ROWS_PER_LEAF` bit-reversed rows concatenated, each element big-endian, + /// over the same column range — which `gather` already carries, since it is + /// the same closure the openings are read with. + fn hash_row_pair_leaf(gather: &G, leaf_idx: usize, num_rows: u64) -> Commitment + where + C: IsField, + FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, + G: Fn(usize) -> Vec>, + { + use math::traits::ByteConversion; + const ROWS_PER_LEAF: usize = crate::commitment::ROWS_PER_LEAF; + + let byte_len = as ByteConversion>::BYTE_LEN; + let mut buf = Vec::new(); + for k in 0..ROWS_PER_LEAF { + let row = gather(reverse_index(ROWS_PER_LEAF * leaf_idx + k, num_rows)); + let start = buf.len(); + buf.resize(start + row.len() * byte_len, 0u8); + for (i, elem) in row.iter().enumerate() { + let at = start + i * byte_len; + elem.write_bytes_be(&mut buf[at..at + byte_len]); + } + } + BatchedMerkleTreeBackend::::hash_bytes(&buf) + } + /// Like [`Self::open_polys_with`], but uses a Merkle proof already gathered /// from the resident device tree (see [`crate::gpu_lde::gather_proofs_dev`]) /// instead of walking a host tree. Row-pair leaf: one proof at position @@ -2826,10 +3849,11 @@ pub trait IsStarkProver< col_range: std::ops::Range, what: &str, gather: G, + leaves_dropped: Option, ) -> PolynomialOpenings where C: IsField, - FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send + math::traits::ByteConversion, G: Fn(usize) -> Vec>, { let Some(proofs) = dev_proofs else { @@ -2846,7 +3870,7 @@ pub trait IsStarkProver< !tree.is_root_only(), "R4 {what} opening fell back to a root-only host tree (nodes device-resident)" ); - return Self::open_polys_with(domain, tree, challenge, gather); + return Self::open_polys_with(domain, tree, challenge, gather, leaves_dropped); }; let proof = proofs[qi].clone(); let Some(dev_vals) = dev_values else { @@ -3079,12 +4103,17 @@ pub trait IsStarkProver< |row| { lde_trace.gather_main_row_range(row, num_precomputed_cols, total_cols) }, + main_commit.leaves_dropped, ) } #[cfg(not(feature = "cuda"))] - Self::open_polys_with(domain, &main_commit.tree, *index, |row| { - lde_trace.gather_main_row_range(row, num_precomputed_cols, total_cols) - }) + Self::open_polys_with( + domain, + &main_commit.tree, + *index, + |row| lde_trace.gather_main_row_range(row, num_precomputed_cols, total_cols), + main_commit.leaves_dropped, + ) } else { #[cfg(feature = "cuda")] { @@ -3100,13 +4129,18 @@ pub trait IsStarkProver< 0..total_cols, "main", |row| lde_trace.gather_main_row(row), + main_commit.leaves_dropped, ) } #[cfg(not(feature = "cuda"))] { - Self::open_polys_with(domain, &main_commit.tree, *index, |row| { - lde_trace.gather_main_row(row) - }) + Self::open_polys_with( + domain, + &main_commit.tree, + *index, + |row| lde_trace.gather_main_row(row), + main_commit.leaves_dropped, + ) } }; @@ -3159,16 +4193,24 @@ pub trait IsStarkProver< "R4 precomputed opening fell back to the host gather, \ but it is device-only (empty)" ); - Self::open_polys_with(domain, tree, *index, |row| { - lde_trace.gather_main_row_range(row, 0, num_precomputed_cols) - }) + Self::open_polys_with( + domain, + tree, + *index, + |row| lde_trace.gather_main_row_range(row, 0, num_precomputed_cols), + None, + ) } } } #[cfg(not(feature = "cuda"))] - Self::open_polys_with(domain, tree, *index, |row| { - lde_trace.gather_main_row_range(row, 0, num_precomputed_cols) - }) + Self::open_polys_with( + domain, + tree, + *index, + |row| lde_trace.gather_main_row_range(row, 0, num_precomputed_cols), + None, + ) }); let composition_openings = { @@ -3255,13 +4297,18 @@ pub trait IsStarkProver< 0..lde_trace.num_aux_cols(), "aux", |row| lde_trace.gather_aux_row(row), + aux.leaves_dropped, ) } #[cfg(not(feature = "cuda"))] { - Self::open_polys_with(domain, &aux.tree, *index, |row| { - lde_trace.gather_aux_row(row) - }) + Self::open_polys_with( + domain, + &aux.tree, + *index, + |row| lde_trace.gather_aux_row(row), + aux.leaves_dropped, + ) } }); @@ -3297,9 +4344,36 @@ pub trait IsStarkProver< /// /// The transcript must be safely initialized before passing it to this method. fn multi_prove( + air_trace_pairs: Vec>, + transcript: &mut (impl IsStarkTranscript + Clone + Send), + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + ) -> Result, ProvingError> + where + FieldElement: AsBytes, + FieldElement: AsBytes, + PI: Send + Sync + Clone, + Field: Copy + 'static, + FieldExtension: Copy + 'static, + ::BaseType: SpillSafe, + ::BaseType: SpillSafe, + { + Self::multi_prove_with_provider( + air_trace_pairs, + transcript, + #[cfg(feature = "disk-spill")] + storage_mode, + None, + ) + } + + /// `multi_prove`, with the traces of some tables retired: `provider` rebuilds + /// them on demand. Passing `None` is byte-for-byte the resident path. + #[allow(clippy::too_many_arguments)] + fn multi_prove_with_provider( #[allow(unused_mut)] mut air_trace_pairs: Vec>, transcript: &mut (impl IsStarkTranscript + Clone + Send), #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + provider: Option<&(dyn TraceProvider + '_)>, ) -> Result, ProvingError> where FieldElement: AsBytes, @@ -3336,8 +4410,13 @@ pub trait IsStarkProver< let mut domains = Vec::with_capacity(num_airs); let mut twiddle_caches: Vec>> = Vec::with_capacity(num_airs); - for (air, trace, _pub_inputs) in &*air_trace_pairs { - let (domain, twiddles) = domain_and_twiddles(*air, trace.num_rows()); + for (idx, (air, trace, _pub_inputs)) in air_trace_pairs.iter().enumerate() { + // A retired table has no trace yet; its provider knows the shape. + let num_rows = match provider { + Some(p) if p.is_retired(idx) => p.num_rows(idx), + _ => trace.num_rows(), + }; + let (domain, twiddles) = domain_and_twiddles(*air, num_rows); domains.push(domain); twiddle_caches.push(twiddles); } @@ -3373,7 +4452,11 @@ pub trait IsStarkProver< .enumerate() .map(|(idx, (_, trace, _))| { let lde_size = domains[idx].interpolation_domain_size * domains[idx].blowup_factor; - estimate_table_vram_bytes(trace.num_main_columns, 0, lde_size) + let main_cols = match provider { + Some(p) if p.is_retired(idx) => p.num_main_columns(idx), + _ => trace.num_main_columns, + }; + estimate_table_vram_bytes(main_cols, 0, lde_size) }) .collect(); @@ -3410,6 +4493,10 @@ pub trait IsStarkProver< let mut main_commits: Vec> = Vec::with_capacity(num_airs); let mut main_ldes: Vec<(Vec>, usize)> = Vec::with_capacity(num_airs); + // Read once: the flag is process-global and must not change mid-proof, + // or a table would be rebuilt against a commitment it never produced. + #[cfg(not(feature = "cuda"))] + let retire_main_lde = streaming_retire_lde(); // Optional device-side LDE handle per table, populated only when the // R1 fused GPU pipeline produced one. Pairing is by index: this vector // is moved into the per-table `gpu_main_cells` mutex slots below, and @@ -3429,10 +4516,22 @@ pub trait IsStarkProver< &vram_gate, k, |idx| { - let (air, trace, _) = &air_trace_pairs[idx]; + let (air, resident_trace, _) = &air_trace_pairs[idx]; let domain = &domains[idx]; let twiddles = &twiddle_caches[idx]; + // Retired: build the trace just to commit it, and let it die at + // the end of this closure. The table's fused chain builds its own + // copy later — `build_main` is deterministic, so both agree. + let rebuilt; + let trace: &TraceTable = match provider { + Some(p) if p.is_retired(idx) => { + rebuilt = p.build_main(idx); + &rebuilt + } + _ => resident_trace, + }; + let precomputed = air .is_preprocessed() .then(|| (air.precomputed_commitment(), air.num_precomputed_columns())); @@ -3443,7 +4542,7 @@ pub trait IsStarkProver< let device_only = Self::device_only_for(*air, domain); Self::commit_main_trace( - *trace, + trace, domain, twiddles, precomputed, @@ -3465,6 +4564,16 @@ pub trait IsStarkProver< } transcript.append_bytes(&commit.root); main_commits.push(commit); + // Retire-LDE: drop the row-major main LDE here, keeping only its + // column count, so the O(N x main_cols x lde_size) cache never + // forms across this barrier. `rounds_stage` rebuilds each table's + // LDE inside its own fused chain. + #[cfg(not(feature = "cuda"))] + let cached_main = if retire_main_lde { + (Vec::new(), cached_main.1) + } else { + cached_main + }; main_ldes.push(cached_main); #[cfg(feature = "cuda")] main_gpu_handles.push(gpu_main); @@ -3612,6 +4721,16 @@ pub trait IsStarkProver< let domain = &domains[idx]; let twiddles = &twiddle_caches[idx]; + // Retired: rebuild into the cell, not into a local. The aux columns + // are written into this trace below and the LDE rebuild in + // `rounds_stage` reads it, so it has to outlive this stage; that + // stage drops it again once the table's proof is done. + if let Some(p) = provider + && p.is_retired(idx) + { + **trace = p.build_main(idx); + } + #[cfg(feature = "instruments")] let __sp = crate::instruments::span("r1_aux_build_table"); let bus_public_inputs = if air.has_aux_trace() { @@ -3912,9 +5031,9 @@ pub trait IsStarkProver< commitment: Round1Commitments, lde: Lde| -> Result, ProvingError> { - let pair = pair_cells[idx].lock().unwrap(); - let (air, trace, pub_inputs) = &*pair; - let _ = trace; // used by instruments + let mut pair = pair_cells[idx].lock().unwrap(); + let (air, trace, pub_inputs) = &mut *pair; + let _ = &trace; // used by instruments, and dropped below when retired let domain = &domains[idx]; #[cfg(feature = "instruments")] @@ -3922,6 +5041,20 @@ pub trait IsStarkProver< #[cfg(feature = "instruments")] let table_start = Instant::now(); + // Retire-LDE: the main LDE was dropped right after the Round 1 + // commit; rebuild it here so at most `k` of them are ever live. + #[cfg(not(feature = "cuda"))] + let lde = if retire_main_lde { + #[cfg(feature = "instruments")] + let __sp_rebuild = crate::instruments::span("r1_main_lde_rebuild"); + let main = Self::rebuild_main_lde(trace, domain, &twiddle_caches[idx]); + #[cfg(feature = "instruments")] + drop(__sp_rebuild); + Lde { main, ..lde } + } else { + lde + }; + let mut round_1_result = commitment.build_round1(lde, air.step_size(), domain.blowup_factor); @@ -3932,7 +5065,7 @@ pub trait IsStarkProver< let proof = Self::prove_rounds_2_to_4( *air, - *pub_inputs, + pub_inputs, &mut round_1_result, &mut *tguard, domain, @@ -3949,6 +5082,14 @@ pub trait IsStarkProver< sub_ops, )); } + // This table is proved: nothing reads its trace again, so a retired + // one goes back to being just its op lists. + if let Some(p) = provider + && p.is_retired(idx) + { + **trace = TraceTable::from_columns_main(Vec::new(), air.step_size()); + } + Ok(proof) }; @@ -4013,6 +5154,13 @@ pub trait IsStarkProver< for result in table_results { proofs.push(result.expect("run_admitted fills every slot")?); } + // Every table is proved and its transients are gone, so whatever is + // still held here is retained, not in flight. Read against the peak, + // this says how much of the peak a residency mode could ever reach. + #[cfg(feature = "instruments")] + if let Some(s) = crate::instruments::snap("After rounds 2-4") { + heap_snaps.push(s); + } #[cfg(feature = "instruments")] drop(__sp); #[cfg(feature = "instruments")] diff --git a/crypto/stark/src/tests/prove_verify_roundtrip_tests.rs b/crypto/stark/src/tests/prove_verify_roundtrip_tests.rs index a387df476..42ab4f74c 100644 --- a/crypto/stark/src/tests/prove_verify_roundtrip_tests.rs +++ b/crypto/stark/src/tests/prove_verify_roundtrip_tests.rs @@ -245,3 +245,127 @@ fn create_mul_air( EmptyConstraints, ) } + +/// THE retire-LDE correctness invariant: a proof produced with +/// `LAMBDA_STREAM_LDE=1` must be byte-identical to one produced with the flag +/// off. Retiring changes only *when* the main LDE exists — dropped after the +/// Round 1 commit, rebuilt from the same trace through the same production +/// row-major coset LDE inside the table's fused chain — never its contents. +/// A mismatch means the rebuild diverged from what was committed. +/// +/// Skipped under `cuda`: there `retire_leaves` returns `None` unconditionally +/// and `retire_main_lde` is compiled out, so both arms would take the resident +/// path and the assertion would compare a proof against itself. +/// +/// The env var is process-global. `cargo nextest`, which is what CI runs, gives +/// each test its own process; a plain `cargo test` does not, so under `make +/// test` the other proving tests in this binary may observe the flag while this +/// one holds it. That costs those tests the mode they meant to exercise, not +/// memory safety: `std::env`'s own `RwLock` serialises every Rust-side reader +/// against the write. The proper fix is this test's own integration binary, the +/// way `prover/tests/gpu_force_downgrade.rs` does it. +#[test] +#[cfg(not(feature = "cuda"))] +fn retire_lde_proof_is_byte_identical() { + fn prove_once() -> Vec { + let add_column = vec![ + FE::one(), + FE::zero(), + FE::one(), + FE::zero(), + FE::one(), + FE::one(), + FE::zero(), + FE::zero(), + ]; + let mul_column = vec![ + FE::zero(), + FE::one(), + FE::zero(), + FE::one(), + FE::zero(), + FE::zero(), + FE::one(), + FE::one(), + ]; + let a_column = (1..=8u64).map(FE::from).collect::>(); + let b_column = (1..=8u64).map(|i| FE::from(i * 10)).collect::>(); + let c_column = vec![ + FE::from(11), + FE::from(40), + FE::from(33), + FE::from(160), + FE::from(55), + FE::from(66), + FE::from(490), + FE::from(640), + ]; + let mut cpu_trace = crate::trace::TraceTable::from_columns_main( + vec![add_column, mul_column, a_column, b_column, c_column], + 1, + ); + + let add_a = vec![FE::from(1), FE::from(3), FE::from(5), FE::from(6)]; + let add_b = vec![FE::from(10), FE::from(30), FE::from(50), FE::from(60)]; + let add_c = vec![FE::from(11), FE::from(33), FE::from(55), FE::from(66)]; + let add_m = vec![FE::one(), FE::one(), FE::one(), FE::one()]; + let mut add_trace = + crate::trace::TraceTable::from_columns_main(vec![add_a, add_b, add_c, add_m], 1); + + let mul_a = vec![FE::from(2), FE::from(4), FE::from(7), FE::from(8)]; + let mul_b = vec![FE::from(20), FE::from(40), FE::from(70), FE::from(80)]; + let mul_c = vec![FE::from(40), FE::from(160), FE::from(490), FE::from(640)]; + let mul_m = vec![FE::one(), FE::one(), FE::one(), FE::one()]; + let mut mul_trace = + crate::trace::TraceTable::from_columns_main(vec![mul_a, mul_b, mul_c, mul_m], 1); + + let proof_options = ProofOptions::default_test_options(); + let cpu_air = create_cpu_air(&proof_options); + let add_air = create_add_air(&proof_options); + let mul_air = create_mul_air(&proof_options); + + #[allow(clippy::type_complexity)] + let air_trace_pairs: Vec<( + &dyn AIR, + &mut crate::trace::TraceTable, + &(), + )> = vec![ + (&cpu_air, &mut cpu_trace, &()), + (&add_air, &mut add_trace, &()), + (&mul_air, &mut mul_trace, &()), + ]; + + let proofs = + multi_prove_ram(air_trace_pairs, &mut DefaultTranscript::::new(&[])).unwrap(); + serde_cbor::to_vec(&proofs).expect("serialize proofs") + } + + let prev = std::env::var("LAMBDA_STREAM_LDE").ok(); + + // SAFETY: the only writer in this binary, and every reader of it goes + // through `std::env`, which serialises reads against writes on its own + // lock. Restored below. + unsafe { std::env::set_var("LAMBDA_STREAM_LDE", "0") }; + assert!( + !crate::prover::streaming_retire_lde(), + "the flag did not take: this test would compare a proof against itself" + ); + let resident = prove_once(); + + unsafe { std::env::set_var("LAMBDA_STREAM_LDE", "1") }; + assert!( + crate::prover::streaming_retire_lde(), + "the flag did not take: this test would compare a proof against itself" + ); + let retired = prove_once(); + + match prev { + Some(v) => unsafe { std::env::set_var("LAMBDA_STREAM_LDE", v) }, + None => unsafe { std::env::remove_var("LAMBDA_STREAM_LDE") }, + } + + assert_eq!( + resident, retired, + "retire-LDE proof must be byte-identical to the resident-LDE proof" + ); +} diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index 44add9c21..94694119c 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -84,6 +84,16 @@ where pub grinding_seed: [u8; 32], } +/// The challenges of rounds 2 and 3 alone: what a table's fork yields before +/// any FRI, which is where a batched proof parts ways with a per-table one. +pub struct RoundsChallenges { + pub z: FieldElement, + pub boundary_coeffs: Vec>, + pub transition_coeffs: Vec>, + pub trace_term_coeffs: Vec>>, + pub gammas: Vec>, +} + pub type DeepPolynomialEvaluations = (Vec>, Vec>); /// Deep-composition sums that are identical across all FRI queries of a @@ -1445,16 +1455,21 @@ pub trait IsStarkVerifier< } /// Replays rounds 2, 3 and 4 of the protocol for a given proof, assuming round 1 has - /// already been replayed and the RAP challenges are known. - fn replay_rounds_after_round_1( + /// already been replayed and the RAP challenges are known: the constraint + /// coefficients, the out-of-domain point, and round 4's DEEP coefficients. + /// + /// Stops where the two proof formats part company. The per-table proof goes + /// on to its own FRI from here; the batched proof samples its fold + /// coefficient instead. + fn replay_rounds_2_to_4( air: &dyn AIR, proof: StarkProofView<'_, Field, FieldExtension, PI>, public_inputs: &PI, domain: &VerifierDomain, transcript: &mut impl IsStarkTranscript, - rap_challenges: Vec>, + rap_challenges: &[FieldElement], layout: &crate::ood::OodLayout, - ) -> Challenges + ) -> RoundsChallenges where FieldElement: AsBytes, FieldElement: AsBytes, @@ -1475,7 +1490,7 @@ pub trait IsStarkVerifier< let num_boundary_constraints = air .boundary_constraints( public_inputs, - &rap_challenges, + rap_challenges, bus_public_inputs.as_ref(), trace_length, ) @@ -1549,6 +1564,43 @@ pub trait IsStarkVerifier< let gammas = deep_composition_coefficients; // FRI commit phase + RoundsChallenges { + z, + boundary_coeffs, + transition_coeffs, + trace_term_coeffs, + gammas, + } + } + + fn replay_rounds_after_round_1( + air: &dyn AIR, + proof: StarkProofView<'_, Field, FieldExtension, PI>, + public_inputs: &PI, + domain: &VerifierDomain, + transcript: &mut impl IsStarkTranscript, + rap_challenges: Vec>, + layout: &crate::ood::OodLayout, + ) -> Challenges + where + FieldElement: AsBytes, + FieldElement: AsBytes, + { + let RoundsChallenges { + z, + boundary_coeffs, + transition_coeffs, + trace_term_coeffs, + gammas, + } = Self::replay_rounds_2_to_4( + air, + proof, + public_inputs, + domain, + transcript, + &rap_challenges, + layout, + ); let merkle_roots = proof.fri_layers_merkle_roots(); let mut zetas = merkle_roots .iter() diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8ba066462..94d7501b7 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -18,6 +18,7 @@ - [Lookup argument](./cryptography/lookup.md) - [Virtual machine](./virtual_machine/introduction.md) - [Continuations design](./continuations_design.md) +- [Prove-and-retire design](./prove_and_retire_design.md) ## Getting started diff --git a/docs/prove_and_retire_design.md b/docs/prove_and_retire_design.md new file mode 100644 index 000000000..0b2b111a3 --- /dev/null +++ b/docs/prove_and_retire_design.md @@ -0,0 +1,260 @@ +# Prove-and-retire prover (Approach 1) — how it works and how to run it + +This is the design and usage document for the prove-and-retire prover — +Approach 1 of the streaming spec (`spec/streaming.typ` at `624998db`), the +sibling of [Continuations design](./continuations_design.md), which is the +spec's Approach 2 ("prove-epoch"). Both are "streaming" in the spec's sense of +bounding the prover's memory; this one does it by re-walking the execution and +retiring tables, continuations by splitting it into epochs. It +covers what the prover does in each of its passes, the two proof formats it can +produce and when each is the right one, every knob, what verifies with what, +the numbers measured on a mainnet ethrex block, where the code lives, and the +correctness rule that every future change to the walk has to respect. + +It is written to be read by a human picking this up cold. + +## 1. The problem and the idea + +A proof of an execution is a LogUp over many tables — the CPU, the memory +tables, the ALU chips, the preprocessed tables (BITWISE, DECODE), the +accelerators, one table per page. The monolithic prover builds every trace, +commits every one (LDE + Merkle), samples **one** LogUp challenge `(z, α)` that +ties all tables together through the bus, and only then proves each table. Until +every round-1 root exists nothing can be dropped, so the peak is the sum of all +tables: **108 GB** for a mainnet ethrex block of 30.5 M cycles. + +Approach 1 accepts redoing work in exchange for not retaining it. The execution +is **walked several times**; each pass proves what it can from a table and +drops the table. What crosses from one pass to the next is small: roots, +challenges, an accumulated codeword. + +## 2. The passes + +"Walk" means re-executing the program and rebuilding the tables chunk by chunk +(`prover/src/pass.rs`, `trace_builder::walk_and_emit_chunks`). Tables fall into three +groups, not two. Most chunked tables (CPU, MEMW, MEMW_A, MEMW_R, LOAD, CPU32, +BRANCH, EQ, BYTEWISE, STORE — the list is `CHUNKED_KINDS`) are handed to the +pass's visitor as soon as a chunk fills. Four more — **LT, MUL, DVRM and +SHIFT** — are chunked tables in the finished proof but are *not* handed over +during the walk: later derivations keep appending to them (MEMW and HINT feed +LT, DVRM feeds LT and MUL, CPU32 feeds SHIFT, MUL and DVRM), and the finished +run concatenates each source whole, so closing them early would cut their +chunks somewhere the monolithic build does not — which would move their roots +and cost the byte-identical property of §3. Their op lists are held to the end +and chunked in `pass::finish`. The tables that cannot be chunked at all — the +preprocessed ones, the accumulators (KECCAK, ECSM, …), REGISTER, HALT, one PAGE +per page — are the *residents*, handed over when the walk ends. + +So what the walk holds is one chunk per table of the first group, plus the +residents, plus the op lists of the second group and of BITWISE. The last term +grows with the execution rather than with `k`: on the ethrex block it is a low +single-digit percentage of the peak, but a workload that sends many wide memory +accesses down the general MEMW path (up to eight LT ops each, against one for +the aligned path) grows it considerably faster. §10 lists it as the known +asymptotic gap. + +| pass | what it does | keeps | drops | +|---|---|---|---| +| **1 Commit** (walk) | per chunk: trace → LDE main → Merkle → root, in pipelined batches of *k* tables; the walk never waits for a batch | 245 roots, the resident traces | traces, LDEs, trees | +| **2 Challenge** | commits the residents (in parallel), absorbs the statement and every root in AIR order, samples **one `(z, α)`**, builds the AIRs once | `(z, α)`, transcript, AIRs | — | +| **3 LogUp** (walk) | per table: aux (LogUp), rounds 2–3 (β, z_ood, γ), composition. **Per-table variant**: continues with its own FRI and openings → one `StarkProof` per table. **Batched variant**: stops at the DEEP codeword | per-table proofs, or DEEP codewords folded per height | everything else | +| **4 Fold** (batched, inside pass 3) | each DEEP codeword is multiplied by a coefficient drawn from the shared transcript and added to the accumulator of its height; then one FRI + grinding + query indices per height group | 13 group FRIs (ethrex), the fold order | codewords | +| **5 Open** (walk, batched) | the query indices of a group are only known after its FRI, and its tables are gone: each table is **rebuilt** (round 1, and round 2 unless `A1_KEEP_COMPOSITION`) and opened at its group's indices | openings → `BatchedProof` | — | + +The residents are proved / folded / opened as one parallel batch after each +walk; one at a time they left the machine idle. + +## 3. The two variants + +### Per-table (`--through logup`) + +Ends at pass 3 and emits the same `MultiProof` the monolithic prover emits — +on ethrex, 245 tables with roots **byte for byte identical** to the monolithic +proof — so the existing verifier (`prover::verify`) checks it unchanged. This is +the drop-in: same proof, same verifier, one fifth of the memory, 1.27× the time. + +### Batched (`--through batched`) + +Continues with passes 4 and 5. Folding every DEEP codeword of one height into a +single codeword leaves **13 FRIs instead of 245** on ethrex. It is a new proof +format (`logup_phase::BatchedProof`) with its own verifier +(`prover::batched_verifier::verify`, §5). It costs 1.96× the monolithic time +(1.75× with the knob below) and buys a proof that is 57% smaller in the same +encoding and verifies in half the time with 40% fewer hashes — which is what +recursion pays for. Choose it when the proof will be verified inside a guest. + +### `A1_KEEP_COMPOSITION=1` (batched only) + +Pass 5 rebuilds each table to open it; the constraint evaluation (round 2) is +the costliest part of that rebuild and its output — the composition parts over +the LDE domain — was already computed in pass 3-4 and dropped. With the knob +the fold pass keeps them and pass 5 only rebuilds round 1 and re-commits the +kept parts: **−24 s for +8.6 GB** on ethrex. Off by default because it is the +memory-for-time trade the approach otherwise avoids. + +## 4. Running it + +```sh +# The Rust ELFs in the repo may predate a syscall the branch decodes: rebuild them. +SYSROOT_DIR=$HOME/.lambda-vm-sysroot make compile-programs-rust +cargo build --release -p cli --features jemalloc-stats # jemalloc-stats prints the peak heap + +E=executor/program_artifacts/rust/ethrex.elf +I=executor/tests/ethrex_mainnet_25368371.bin + +# Per-table: the monolithic proof format, verified with the existing verifier. +./target/release/cli trace-build $E --private-input $I --prove-and-retire --through logup --verify + +# Batched: one FRI per height, verified with the batched verifier. +./target/release/cli trace-build $E --private-input $I --prove-and-retire --through batched --verify +A1_KEEP_COMPOSITION=1 ./target/release/cli trace-build $E --private-input $I --prove-and-retire --through batched --verify + +# Stages, for measuring one pass at a time: walk | commit | challenge | logup | batched +# --output writes the per-table proof (the batched one is not serialized yet). +``` + +Each run prints the pass timings, `Trace build (prove-and-retire): N tables, T s`, +`Peak heap: M MB` and, with `--verify`, `A1 proof verifies: 245 tables` or +`Batched proof verifies: 245 tables in 13 groups`. + +| knob | default | what it does | +|---|---|---| +| `A1_TABLE_PARALLELISM` | cores / 6, at most 16 | tables in flight per batch. The sweep recorded on `pass::table_parallelism` is flat from 1 to 16 (21550 MB) and steps at 32 (26640 MB, +5.0 GB) for 1% of time | +| `A1_PIPELINE=0` | pipelined | run each batch inline instead of on the consumer thread (diagnostic) | +| `A1_KEEP_COMPOSITION=1` | off | keep the composition parts for the Open pass (§3) | +| `A1_INFLIGHT` | 0 | full batches allowed to wait in the channel beyond the one being processed | +| `LAMBDA_STREAM_LDE` | off | `1`/`true` retires each table's main LDE after the Round 1 commit and rebuilds it on demand, and frees the leaf half of every committed Merkle tree; `auto` decides from the peak-RAM estimate (needs `disk-spill`). Off by default, and it changes this approach's memory profile too — §6's numbers were taken with it off | +| `_RJEM_MALLOC_CONF` | — | jemalloc options for experiments; the CLI's own setting is §6 | + +Verification on its own: `cli verify ` for a per-table proof +written with `--output`. The `hash-metrics` feature that prints a verify's +keccak count comes from #987, which is **not on this branch** — the verify-hash +column in §6 was measured on a tree that has it, and cannot be reproduced here +until this branch is rebased onto it. + +## 5. What verifies, and with what + +| variant | verifier | evidence | +|---|---|---| +| per-table | `prover::verify` (unchanged) | ethrex: "A1 proof verifies: 245 tables"; tables byte-identical to the monolithic proof (`examples/cmp_proofs.rs`) | +| batched | `prover::batched_verifier::verify` + `stark::batched_verifier::verify_batched` | ethrex: "Batched proof verifies: 245 tables in 13 groups"; `a_tampered_batched_proof_is_rejected` | + +The batched verifier replays the transcript from the proof — statement, roots +in AIR order, `(z, α)`, the fold coefficients in the order the proof records, +per group the FRI challenges, grinding and query indices — then, per table, +runs the ordinary verifier's steps on a view of the table's data with the FRI +left empty (out-of-domain consistency, authentication of the openings at the +group's indices, reconstruction of its DEEP value there), and per group sums +those values with the coefficients and verifies the group's FRI from that first +layer down to the final polynomial. On the VM side it rebuilds the AIRs from the +layout the proof declares, checks the preprocessed roots against the AIRs' +constants and the LogUp bus balance against the public output. Its tamper test +changes one thing at a time — an out-of-domain value, an opening, the fold +order, a final-polynomial coefficient, a query index, the public output, the +layout — and requires each to be rejected while the untouched proof passes. + +Still to be reviewed by someone who did not write it: the soundness of the +fold coefficient (sampled per table from the shared seed after absorbing that +table's round-3 data) and of the absorption order. + +## 6. Numbers (ethrex block 25368371, 30.5 M cycles, 96 cores, blowup 2) + +Taken with `LAMBDA_STREAM_LDE` off and `A1_KEEP_COMPOSITION` off except where +the row names it. `trace-build` hard-codes blowup 2 and has no `--blowup` +flag, so the batched path cannot yet be measured at the blowup-4 posture. + +| | peak heap | prove | vs monolithic | proof | verify | verify hashes | +|---|---|---|---|---|---|---| +| monolithic | 107.7 GB | 88.1 s | 1× | 437 MB (838 CBOR) | 6.0 s | 19.8 M | +| **A1 per-table** | **22.9 GB** | **112 s** | **1.27×** | identical | 6.0 s | 19.8 M | +| A1 batched | 31.3 GB | 173 s | 1.96× | 360 MB CBOR (−57%) | 3.1 s | **11.9 M** | +| A1 batched + `KEEP_COMPOSITION` | 39.0 GB | 154 s | 1.75× | same | 3.0 s | 11.9 M | +| continuations 2^22 (CI bench) | 46.6 GB | 110.6 s | 1.26× | 727 MB | 11.5 s | 33.1 M | +| continuations 2^21 | 29.0 GB | 120.7 s | 1.37× | 1 045 MB | 17.8 s | 51.9 M | +| continuations 2^20 | 18.7 GB | 142.2 s | 1.61× | 1 671 MB | 30.4 s | 90.5 M | + +Verify hashes are the keccak-256 finalizes the verifier does (grinding +excluded), the proxy for the recursion guest's cost. Every epoch of a +continuation carries its own fixed tables and its own FRIs, hence 1.7×–4.6× the +hashes of a single proof; the batched proof needs 0.36× of the CI bench's. + +Where the time goes: prove-and-retire per-table = monolithic − trace build + round 1 twice + +pass 2 + pipeline edges, almost to the second; the profile and the core +utilization are the monolithic prover's. What is left without a protocol +change: chunking KECCAK_RND (the 3.8 s of pass 2). + +Two things that were not the approach but decided its speed: jemalloc purges +every extent of 8 MiB or more the moment it is freed (`extent.c`, +`extent_may_force_decay`), which made every chunk re-fault its pages — the CLI +disables that arena's decay and purges it every 10 s +(`keep_large_buffers_warm`, Linux only, −13%); and the resident tables were +processed one at a time after each walk (−10% once batched). + +## 7. Against the spec + +- *Commit*, *Challenge*, *LogUp re-execution*, *FRI*, *Open*: as written. +- The spec has the Commit phase already accumulating "FRI polynomials"; nothing + FRI-able exists before the LogUp challenge (aux and composition need `(z, α)`), + so batching starts in the re-execution pass. +- One batch polynomial **per height**, not one in total: the fold squares the + coset offset each layer, so codewords of different lengths do not line up + without a mixed-height commitment (#951's direction). +- Holding each table's whole Merkle tree from the fold pass to the Open pass + measured +9 GB for −4 s and was not kept; keeping the composition parts + (`A1_KEEP_COMPOSITION`) is the trade that pays. The spec's Open optimization + proper — keep the internal nodes, drop the leaves — *is* implemented + (`MerkleTree::drop_leaves`, `get_proof_by_pos_with_leaf_sibling`, + `TableCommit::retire_leaves`) and ships behind `LAMBDA_STREAM_LDE`. +- The per-table variant is not in the spec; it is what makes the approach a + drop-in. Distribution across workers is not attempted; the pipelined batch + worker is the seam for it. + +## 8. Code map + +``` +prover/src/pass.rs Visitor (one table per call), Batched (channel → consumer thread, + batches of k), Resident, the walk driver (walk / finish) +prover/src/commit_phase.rs pass 1; Precomputed (the ELF-only preprocessed commitments, beside the walk) +prover/src/challenge_phase.rs pass 2: residents in parallel, roots in AIR order, (z, α), the AIRs +prover/src/logup_phase.rs run (per-table) · run_batched (3-4) · run_open (5) · assemble_vm_proof · + assemble_batched_proof · BatchedProof +prover/src/batched_verifier.rs the VM half of the batched verifier +crypto/stark/src/batched_verifier.rs the STARK half: rounds 2-3 per table, openings, accumulated DEEP, + one FRI per group +crypto/stark/src/prover.rs round_1_from_trace, rounds_2_and_3, deep_for_table, fold_coefficient, + batch_fri, open_for_table / open_for_table_kept +prover/src/streaming.rs AirOrder (the AIR order every pass and the verifier agree on) +prover/src/tables/trace_builder.rs walk_and_emit_chunks, WalkLeftover::finalize (what the shared tables owe) +bin/cli/src/main.rs trace-build --prove-and-retire, keep_large_buffers_warm; examples/cmp_proofs.rs +``` + +## 9. The rule every change to the walk must respect + +Several tables are fed by others: LT gets a row for every MEMW timestamp check, +for every DVRM `|r| < |d|` and for every HINT range check; BITWISE gets the +lookups of every chip including CPU32; MUL and DVRM get the CPU's derived ops +and the CPU32 dispatch; SHIFT gets the CPU32 dispatch. The monolithic build +derives all of that from complete op lists. The walk retires chunks before +those lists are complete, so **whatever a retired chunk owes another table has +to be derived when the chunk closes**, and the tail's share in +`WalkLeftover::finalize`, in the monolithic build's order. Five such gaps kept +the ethrex proof from verifying while every small-program test passed; two +tests now pin the pattern — `bitwise_multiplicities_match_the_ordinary_build` +(cell-by-cell diff of the walk's BITWISE against the ordinary build) and +`a1_verifies_with_many_chunks` (small chunks of every kind on a Rust program +with memory and ALU, proved and verified) — and `--verify` on a real block is +the last word. When adding a table or a derived lookup, `grep` every producer +of it in the monolithic build and check the walk has each. + +## 10. Not done + +- Chunking KECCAK_RND (protocol change; the remaining ~4 s of pass 2). +- Serializing `BatchedProof` to disk (`--output` covers the per-table proof). +- Distributing retirement batches across workers. +- An independent soundness review of the batched fold (§5). +- Bounding the op lists the walk holds whole (§2): LT, MUL, DVRM and SHIFT + are excluded from `CHUNKED_KINDS` because their chunk boundaries are not + knowable until the run ends, so residency is O(cycles) rather than O(chunk) + for that term. BITWISE's lookup list is the cheaper half of the same problem + and has no ordering constraint — a histogram is commutative, so it could be + folded per segment without moving any root. diff --git a/executor/src/tests/checkpoint_tests.rs b/executor/src/tests/checkpoint_tests.rs new file mode 100644 index 000000000..3cac51b85 --- /dev/null +++ b/executor/src/tests/checkpoint_tests.rs @@ -0,0 +1,67 @@ +//! Executor checkpoints: snapshot the VM mid-execution, rebuild an `Executor` +//! from it, and resume — the concatenated logs must equal a straight run's. +//! +//! That equality is the property everything built on re-execution rests on: a +//! prover may drop what it produced only if it can get exactly that back. +//! +//! The program is built by hand rather than loaded from an ELF fixture, so the +//! test is hermetic: 100_005 `ADDI x5, x5, 1` followed by `JALR x0, 0(x0)`, +//! which jumps to address 0 and halts. No syscalls, and the instruction count +//! is over 100_000 on purpose — the snapshot then lands mid-execution, across a +//! `resume()` chunk boundary, instead of at a point the chunking makes easy. + +use crate::elf::{Elf, Segment}; +use crate::vm::execution::Executor; + +const ADDI_X5_X5_1: u32 = 0x0012_8293; // addi x5, x5, 1 +const JALR_X0_0_X0: u32 = 0x0000_0067; // jalr x0, 0(x0) -> pc = 0 -> halt +const N_ADDI: usize = 100_005; +const BASE: u64 = 0x1000; + +fn long_program() -> Elf { + let mut values = vec![ADDI_X5_X5_1; N_ADDI]; + values.push(JALR_X0_0_X0); + Elf { + entry_point: BASE, + data: vec![Segment { + base_addr: BASE, + values, + is_executable: true, + }], + } +} + +#[test] +fn snapshot_resume_produces_identical_logs() { + let elf = long_program(); + + let full = Executor::new(&elf, vec![]).unwrap().run().unwrap().logs; + assert_eq!(full.len(), N_ADDI + 1, "every instruction should log once"); + + // One chunk, then snapshot: the cut lands mid-execution. + let mut exec = Executor::new(&elf, vec![]).unwrap(); + let mut logs = Vec::new(); + { + let chunk0 = exec.resume().unwrap().expect("at least one chunk"); + logs.extend_from_slice(chunk0); + } + assert!( + logs.len() < full.len(), + "the snapshot must be taken before the program ends (got {} of {})", + logs.len(), + full.len() + ); + + let snapshot = exec.snapshot(); + let mut resumed = Executor::from_snapshot(&elf, snapshot).expect("recreate from snapshot"); + while let Some(chunk) = resumed.resume().unwrap() { + logs.extend_from_slice(chunk); + } + + assert_eq!( + logs.len(), + full.len(), + "log count differs after snapshot + resume" + ); + assert_eq!(logs, full, "resumed logs must equal the straight run's"); +} diff --git a/executor/src/tests/mod.rs b/executor/src/tests/mod.rs index 244447b22..3dbb161cf 100644 --- a/executor/src/tests/mod.rs +++ b/executor/src/tests/mod.rs @@ -1,3 +1,4 @@ +pub mod checkpoint_tests; pub mod ecsm_tests; pub mod flamegraph_tests; pub mod hint_tests; diff --git a/executor/src/vm/execution.rs b/executor/src/vm/execution.rs index dc0660178..27a6c9788 100644 --- a/executor/src/vm/execution.rs +++ b/executor/src/vm/execution.rs @@ -41,6 +41,24 @@ pub struct EpochExecution { pub end_memory: Memory, } +/// The mutable VM state at a cycle boundary: enough, with the program ELF, to +/// recreate an [`Executor`] that resumes byte-identically. +/// +/// The instruction cache is rebuilt from the ELF rather than stored. Replay is +/// deterministic because every nondeterministic input (the private inputs) is +/// already loaded into `memory` before the first cycle, so a resumed run +/// produces the same logs the original would have. +/// +/// This is what lets a prover drop what it built and get it back: re-execution +/// from a checkpoint is bounded by the distance to the next boundary instead of +/// restarting at cycle zero. +#[derive(Clone)] +pub struct VmSnapshot { + memory: Memory, + registers: Registers, + pc: u64, +} + /// Executor state for chunked execution pub struct Executor { memory: Memory, @@ -66,6 +84,33 @@ impl Executor { }) } + /// Capture the VM state as a [`VmSnapshot`]. Cheap except for the memory + /// clone, which copies the touched-cell map. + pub fn snapshot(&self) -> VmSnapshot { + VmSnapshot { + memory: self.memory.clone(), + registers: self.registers.clone(), + pc: self.pc, + } + } + + /// Recreate an `Executor` sitting exactly where `snapshot` was taken. + /// + /// `program` must be the ELF the snapshot was taken under: it rebuilds the + /// instruction cache. The program image is NOT reloaded — the snapshot's + /// memory already carries it, along with everything execution has written + /// since. + pub fn from_snapshot(program: &Elf, snapshot: VmSnapshot) -> Result { + let instructions = InstructionCache::new(&program.data)?; + Ok(Self { + memory: snapshot.memory, + registers: snapshot.registers, + pc: snapshot.pc, + instructions, + logs: Vec::with_capacity(CHUNK_SIZE), + }) + } + /// Resume execution and return next logs. Returns None when program is finished. pub fn resume(&mut self) -> Result, ExecutorError> { self.resume_with_limit(CHUNK_SIZE) diff --git a/executor/src/vm/logs.rs b/executor/src/vm/logs.rs index de6b73d0b..a5aa426b0 100644 --- a/executor/src/vm/logs.rs +++ b/executor/src/vm/logs.rs @@ -11,7 +11,7 @@ /// - `src1_val` = syscall number (from x17): 64=Commit, 93=Halt, etc. /// - `src2_val` = buf_addr (x11) for Commit, 0 otherwise /// - `dst_val` = count (x12) for Commit, 0 otherwise -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Log { /// PC before instruction execution (use this to look up the instruction) pub current_pc: u64, diff --git a/prover/src/auto_storage.rs b/prover/src/auto_storage.rs index 8cc437edd..1805fdc27 100644 --- a/prover/src/auto_storage.rs +++ b/prover/src/auto_storage.rs @@ -233,6 +233,29 @@ pub fn decide(lengths: &TableLengths, blowup_factor: u8) -> StorageMode { mode } +/// Whether to retire each table's main LDE after the Round 1 commit, from the +/// same analytical estimate that picks the storage mode. +/// +/// Policy: retire exactly when the estimate does not fit under the safety +/// threshold — the regime where the prover is about to swap or die, and where +/// trading ~11 % of prove time for the N-wide main-LDE term is the trade you +/// want. Below the threshold the term is affordable and the time is not worth +/// paying. Resolves `LAMBDA_STREAM_LDE=auto`; an explicit `0`/`1` overrides it. +pub fn decide_retire_lde(lengths: &TableLengths, blowup_factor: u8) -> bool { + let estimated = peak_bytes(lengths, blowup_factor, storage_estimate_parallelism()); + let retire = retire_lde_for(estimated, available_ram_bytes()); + log::info!("estimated_peak_bytes: {estimated}, retire_lde: {retire}"); + retire +} + +/// The policy itself, over an explicit estimate and available RAM: retire on +/// exactly the inputs that pick `Disk`, so the two memory levers share one +/// trigger and one safety margin. Unknown available RAM retires, matching the +/// storage mode's conservative default. +pub(crate) fn retire_lde_for(estimated: u64, available: Option) -> bool { + select_storage_mode(estimated, available) == StorageMode::Disk +} + /// Peak RAM estimate in bytes for a proof whose trace shape matches `lengths`. /// /// `table_parallelism` is how many tables' rounds 2-4 transients this assumes diff --git a/prover/src/batched_proof.rs b/prover/src/batched_proof.rs new file mode 100644 index 000000000..b83f0a3c5 --- /dev/null +++ b/prover/src/batched_proof.rs @@ -0,0 +1,60 @@ +//! The batched proof, as a value. +//! +//! Kept apart from the passes that build it so the recursion guest, which +//! compiles the prover without `parallel`, can carry and verify one. + +use math::field::element::FieldElement; + +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +/// One table's DEEP openings at its group's query indices. +pub type Open = stark::proof::stark::DeepPolynomialOpenings; + +/// A table's half of a batched proof: everything it contributes that is not a +/// FRI, which is now its group's business. +/// +/// This is what the per-table `StarkProof` keeps once the layers, the final +/// polynomial, the queries and the nonce move to the group — the 57.9% of the +/// proof that stops being paid once per table. +#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct TablePublic { + pub trace_rows: usize, + pub main_root: stark::config::Commitment, + pub precomputed_root: Option, + pub aux_root: Option, + pub composition_poly_root: stark::config::Commitment, + pub trace_ood: stark::table::Table, + pub trace_ood_next: stark::table::Table, + pub parts_ood: Vec>, + pub bus_public_inputs: Option>, +} + +/// A batched proof: what the five passes produce, assembled. +/// +/// Additive, not a replacement. `StarkProof` and `multi_verify` are untouched +/// and still produce byte-identical proofs; this is a second format alongside +/// them, for the path that folds one FRI per domain instead of one per table. +/// +/// The split is the whole point. A table keeps what only it can answer for — +/// its roots, its out-of-domain values, its openings — and a group carries the +/// FRI those tables share. That is the 57.9% of a per-table proof that stops +/// being paid 227 times. +#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct BatchedProof { + /// Per table, in AIR order. + pub tables: Vec, + /// The chunk layout the tables follow; the verifier rebuilds the AIRs from it. + pub table_counts: crate::TableCounts, + /// Per table, in AIR order: its rows at its group's indices. + pub openings: Vec, + /// Which group each table belongs to. + pub group_of: Vec, + /// The AIR indices in the order they were folded, which the verifier + /// replays because a table's coefficient depends on every table before it. + pub fold_order: Vec, + /// Per group, in ascending domain: the FRI they share. + pub groups: Vec<(usize, stark::prover::GroupFri)>, + /// The statement, which the verifier binds before absorbing any root. + pub public_output: Vec, + pub page_configs: Vec, +} diff --git a/prover/src/batched_verifier.rs b/prover/src/batched_verifier.rs new file mode 100644 index 000000000..a07987c96 --- /dev/null +++ b/prover/src/batched_verifier.rs @@ -0,0 +1,327 @@ +//! The batched proof's verifier. +//! +//! Alongside `multi_verify`, never in place of it: the per-table path is +//! untouched and its proofs are byte-identical to what they always were. +//! +//! What a verifier decides is not "did the prover follow its own steps" — it is +//! whether a proof it has never seen a prover produce is valid. That is built in +//! pieces, and this is the first: the transcript replay, which derives every +//! challenge from the proof alone. Nothing below it can be checked until the +//! challenges are the prover's, and if they are, a forged proof has to be wrong +//! about something the later pieces test rather than about which questions were +//! asked. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use log::error; +use math::field::element::FieldElement; +use stark::batched_verifier::BatchedGroup; +use stark::proof::options::ProofOptions; +use stark::proof::stark::StarkProof; + +use crate::Error; +use crate::batched_proof::BatchedProof; +use crate::tables::trace_builder::Traces; +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +/// Every challenge a batched proof's verification needs, derived from the proof. +pub struct Replay { + /// The one challenge the whole execution shares. + pub logup: Vec>, + /// Per table, in AIR order: its fold coefficient. + pub coefficients: Vec>, +} + +/// Replay the transcript the prover walked, from the proof. +/// +/// `elf_bytes` and the statement come from outside the proof on purpose: a +/// proof that could choose its own statement would prove nothing. +/// +/// # This is a test oracle, not a verifier +/// +/// It derives the challenges and stops. [`verify`] does not call it — it walks +/// the same prefix itself and is the stricter of the two: it checks each +/// preprocessed root against the AIR's own constant where this takes the +/// prover's word, and it rejects a `fold_order` that is not a permutation. +/// Promoting this function to a verification path without closing both gaps +/// would be a soundness hole; it exists so a test can compare challenges. +pub fn replay( + proof: &BatchedProof, + elf_bytes: &[u8], + table_counts: &crate::TableCounts, + proof_options: &stark::proof::options::ProofOptions, +) -> Result { + let mut transcript = DefaultTranscript::::new(&[]); + crate::statement::absorb_statement( + &mut transcript, + crate::statement::StatementKind::Monolithic, + elf_bytes, + &proof.public_output, + table_counts, + proof + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(), + &crate::tables::trace_builder::runtime_page_ranges(&proof.page_configs), + proof_options.fri_final_poly_log_degree, + ); + + // Round 1, in AIR order: a preprocessed table's precomputed root first. + for t in proof.tables.iter() { + if let Some(ref pre) = t.precomputed_root { + transcript.append_bytes(pre); + } + transcript.append_bytes(&t.main_root); + } + let logup: Vec<_> = (0..stark::lookup::LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect(); + + // The fold coefficients, in the order the prover folded — which the proof + // carries because a table's coefficient depends on every table before it. + if proof.fold_order.len() != proof.tables.len() { + return Err(Error::Prover(format!( + "batched verify: {} tables folded of {}", + proof.fold_order.len(), + proof.tables.len() + ))); + } + // Shapes before the seed reads a value, for the same reason + // `verify_batched` pins them before its own fold loop: `Table::columns` + // indexes at the advertised dimensions. + for (idx, t) in proof.tables.iter().enumerate() { + for block in [&t.trace_ood, &t.trace_ood_next] { + if !block.dimensions_consistent() { + return Err(Error::Prover(format!( + "batched verify: table {idx}'s out-of-domain block is malformed" + ))); + } + } + } + let mut seed = transcript.clone(); + let mut coefficients = vec![FieldElement::::zero(); proof.tables.len()]; + for &idx in proof.fold_order.iter() { + let t = proof.tables.get(idx).ok_or_else(|| { + Error::Prover(format!("batched verify: fold order names table {idx}")) + })?; + if let Some(ref bpi) = t.bus_public_inputs { + seed.append_field_element(&bpi.table_contribution); + } + seed.append_bytes(&t.composition_poly_root); + let blocks: [&stark::table::Table; 2] = + [&t.trace_ood, &t.trace_ood_next]; + for block in blocks { + for col in block.columns().iter() { + for elem in col.iter() { + seed.append_field_element(elem); + } + } + } + for elem in t.parts_ood.iter() { + seed.append_field_element(elem); + } + coefficients[idx] = seed.sample_field_element(); + } + + Ok(Replay { + logup, + coefficients, + }) +} + +/// Verify a batched proof of `elf_bytes`. +/// +/// The VM half — the statement, the AIRs rebuilt from the layout the proof +/// declares, the preprocessed roots, the LogUp bus balance — is here; every +/// table's rounds after round 1 and each group's FRI are +/// [`stark::batched_verifier::verify_batched`]. +pub fn verify( + proof: &BatchedProof, + elf_bytes: &[u8], + proof_options: &ProofOptions, +) -> Result { + verify_with_precomputed(proof, elf_bytes, proof_options, None, None) +} + +/// [`verify`] with the ELF-only preprocessed roots supplied instead of +/// recomputed. The recursion guest holds them already; recomputing DECODE and +/// every data page in-VM is the single most expensive thing a verifier can do. +pub fn verify_with_precomputed( + proof: &BatchedProof, + elf_bytes: &[u8], + proof_options: &ProofOptions, + decode_commitment: Option, + page_commitments: Option<&[(u64, stark::config::Commitment)]>, +) -> Result { + let table_counts = &proof.table_counts; + table_counts.validate()?; + let n = proof.tables.len(); + if proof.openings.len() != n || proof.group_of.len() != n || proof.fold_order.len() != n { + return Err(Error::InvalidTableCounts(format!( + "batched proof: {n} tables, {} openings, {} group slots, {} fold entries", + proof.openings.len(), + proof.group_of.len(), + proof.fold_order.len() + ))); + } + let elf = executor::elf::Elf::load(elf_bytes) + .map_err(|e| Error::Prover(format!("batched verify: ELF: {e}")))?; + let num_private_input_pages = proof + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(); + let max_pages = crate::tables::page::max_private_input_pages(); + if num_private_input_pages > max_pages { + return Err(Error::InvalidTableCounts(format!( + "num_private_input_pages ({num_private_input_pages}) exceeds max ({max_pages})", + ))); + } + let runtime_page_ranges = + crate::tables::trace_builder::runtime_page_ranges(&proof.page_configs); + let page_configs = Traces::page_configs_from_elf_and_runtime( + &elf, + &runtime_page_ranges, + num_private_input_pages, + n, + )?; + // `total()` is checked: a proof whose counts sum past `usize` is rejected + // here rather than wrapping into a plausible-looking expectation. + let total = table_counts.total().ok_or_else(|| { + Error::InvalidTableCounts("table_counts total overflows usize".to_string()) + })?; + let expected = total + crate::FIXED_TABLE_COUNT + page_configs.len(); + if expected != n { + return Err(Error::InvalidTableCounts(format!( + "table_counts total ({total}) + {} fixed + {} pages = {expected}, but the proof has {n} tables", + crate::FIXED_TABLE_COUNT, + page_configs.len(), + ))); + } + let vm_airs = crate::VmAirs::new( + &elf, + proof_options, + false, + &page_configs, + table_counts, + decode_commitment, + true, + None, + page_commitments, + None, + ); + let airs = vm_airs.air_refs(); + if airs.len() != n { + error!("batched verify: {} AIRs for {n} tables", airs.len()); + return Ok(false); + } + + let mut transcript = DefaultTranscript::::new(&[]); + crate::statement::absorb_statement( + &mut transcript, + crate::statement::StatementKind::Monolithic, + elf_bytes, + &proof.public_output, + table_counts, + num_private_input_pages, + &runtime_page_ranges, + proof_options.fri_final_poly_log_degree, + ); + // Round 1, in AIR order. A preprocessed table's precomputed root is the + // AIR's constant, not the prover's word. + for (idx, (air, t)) in airs.iter().zip(proof.tables.iter()).enumerate() { + if air.is_preprocessed() { + let expected = air.precomputed_commitment(); + match t.precomputed_root { + Some(actual) if actual == expected => {} + _ => { + error!("batched verify: table {idx}'s precomputed root is not the AIR's"); + return Ok(false); + } + } + transcript.append_bytes(&expected); + } else if t.precomputed_root.is_some() { + error!("batched verify: table {idx} carries a precomputed root it should not"); + return Ok(false); + } + transcript.append_bytes(&t.main_root); + } + let logup: Vec> = (0..stark::lookup::LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect(); + + // Every interacting table contributes to the bus, no other does, and the + // contributions balance against the public output. + for (idx, (air, t)) in airs.iter().zip(proof.tables.iter()).enumerate() { + if air.has_trace_interaction() != t.bus_public_inputs.is_some() { + error!("batched verify: table {idx}'s bus inputs do not match its AIR"); + return Ok(false); + } + } + let Some(expected_balance) = crate::compute_commit_bus_offset( + &proof.public_output, + 0, + &logup[0], + &logup[stark::lookup::LOGUP_CHALLENGE_ALPHA], + ) else { + error!("batched verify: the public output has no bus balance"); + return Ok(false); + }; + let mut total = FieldElement::::zero(); + for (air, t) in airs.iter().zip(proof.tables.iter()) { + if air.has_trace_interaction() + && let Some(ref bpi) = t.bus_public_inputs + { + total += bpi.table_contribution; + } + } + if total != expected_balance { + error!("batched verify: LogUp bus does not balance"); + return Ok(false); + } + + // Each table as the ordinary verifier reads one, with the FRI left empty. + let tables: Vec> = proof + .tables + .iter() + .zip(proof.openings.iter()) + .map(|(t, opening)| StarkProof { + trace_length: t.trace_rows, + lde_trace_main_merkle_root: t.main_root, + lde_trace_aux_merkle_root: t.aux_root, + lde_trace_precomputed_merkle_root: t.precomputed_root, + trace_ood_evaluations: t.trace_ood.clone(), + trace_ood_next_evaluations: t.trace_ood_next.clone(), + composition_poly_root: t.composition_poly_root, + composition_poly_parts_ood_evaluation: t.parts_ood.clone(), + fri_layers_merkle_roots: Vec::new(), + fri_final_poly_coeffs: Vec::new(), + query_list: Vec::new(), + deep_poly_openings: opening.clone(), + nonce: None, + bus_public_inputs: t.bus_public_inputs.clone(), + public_inputs: (), + }) + .collect(); + let blowup = proof_options.blowup_factor as usize; + let groups: Vec> = proof + .groups + .iter() + .map(|(lde_size, fri)| BatchedGroup { + trace_rows: lde_size / blowup, + fri, + }) + .collect(); + let public_inputs = vec![(); n]; + Ok(stark::batched_verifier::verify_batched( + &airs, + &public_inputs, + &tables, + &proof.group_of, + &groups, + &proof.fold_order, + &transcript, + &logup, + )) +} diff --git a/prover/src/challenge_phase.rs b/prover/src/challenge_phase.rs new file mode 100644 index 000000000..a40115a57 --- /dev/null +++ b/prover/src/challenge_phase.rs @@ -0,0 +1,331 @@ +//! Approach 1's Challenge phase: absorb every root and sample the one challenge +//! the whole execution shares. +//! +//! The spec's step after Commit is to pad and commit the remaining tables and +//! then sample the LogUp challenges. [`crate::commit_phase::run_to_end`] does +//! the padding; this does the sampling, and it is where Approach 1 differs from +//! the continuations in `main`. There, each epoch samples its own challenges, so +//! the tables of different epochs live on different buses and need the +//! local-to-global apparatus to be tied back together. Here every chunk of the +//! run is absorbed into one transcript and answers to one `(z, alpha)`, so there +//! is nothing to tie. +//! +//! The order the roots are absorbed in *is* the protocol: it has to be the AIR +//! order the ordinary prover uses, with a preprocessed table's precomputed root +//! ahead of its own. `challenge_matches_the_ordinary_prover` pins that against a +//! real proof rather than against this file's idea of the order. + +use std::collections::HashMap; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::proof::options::ProofOptions; +use stark::prover::MainRoots; + +use crate::Error; +use crate::commit_phase::Committed; +use crate::statement::{StatementKind, absorb_statement}; +use crate::streaming::{GROUP_ORDER, NUM_FIXED_AIRS}; +use crate::tables::trace_builder::{TableKind, runtime_page_ranges}; +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; +use crate::{TableCounts, VmAirs}; +use executor::elf::Elf; +use math::field::element::FieldElement; +use stark::trace::TraceTable; + +/// The one challenge the whole execution shares, and the roots it was drawn +/// from. +pub struct Challenge { + /// `z` and `alpha`, in sampling order. + pub challenges: Vec>, + /// Every root absorbed, in AIR order. + pub roots: Vec, + /// The AIRs of this proof, built once here: their preprocessed commitments + /// (DECODE from the ELF, one per ELF data page, ...) are the bulk of this pass. + pub(crate) airs: crate::VmAirs, + /// The transcript right after the sampling, which every later pass forks + /// per table. Kept rather than rebuilt: re-absorbing every root to get back + /// to this state is both slower and a second place for the order to be + /// wrong. + pub transcript: DefaultTranscript, + /// The layout the roots were assembled in, so a later pass can ask where a + /// chunk sits without recounting. + pub(crate) order: crate::streaming::AirOrder, +} + +/// Sample the shared LogUp challenges from a finished Commit phase. +/// +/// `elf_bytes` is the raw program: the statement binds its digest, so the +/// challenge depends on the program proved and not only on the tables it +/// produced. +pub fn run( + committed: &Committed, + elf: &Elf, + elf_bytes: &[u8], + proof_options: &ProofOptions, +) -> Result { + let remaining = &committed.remaining; + let mut table_counts = count_chunks_by_kind( + committed + .chunks + .iter() + .map(|(kind, chunk, _)| (*kind, *chunk)), + ); + // The accelerators are resident: this approach accumulates one table per + // kind across the whole run, so each reports 1 when it has rows and 0 when + // the run never called it. That is the shape #977 expects, and the only one + // `TableCounts::validate` accepts — it rejects any accelerator count above 1. + { + let [commit, keccak, keccak_rnd, ecsm, ecdas, hint] = remaining.accumulated.present; + table_counts.commit = usize::from(commit); + table_counts.keccak = usize::from(keccak); + table_counts.keccak_rnd = usize::from(keccak_rnd); + table_counts.ecsm = usize::from(ecsm); + table_counts.ecdas = usize::from(ecdas); + table_counts.hint = usize::from(hint); + } + let table_counts = table_counts; + let airs = VmAirs::new( + elf, + proof_options, + false, + &remaining.page_configs, + &table_counts, + Some(committed.precomputed.decode), + true, + None, + Some(&committed.precomputed.pages), + None, + ); + + let roots = assemble_roots(committed, &airs, &table_counts)?; + + let mut transcript = DefaultTranscript::::new(&[]); + absorb_statement( + &mut transcript, + StatementKind::Monolithic, + elf_bytes, + &remaining.public_output, + &table_counts, + remaining + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(), + &runtime_page_ranges(&remaining.page_configs), + proof_options.fri_final_poly_log_degree, + ); + for root in &roots { + if let Some(ref precomputed) = root.precomputed { + transcript.append_bytes(precomputed); + } + transcript.append_bytes(&root.main); + } + + let challenges = (0..stark::lookup::LOGUP_NUM_CHALLENGES) + .map(|_| transcript.sample_field_element()) + .collect(); + + let order = crate::streaming::AirOrder::new( + table_counts, + airs.include_halt, + remaining.page_configs.len(), + ); + Ok(Challenge { + challenges, + roots, + airs, + transcript, + order, + }) +} + +/// Every root the transcript absorbs, in `VmAirs::air_trace_pairs` order. +/// +/// The tables the walk committed come back keyed by `(kind, chunk)` in the order +/// they closed, which is not the AIR order; the ones it could not commit are +/// still traces and are committed here. +fn assemble_roots( + committed: &Committed, + airs: &VmAirs, + table_counts: &TableCounts, +) -> Result, Error> { + let remaining = &committed.remaining; + let accumulated = &remaining.accumulated; + + let mut roots = Vec::new(); + let fixed: [( + &crate::VmAir, + &TraceTable, + &str, + ); NUM_FIXED_AIRS] = [ + (&airs.bitwise, &remaining.bitwise, "BITWISE"), + (&airs.decode, &remaining.decode, "DECODE"), + (&airs.keccak_rc, &accumulated.keccak_rc, "KECCAK_RC"), + (&airs.register, &remaining.register, "REGISTER"), + ]; + // Small tables, many of them: one commit at a time leaves most cores idle. + #[cfg(feature = "parallel")] + let fixed_roots = fixed + .par_iter() + .map(|(air, trace, name)| commit_resident(air, trace, name)) + .collect::, _>>()?; + #[cfg(not(feature = "parallel"))] + let fixed_roots = fixed + .iter() + .map(|(air, trace, name)| commit_resident(air, trace, name)) + .collect::, _>>()?; + roots.extend(fixed_roots); + if airs.include_halt { + roots.push(commit_resident(&airs.halt, &remaining.halt, "HALT")?); + } + + // Then the accelerators, in `air_trace_pairs` order, skipping the ones the + // run never called. An absent kind has an empty AIR vec and contributes no + // root — the order is the protocol, so both sides drop the same slot. + for (air_vec, trace, name) in [ + (&airs.commits, &accumulated.commit, "COMMIT"), + (&airs.keccaks, &accumulated.keccak, "KECCAK"), + (&airs.keccak_rnds, &accumulated.keccak_rnd, "KECCAK_RND"), + (&airs.ecsms, &accumulated.ecsm, "ECSM"), + (&airs.ecdases, &accumulated.ecdas, "ECDAS"), + (&airs.hints, &accumulated.hint, "HINT"), + ] { + if let Some(air) = air_vec.first() { + roots.push(commit_resident(air, trace, name)?); + } + } + + let mut by_slot: HashMap<(TableKind, usize), &MainRoots> = HashMap::new(); + for (kind, chunk, root) in &committed.chunks { + if by_slot.insert((*kind, *chunk), root).is_some() { + return Err(Error::Prover(format!( + "challenge phase: {kind:?} chunk {chunk} committed twice" + ))); + } + } + + let mut page_airs = airs.pages.iter().zip(remaining.pages.iter()); + for group in GROUP_ORDER { + let Some(kind) = group else { + // PAGE is built from the ELF image rather than from an op list, so + // it is never retired and is committed here with the rest. + let pages: Vec<_> = page_airs.by_ref().collect(); + #[cfg(feature = "parallel")] + let page_roots = pages + .par_iter() + .map(|(air, trace)| commit_resident(air, trace, "PAGE")) + .collect::, _>>()?; + #[cfg(not(feature = "parallel"))] + let page_roots = pages + .iter() + .map(|(air, trace)| commit_resident(air, trace, "PAGE")) + .collect::, _>>()?; + roots.extend(page_roots); + continue; + }; + for chunk in 0..count_for(table_counts, kind) { + let root = by_slot.remove(&(kind, chunk)).ok_or_else(|| { + Error::Prover(format!( + "challenge phase: no root for {kind:?} chunk {chunk}" + )) + })?; + roots.push(root.clone()); + } + } + if let Some(((kind, chunk), _)) = by_slot.into_iter().next() { + return Err(Error::Prover(format!( + "challenge phase: {kind:?} chunk {chunk} has a root but no AIR" + ))); + } + + Ok(roots) +} + +fn commit_resident( + air: &crate::VmAir, + trace: &TraceTable, + name: &str, +) -> Result { + type P = stark::prover::Prover; +

>::commit_table_root(air.as_ref(), trace) + .ok_or_else(|| Error::Prover(format!("challenge phase: no commitment for {name}"))) +} + +/// How many chunks a pass produced per kind. +/// +/// Taken as `(kind, chunk)` pairs rather than as a phase's own output, because +/// every pass over the execution produces the same layout and each has its own +/// per-chunk payload. +pub(crate) fn count_chunks_by_kind( + chunks: impl Iterator, +) -> TableCounts { + let mut counts = TableCounts { + cpu: 0, + lt: 0, + memw: 0, + memw_aligned: 0, + load: 0, + mul: 0, + dvrm: 0, + shift: 0, + branch: 0, + memw_register: 0, + eq: 0, + bytewise: 0, + store: 0, + cpu32: 0, + keccak: 0, + keccak_rnd: 0, + ecsm: 0, + ecdas: 0, + hint: 0, + commit: 0, + }; + for (kind, chunk) in chunks { + let slot = slot_for(&mut counts, kind); + *slot = (*slot).max(chunk + 1); + } + counts +} + +pub(crate) fn count_for(counts: &TableCounts, kind: TableKind) -> usize { + match kind { + TableKind::Cpu => counts.cpu, + TableKind::Lt => counts.lt, + TableKind::Memw => counts.memw, + TableKind::MemwAligned => counts.memw_aligned, + TableKind::Load => counts.load, + TableKind::Mul => counts.mul, + TableKind::Dvrm => counts.dvrm, + TableKind::Shift => counts.shift, + TableKind::Branch => counts.branch, + TableKind::MemwRegister => counts.memw_register, + TableKind::Eq => counts.eq, + TableKind::Bytewise => counts.bytewise, + TableKind::Store => counts.store, + TableKind::Cpu32 => counts.cpu32, + } +} + +fn slot_for(counts: &mut TableCounts, kind: TableKind) -> &mut usize { + match kind { + TableKind::Cpu => &mut counts.cpu, + TableKind::Lt => &mut counts.lt, + TableKind::Memw => &mut counts.memw, + TableKind::MemwAligned => &mut counts.memw_aligned, + TableKind::Load => &mut counts.load, + TableKind::Mul => &mut counts.mul, + TableKind::Dvrm => &mut counts.dvrm, + TableKind::Shift => &mut counts.shift, + TableKind::Branch => &mut counts.branch, + TableKind::MemwRegister => &mut counts.memw_register, + TableKind::Eq => &mut counts.eq, + TableKind::Bytewise => &mut counts.bytewise, + TableKind::Store => &mut counts.store, + TableKind::Cpu32 => &mut counts.cpu32, + } +} diff --git a/prover/src/commit_phase.rs b/prover/src/commit_phase.rs new file mode 100644 index 000000000..c07062190 --- /dev/null +++ b/prover/src/commit_phase.rs @@ -0,0 +1,230 @@ +//! Approach 1's Commit phase: walk the execution, committing and retiring each +//! table as it fills. +//! +//! The spec has the prover go through execution "and once the memory pressure +//! becomes too large, batch commit to all full tables in memory; then these +//! tables are dropped". This is that pass, expressed as a [`pass::Visitor`]: +//! what it does with a finished table is commit its main trace and let the +//! table die. What it produces is a root per chunk and, at the end, the tables +//! that cannot be retired — which the Challenge phase commits and samples from. + +use stark::config::Commitment; +use stark::proof::options::ProofOptions; +use stark::prover::{IsStarkProver, MainRoots}; + +use crate::Error; +use crate::pass::{self, ChunkAirs, Resident, Visitor}; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::{TableKind, Traces}; +use crate::tables::types::*; +use executor::elf::Elf; +use stark::trace::TraceTable; + +/// One chunked table's commitment, by kind and position. +pub type ChunkCommitment = (TableKind, usize, MainRoots); + +/// Every chunk committed, and what the Commit phase leaves resident. +pub struct Committed { + /// A root per chunk of every chunked table, walk-closed and tail alike. + pub chunks: Vec, + /// The tables the walk could not commit, still as traces. + pub remaining: Resident, + /// The preprocessed commitments the ELF alone determines, computed beside the walk. + pub precomputed: Precomputed, +} + +/// The preprocessed commitments that depend on the ELF and nothing else — DECODE and +/// one per ELF data page. They are most of what building the AIRs costs, and they +/// need nothing from the execution, so they run on a thread beside the walk. +pub struct Precomputed { + pub decode: Commitment, + pub pages: Vec<(u64, Commitment)>, +} + +impl Precomputed { + pub fn new(elf: &Elf, proof_options: &ProofOptions) -> Result { + let decode = crate::tables::decode::commitment_from_elf(elf, proof_options) + .map_err(|e| Error::Prover(format!("decode commitment: {e}")))?; + let pages = Traces::page_configs_from_elf(elf) + .iter() + .filter(|config| config.init_values.is_some()) + .map(|config| { + ( + config.page_base, + crate::tables::page::compute_precomputed_commitment(config, proof_options), + ) + }) + .collect(); + Ok(Self { decode, pages }) + } +} + +/// What the walk alone produced. +pub struct CommitPhase { + /// One entry per chunk closed during the walk. Unordered: the batch runs + /// its tables in parallel, so what reads this indexes by `(kind, chunk)`. + pub closed: Vec, + /// Everything the walk still held when the execution ended. + pub walked: pass::Walked, +} + +/// One table on its way to a pass: what it is, where it sits, and its trace. +type Item = ( + TableKind, + usize, + TraceTable, +); + +/// Commits each table's main trace and drops it, `k` tables at a time. +struct CommitMain<'a> { + batch: pass::Batched<'a, Item>, +} + +impl<'a> CommitMain<'a> { + fn new( + scope: &'a std::thread::Scope<'a, '_>, + airs: &'a ChunkAirs, + roots: &'a std::sync::Mutex>, + ) -> Self { + Self { + batch: pass::Batched::new(scope, move |items| commit_batch(airs, roots, items)), + } + } +} + +/// One batch, in parallel. The tables in a batch are independent — each commits +/// its own trace against its own AIR — so the only shared thing is where the +/// roots land. +fn commit_batch( + airs: &ChunkAirs, + roots: &std::sync::Mutex>, + items: Vec, +) -> Result<(), Error> { + #[cfg(feature = "parallel")] + use rayon::prelude::*; + type P = stark::prover::Prover; + let commit = |(kind, chunk, trace): Item| -> Result { +

>::commit_table_root(airs.get(kind).as_ref(), &trace) + .map(|root| (kind, chunk, root)) + .ok_or_else(|| { + Error::Prover(format!("commit phase: no commitment for a {kind:?} chunk")) + }) + }; + #[cfg(feature = "parallel")] + let done: Result, Error> = items.into_par_iter().map(commit).collect(); + #[cfg(not(feature = "parallel"))] + let done: Result, Error> = items.into_iter().map(commit).collect(); + roots.lock().expect("roots").extend(done?); + Ok(()) +} + +impl Visitor for CommitMain<'_> { + fn table( + &mut self, + kind: TableKind, + chunk: usize, + trace: TraceTable, + ) -> Result<(), Error> { + self.batch.push((kind, chunk, trace)) + } + + fn flush(&mut self) -> Result<(), Error> { + self.batch.drain() + } +} + +/// The Commit phase's walk, stopping before the end-of-run tables. +/// +/// Each chunk is committed the moment it fills and its trace is dropped, so +/// nothing that has been committed is still resident. +pub fn run( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, + proof_options: &ProofOptions, +) -> Result { + let airs = ChunkAirs::new(proof_options); + let roots = std::sync::Mutex::new(Vec::new()); + let walked = std::thread::scope(|s| { + let mut visitor = CommitMain::new(s, &airs, &roots); + let walked = pass::walk(elf, private_input, max_rows, &mut visitor)?; + visitor.flush()?; + Ok::<_, Error>(walked) + })?; + Ok(CommitPhase { + closed: roots.into_inner().expect("roots"), + walked, + }) +} + +/// The Commit phase end to end. +/// +/// The walk, then the padding of everything it could not close. What comes back +/// is a root per chunk of every chunked table and, still resident, only the +/// tables that are not built from an op list: the preprocessed ones and the +/// accumulators. That is the state the Challenge phase starts from. +pub fn run_to_end( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, + proof_options: &ProofOptions, +) -> Result { + let airs = ChunkAirs::new(proof_options); + let roots = std::sync::Mutex::new(Vec::new()); + let (remaining, precomputed) = std::thread::scope(|s| { + let precomputed = s.spawn(|| Precomputed::new(elf, proof_options)); + let mut visitor = CommitMain::new(s, &airs, &roots); + let remaining = pass::run(elf, private_input, max_rows, &mut visitor); + (remaining, precomputed.join().expect("precompute thread")) + }); + Ok(Committed { + chunks: roots.into_inner().expect("roots"), + remaining: remaining?, + precomputed: precomputed?, + }) +} + +/// Pad and commit what a walk could not close. +/// +/// The Commit phase's half of the end-of-run step, kept as its own entry point +/// for the caller that ran [`run`] and wants the tails separately. +pub fn commit_remaining( + walked: pass::Walked, + private_input: &[u8], + max_rows: &MaxRowsConfig, + proof_options: &ProofOptions, +) -> Result<(Vec, Resident), Error> { + let airs = ChunkAirs::new(proof_options); + let roots = std::sync::Mutex::new(Vec::new()); + let resident = std::thread::scope(|s| { + let mut visitor = CommitMain::new(s, &airs, &roots); + pass::finish(walked, private_input, max_rows, &mut visitor) + })?; + Ok((roots.into_inner().expect("roots"), resident)) +} + +/// The ordinary build, for comparison against [`run_to_end`]. +/// +/// The same execution and the same tables, built the way `prove` builds them: +/// every trace resident at once and nothing committed. It lives beside the +/// Commit phase so the two arms of the comparison are driven identically, and +/// so the storage-mode argument stays on this side of the feature gate — the +/// lint enables `lambda-vm-prover/disk-spill` without enabling the CLI's, and a +/// caller there would not agree with this signature. +pub fn build_resident( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, +) -> Result { + let executed = executor::vm::execution::Executor::new(elf, private_input.to_vec()) + .and_then(|e| e.run()) + .map_err(|e| Error::Prover(format!("execution failed: {e}")))?; + Traces::from_elf_and_logs( + elf, + &executed.logs, + max_rows, + private_input, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) +} diff --git a/prover/src/instruments.rs b/prover/src/instruments.rs index f15a8a824..7f338bcb7 100644 --- a/prover/src/instruments.rs +++ b/prover/src/instruments.rs @@ -279,4 +279,17 @@ pub fn print_report( eprintln!(" {}", "─".repeat(56)); eprintln!(); } + + // What the memory modes actually cost, counted rather than assumed. Printed + // always: on the resident path the expansions are the per-table floor and + // the retired counters are zero, which is itself the thing to check. + let (expansions, trace_builds, shape_queries) = stark::instruments::residency_counts(); + eprintln!("=== RESIDENCY ==="); + eprintln!(" {:<36} {:>8}", "Main LDE expansions", expansions); + eprintln!(" {:<36} {:>8}", "Retired trace builds", trace_builds); + eprintln!( + " {:<36} {:>8}", + "Shapes answered without building", shape_queries + ); + eprintln!(); } diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 2a1772d0d..797cea5b7 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -12,13 +12,20 @@ #[cfg(feature = "disk-spill")] pub mod auto_storage; +pub mod batched_proof; +pub mod batched_verifier; +pub mod challenge_phase; +pub mod commit_phase; pub mod constraints; pub mod continuation; #[cfg(feature = "debug-checks")] mod debug_report; #[cfg(feature = "instruments")] pub mod instruments; +pub mod logup_phase; mod paged_mem; +pub mod pass; +pub(crate) mod streaming; pub use stark::profile_markers; pub mod recursion; mod statement; @@ -1313,25 +1320,73 @@ pub fn prove_with_options_and_inputs( #[cfg(feature = "instruments")] let __sp = stark::instruments::span("trace_build"); + // The storage mode and `LAMBDA_STREAM_LDE=auto` read the same analytical + // peak estimate, so the log pre-pass behind it is paid once, and only when + // something actually asks for it. #[cfg(feature = "disk-spill")] let storage_mode = { let lengths = count_table_lengths(&program, &result.logs, max_rows, private_inputs)?; + if stark::prover::streaming_retire_lde_is_auto() { + stark::prover::set_retire_lde(auto_storage::decide_retire_lde( + &lengths, + proof_options.blowup_factor, + )); + } auto_storage::decide(&lengths, proof_options.blowup_factor) }; - let mut traces = Traces::from_elf_and_logs( - &program, - &result.logs, - max_rows, - private_inputs, - #[cfg(feature = "disk-spill")] - storage_mode, - )?; + // The estimate lives behind `disk-spill` (so do `TableLengths` and the + // storage mode it feeds). Say so instead of silently proving with the mode + // off, which would read as "auto decided no". + #[cfg(not(feature = "disk-spill"))] + if stark::prover::streaming_retire_lde_is_auto() { + log::warn!( + "LAMBDA_STREAM_LDE=auto needs the `disk-spill` feature for the peak-RAM estimate; \ + proving with the main LDE resident. Pass LAMBDA_STREAM_LDE=1 to force it on." + ); + } + + // Retiring the LDE also retires the traces: the same flag, one rung further + // down the same ladder. The chunked tables come back as placeholders and the + // provider rebuilds each chunk at the two points the prover needs it. + let retire_traces = stark::prover::streaming_retire_lde(); + // The public output is all this path still needs from the run above; keeping + // it lets the logs go before the build, which is the phase that peaks. + let executor_output = result.return_values.memory_values.clone(); + let (mut traces, streaming) = if retire_traces { + // The collector walks the execution itself, one chunk of logs at a time, + // so this run's logs are dead weight from here on. Freeing them costs a + // second execution and is the shape the Commit phase needs: a prover + // that walks an execution instead of being handed it whole. + drop(result); + let (traces, routed) = Traces::from_elf_and_logs_streaming( + &program, + max_rows, + private_inputs, + #[cfg(feature = "disk-spill")] + storage_mode, + )?; + // This path always proves a single, final epoch, so HALT is present — + // passed explicitly rather than assumed, because the slot map is only + // correct if it agrees with `VmAirs::air_trace_pairs`. + let provider = streaming::StreamingProvider::new(routed, max_rows.clone(), &traces, true); + (traces, Some(provider)) + } else { + let traces = Traces::from_elf_and_logs( + &program, + &result.logs, + max_rows, + private_inputs, + #[cfg(feature = "disk-spill")] + storage_mode, + )?; + drop(result); + (traces, None) + }; debug_assert_eq!( - traces.public_output_bytes, result.return_values.memory_values, + traces.public_output_bytes, executor_output, "public output diverged between executor view and trace reconstruction" ); - drop(result); #[cfg(feature = "instruments")] drop(__sp); @@ -1392,11 +1447,14 @@ pub fn prove_with_options_and_inputs( // Phase 4: Prove (multi_prove) #[cfg(feature = "instruments")] let __sp = stark::instruments::span("proving"); - let proof = Prover::multi_prove( + let proof = Prover::multi_prove_with_provider( airs.air_trace_pairs(&mut traces), &mut transcript, #[cfg(feature = "disk-spill")] storage_mode, + streaming + .as_ref() + .map(|p| p as &dyn stark::prover::TraceProvider<_, _>), ) .map_err(|e| Error::Prover(format!("{e:?}")))?; #[cfg(feature = "instruments")] diff --git a/prover/src/logup_phase.rs b/prover/src/logup_phase.rs new file mode 100644 index 000000000..0aee6b87c --- /dev/null +++ b/prover/src/logup_phase.rs @@ -0,0 +1,960 @@ +//! Approach 1's proving pass: walk the execution again and prove each table +//! against the challenge the Commit phase produced. +//! +//! The spec's third step is a re-execution. It has to be: the LogUp columns are +//! a function of the challenge, and the challenge is not known until every main +//! root has been absorbed — by which time the tables that produced them are +//! gone. The ordinary prover avoids the second walk by keeping every trace +//! resident across the Round 1 barrier, which is exactly the residency this +//! approach refuses to pay. +//! +//! So the tables are rebuilt. `build_main` is deterministic, so the trace a +//! chunk gets here is byte-identical to the one the Commit phase committed, and +//! the aux columns are therefore the ones that root answers for. + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use stark::proof::options::ProofOptions; +use stark::proof::stark::StarkProof; +use stark::prover::IsStarkProver; + +use crate::Error; +use crate::challenge_phase::Challenge; +use crate::pass::{self, ChunkAirs, Resident, Visitor}; +use crate::streaming::NUM_FIXED_AIRS; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::TableKind; +use crate::tables::types::*; +use executor::elf::Elf; +use math::field::element::FieldElement; +use stark::trace::TraceTable; + +/// What the pass produced. +pub struct LogUp { + /// One proof per table, in `VmAirs::air_trace_pairs` order. + pub tables: Vec>, + /// The tables the pass could not retire, rebuilt by this walk. + pub resident: Resident, +} + +/// The pass's output as the proof the ordinary verifier takes: the same +/// `MultiProof` the monolithic prover emits, with the layout the walk resolved. +pub fn assemble_vm_proof(logup: LogUp, challenge: &Challenge) -> crate::VmProof { + let resident = &logup.resident; + crate::VmProof { + proof: stark::proof::stark::MultiProof { + proofs: logup.tables, + }, + runtime_page_ranges: crate::tables::trace_builder::runtime_page_ranges( + &resident.page_configs, + ), + table_counts: challenge.order.counts().clone(), + public_output: resident.public_output.clone(), + num_private_input_pages: resident + .page_configs + .iter() + .filter(|c| c.is_private_input) + .count(), + } +} + +type Item = ( + TableKind, + usize, + TraceTable, +); + +/// One table's finished proof, tagged with where it sits in the AIR order. +type Proved = (usize, StarkProof); + +/// Where a batch of proofs lands. Shared because the batch runs in parallel. +type Proofs = std::sync::Mutex>; + +struct BuildAux<'a> { + batch: pass::Batched<'a, Item>, +} + +impl<'a> BuildAux<'a> { + fn new( + scope: &'a std::thread::Scope<'a, '_>, + airs: &'a ChunkAirs, + challenge: &'a Challenge, + done: &'a Proofs, + ) -> Self { + Self { + batch: pass::Batched::new(scope, move |items| { + rounds_batch(airs, challenge, done, items) + }), + } + } +} + +/// One batch, in parallel. Each table runs against its own transcript fork, so +/// nothing crosses between them — the shared state is only where results land. +fn rounds_batch( + airs: &ChunkAirs, + challenge: &Challenge, + done: &Proofs, + items: Vec, +) -> Result<(), Error> { + #[cfg(feature = "parallel")] + use rayon::prelude::*; + let order = &challenge.order; + let n = order.len(); + let run_one = |(kind, chunk, mut trace): Item| { + let idx = order.index_of(kind, chunk).ok_or_else(|| { + Error::Prover(format!( + "logup phase: {kind:?} chunk {chunk} is not in the layout the Commit phase produced" + )) + })?; + let mut transcript = fork(&challenge.transcript, idx, n); + let rounds = prove_table( + airs.get(kind).as_ref(), + &mut trace, + &challenge.challenges, + &mut transcript, + ) + .map_err(|e| Error::Prover(format!("logup phase: {kind:?} chunk {chunk}: {e}")))?; + Ok((idx, rounds)) + }; + #[cfg(feature = "parallel")] + let built: Result, Error> = items.into_par_iter().map(run_one).collect(); + #[cfg(not(feature = "parallel"))] + let built: Result, Error> = items.into_iter().map(run_one).collect(); + done.lock().expect("logup results").extend(built?); + Ok(()) +} + +impl Visitor for BuildAux<'_> { + fn table( + &mut self, + kind: TableKind, + chunk: usize, + trace: TraceTable, + ) -> Result<(), Error> { + self.batch.push((kind, chunk, trace)) + } + + fn flush(&mut self) -> Result<(), Error> { + self.batch.drain() + } +} + +/// A table's own transcript: the shared state after the challenge, separated by +/// AIR index. Reproduces the fused prover's forking exactly — a single-table +/// proof takes no index, and getting that wrong shifts every challenge. +#[cfg(test)] +pub(crate) fn fork_for( + challenge: &Challenge, + idx: usize, + num_airs: usize, +) -> DefaultTranscript { + fork(&challenge.transcript, idx, num_airs) +} + +/// Rebuild only the tables a pass cannot retire, for a caller that wants one of +/// them without proving the run. +/// The walk with nothing done to any table: what the execution costs to +/// replay and rebuild, on its own. This is the floor every pass pays, and the +/// number the pipeline can at best hide behind the proving. +pub fn walk_only( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, +) -> Result { + struct Skip; + impl Visitor for Skip { + fn table( + &mut self, + _kind: TableKind, + _chunk: usize, + _trace: TraceTable, + ) -> Result<(), Error> { + Ok(()) + } + } + pass::run(elf, private_input, max_rows, &mut Skip) +} + +fn fork( + shared: &DefaultTranscript, + idx: usize, + num_airs: usize, +) -> DefaultTranscript { + let mut t = shared.clone(); + if num_airs > 1 { + t.append_bytes(&(idx as u64).to_le_bytes()); + } + t +} + +fn prove_table( + air: &dyn stark::traits::AIR< + Field = GoldilocksField, + FieldExtension = GoldilocksExtension, + PublicInputs = (), + >, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut DefaultTranscript, +) -> Result, String> { + type P = stark::prover::Prover; +

>::prove_table_from_trace(air, &(), trace, challenges, transcript) + .map_err(|e| format!("{e:?}")) +} + +/// Run the LogUp pass over `elf`, against the challenge `challenge` sampled. +/// +/// `challenge` has to come from the Commit phase's roots over this same +/// execution: an aux trace built against a different challenge commits to a bus +/// the main traces never balanced. +pub fn run( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, + proof_options: &ProofOptions, + challenge: &Challenge, +) -> Result { + let airs = ChunkAirs::new(proof_options); + let done = std::sync::Mutex::new(Vec::new()); + let mut resident = std::thread::scope(|s| { + let mut visitor = BuildAux::new(s, &airs, challenge, &done); + pass::run(elf, private_input, max_rows, &mut visitor) + })?; + + let chunks = done.into_inner().expect("logup results"); + let tables = assemble(chunks, &mut resident, challenge)?; + Ok(LogUp { tables, resident }) +} + +/// Every table's rounds 2-3 in `VmAirs::air_trace_pairs` order. +/// +/// The chunked tables come back keyed by the index the walk resolved; the +/// tables that stay have theirs built here, as the Challenge phase built their +/// mains. +fn assemble( + chunks: Vec, + resident: &mut Resident, + challenge: &Challenge, +) -> Result>, Error> { + #[cfg(feature = "parallel")] + use rayon::prelude::*; + let order = &challenge.order; + let airs = &challenge.airs; + + let mut slots: Vec>> = + (0..order.len()).map(|_| None).collect(); + for (idx, rounds) in chunks { + let slot = slots + .get_mut(idx) + .ok_or_else(|| Error::Prover(format!("logup phase: table {idx} is past the layout")))?; + if slot.is_some() { + return Err(Error::Prover(format!( + "logup phase: table {idx} was built twice" + ))); + } + *slot = Some(rounds); + } + + let ch = &challenge.challenges; + let n = order.len(); + let build = |idx: usize, + air: &crate::VmAir, + trace: &mut TraceTable| + -> Result, Error> { + let mut transcript = fork(&challenge.transcript, idx, n); + prove_table(air.as_ref(), trace, ch, &mut transcript) + .map_err(|e| Error::Prover(format!("logup phase: table {idx}: {e}"))) + }; + + let fixed: [( + &crate::VmAir, + &mut TraceTable, + ); NUM_FIXED_AIRS] = [ + (&airs.bitwise, &mut resident.bitwise), + (&airs.decode, &mut resident.decode), + (&airs.keccak_rc, &mut resident.accumulated.keccak_rc), + (&airs.register, &mut resident.register), + ]; + // Independent tables, most of them small: one at a time would leave the + // machine idle after the walk. + let mut jobs: Vec<( + usize, + &crate::VmAir, + &mut TraceTable, + )> = fixed + .into_iter() + .enumerate() + .map(|(idx, (air, trace))| (idx, air, trace)) + .collect(); + if airs.include_halt { + jobs.push((NUM_FIXED_AIRS, &airs.halt, &mut resident.halt)); + } + // Then the accelerators, in `air_trace_pairs` order, skipping the ones + // the run never called: an absent kind has an empty AIR vec and no + // slot. `accel_index` is the single authority for where each lands. + for (slot, (air_vec, trace)) in [ + (&airs.commits, &mut resident.accumulated.commit), + (&airs.keccaks, &mut resident.accumulated.keccak), + (&airs.keccak_rnds, &mut resident.accumulated.keccak_rnd), + (&airs.ecsms, &mut resident.accumulated.ecsm), + (&airs.ecdases, &mut resident.accumulated.ecdas), + (&airs.hints, &mut resident.accumulated.hint), + ] + .into_iter() + .enumerate() + { + if let (Some(air), Some(idx)) = (air_vec.first(), order.accel_index(slot)) { + jobs.push((idx, air, trace)); + } + } + for (i, (air, trace)) in airs.pages.iter().zip(resident.pages.iter_mut()).enumerate() { + let idx = order + .page_index(i) + .ok_or_else(|| Error::Prover(format!("logup phase: page {i} is not in the layout")))?; + jobs.push((idx, air, trace)); + } + #[cfg(feature = "parallel")] + let built: Vec<(usize, StarkProof)> = jobs + .into_par_iter() + .map(|(idx, air, trace)| build(idx, air, trace).map(|proof| (idx, proof))) + .collect::>()?; + #[cfg(not(feature = "parallel"))] + let built: Vec<(usize, StarkProof)> = jobs + .into_iter() + .map(|(idx, air, trace)| build(idx, air, trace).map(|proof| (idx, proof))) + .collect::>()?; + for (idx, proof) in built { + slots[idx] = Some(proof); + } + + slots + .into_iter() + .enumerate() + .map(|(idx, slot)| { + slot.ok_or_else(|| Error::Prover(format!("logup phase: table {idx} was never built"))) + }) + .collect() +} + +pub use crate::batched_proof::{BatchedProof, Open, TablePublic}; + +/// One FRI per height group, instead of one per table. +pub struct Batched { + /// Per table, in AIR order: what it contributes besides its codeword. + pub tables: Vec, + /// Per group, in ascending domain size: the domain and its FRI instance — + /// layers, final polynomial, the shared query indices and their + /// decommitments. + pub groups: Vec<(usize, stark::prover::GroupFri)>, + /// How many tables each group folded, so the collapse is visible. + pub members: Vec, + /// Which group each table belongs to, by AIR index — the group whose query + /// indices its openings answer. + pub group_of: Vec, + /// The AIR indices in the order they were folded. The coefficient of a + /// table depends on every table folded before it, so the verifier has to + /// replay this order and the proof carries it. + pub fold_order: Vec, + /// The chunk layout, which the verifier needs to rebuild the AIRs. + pub table_counts: crate::TableCounts, + /// Per table, by AIR index: its composition parts over the LDE domain when + /// the fold pass kept them (`A1_KEEP_COMPOSITION`), taken by the Open pass. + pub composition_ldes: Vec>>, + pub resident: Resident, +} + +type Deep = stark::prover::TableDeep; +type CompositionLde = Vec>>; + +/// Whether the fold pass keeps each table's composition parts for the Open +/// pass: memory for the constraint evaluation the rebuild would repeat. +fn keep_composition() -> bool { + std::env::var("A1_KEEP_COMPOSITION").is_ok_and(|v| v != "0") +} +/// What the fold walk carries between batches. +/// +/// The seed and the accumulators are advanced sequentially — the coefficient of +/// a table depends on every table folded before it — while the codewords that +/// feed them are computed in parallel. So the expensive half stays concurrent +/// and only the folding is serialised. +struct FoldState { + seed: DefaultTranscript, + /// One accumulator per distinct domain, and the rows behind it. + acc: std::collections::BTreeMap>, usize, usize)>, + /// The AIR indices in the order they were folded, which the proof carries + /// so a verifier can replay the same sequence of coefficients. + order: Vec, + tables: Vec<(usize, TablePublic)>, + kept: Vec<(usize, CompositionLde)>, +} + +type Deeps = std::sync::Mutex; + +struct BuildDeep<'a> { + batch: pass::Batched<'a, Item>, +} + +impl<'a> BuildDeep<'a> { + fn new( + scope: &'a std::thread::Scope<'a, '_>, + airs: &'a ChunkAirs, + challenge: &'a Challenge, + done: &'a Deeps, + ) -> Self { + Self { + batch: pass::Batched::new(scope, move |items| deep_batch(airs, challenge, done, items)), + } + } +} + +fn deep_batch( + airs: &ChunkAirs, + challenge: &Challenge, + done: &Deeps, + items: Vec, +) -> Result<(), Error> { + #[cfg(feature = "parallel")] + use rayon::prelude::*; + let order = &challenge.order; + let n = order.len(); + let run_one = |(kind, chunk, mut trace): Item| { + let idx = order.index_of(kind, chunk).ok_or_else(|| { + Error::Prover(format!( + "batched phase: {kind:?} chunk {chunk} is not in the layout" + )) + })?; + let mut transcript = fork(&challenge.transcript, idx, n); + let deep = deep_of( + airs.get(kind).as_ref(), + &mut trace, + &challenge.challenges, + &mut transcript, + challenge.roots.get(idx).cloned(), + ) + .map_err(|e| Error::Prover(format!("batched phase: {kind:?} chunk {chunk}: {e}")))?; + Ok((idx, deep)) + }; + #[cfg(feature = "parallel")] + let built: Result, Error> = items.into_par_iter().map(run_one).collect(); + #[cfg(not(feature = "parallel"))] + let built: Result, Error> = items.into_iter().map(run_one).collect(); + // Folded in a fixed order within the batch, so the sequence is a function + // of the walk and not of which thread finished first. + let mut built = built?; + built.sort_by_key(|(idx, _)| *idx); + let mut state = done.lock().expect("fold state"); + for (idx, deep) in built { + fold_one(&mut state, idx, deep); + } + Ok(()) +} + +/// Absorb a table, draw its coefficient, add it to its group, drop it. +fn fold_one(state: &mut FoldState, idx: usize, mut deep: Deep) { + type P = stark::prover::Prover; + if let Some(lde) = deep.composition_lde.take() { + state.kept.push((idx, lde)); + } + let coefficient =

>::fold_coefficient(&mut state.seed, &deep); + let entry = state + .acc + .entry(deep.lde_size) + .or_insert_with(|| (Vec::new(), deep.trace_rows, 0)); +

>::accumulate(&mut entry.0, &coefficient, &deep.deep); + entry.2 += 1; + state.order.push(idx); + state.tables.push(( + idx, + TablePublic { + trace_rows: deep.trace_rows, + main_root: deep.main_roots.main, + precomputed_root: deep.main_roots.precomputed, + aux_root: deep.aux_root, + composition_poly_root: deep.composition_poly_root, + trace_ood: deep.trace_ood, + trace_ood_next: deep.trace_ood_next, + parts_ood: deep.parts_ood, + bus_public_inputs: deep.bus_public_inputs, + }, + )); + // `deep` dies here, which is the whole point. +} + +fn deep_of( + air: &dyn stark::traits::AIR< + Field = GoldilocksField, + FieldExtension = GoldilocksExtension, + PublicInputs = (), + >, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut DefaultTranscript, + known_main: Option, +) -> Result { + type P = stark::prover::Prover; +

>::deep_for_table( + air, + &(), + trace, + challenges, + transcript, + known_main, + keep_composition(), + ) + .map_err(|e| format!("{e:?}")) +} + +impl Visitor for BuildDeep<'_> { + fn table( + &mut self, + kind: TableKind, + chunk: usize, + trace: TraceTable, + ) -> Result<(), Error> { + self.batch.push((kind, chunk, trace)) + } + + fn flush(&mut self) -> Result<(), Error> { + self.batch.drain() + } +} + +/// Walk the execution and fold every table into one FRI per height group. +/// +/// The codewords are held, not the LDEs they came from — one extension element +/// per row instead of every column — which is what lets the fold wait until +/// every table is done without walking the execution a third time. +/// +/// Grouping is by exact domain and can only be: the fold squares the coset +/// offset each layer, so a short codeword never lines up with a tall fold. +pub fn run_batched( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, + proof_options: &ProofOptions, + challenge: &Challenge, +) -> Result { + type P = stark::prover::Prover; + + let chunk_airs = ChunkAirs::new(proof_options); + let done = std::sync::Mutex::new(FoldState { + seed: challenge.transcript.clone(), + acc: std::collections::BTreeMap::new(), + order: Vec::new(), + tables: Vec::new(), + kept: Vec::new(), + }); + let mut resident = std::thread::scope(|s| { + let mut visitor = BuildDeep::new(s, &chunk_airs, challenge, &done); + pass::run(elf, private_input, max_rows, &mut visitor) + })?; + + // The tables the walk could not retire, folded after it in AIR order. + let order = &challenge.order; + let airs = &challenge.airs; + let n = order.len(); + { + #[cfg(feature = "parallel")] + use rayon::prelude::*; + let mut state = done.lock().expect("fold state"); + let build = |idx: usize, + air: &crate::VmAir, + trace: &mut TraceTable| + -> Result<(usize, Deep), Error> { + let mut transcript = fork(&challenge.transcript, idx, n); + let deep = deep_of( + air.as_ref(), + trace, + &challenge.challenges, + &mut transcript, + challenge.roots.get(idx).cloned(), + ) + .map_err(|e| Error::Prover(format!("batched phase: table {idx}: {e}")))?; + Ok((idx, deep)) + }; + let fixed: [( + &crate::VmAir, + &mut TraceTable, + ); NUM_FIXED_AIRS] = [ + (&airs.bitwise, &mut resident.bitwise), + (&airs.decode, &mut resident.decode), + (&airs.keccak_rc, &mut resident.accumulated.keccak_rc), + (&airs.register, &mut resident.register), + ]; + // The codewords are independent and computed in parallel; the fold + // itself is sequential and keeps this order, which the proof records. + let mut jobs: Vec<( + usize, + &crate::VmAir, + &mut TraceTable, + )> = fixed + .into_iter() + .enumerate() + .map(|(idx, (air, trace))| (idx, air, trace)) + .collect(); + if airs.include_halt { + jobs.push((NUM_FIXED_AIRS, &airs.halt, &mut resident.halt)); + } + // Then the accelerators, in `air_trace_pairs` order, skipping the ones + // the run never called: an absent kind has an empty AIR vec and no + // slot. `accel_index` is the single authority for where each lands. + for (slot, (air_vec, trace)) in [ + (&airs.commits, &mut resident.accumulated.commit), + (&airs.keccaks, &mut resident.accumulated.keccak), + (&airs.keccak_rnds, &mut resident.accumulated.keccak_rnd), + (&airs.ecsms, &mut resident.accumulated.ecsm), + (&airs.ecdases, &mut resident.accumulated.ecdas), + (&airs.hints, &mut resident.accumulated.hint), + ] + .into_iter() + .enumerate() + { + if let (Some(air), Some(idx)) = (air_vec.first(), order.accel_index(slot)) { + jobs.push((idx, air, trace)); + } + } + for (i, (air, trace)) in airs.pages.iter().zip(resident.pages.iter_mut()).enumerate() { + let idx = order.page_index(i).ok_or_else(|| { + Error::Prover(format!("batched phase: page {i} is not in the layout")) + })?; + jobs.push((idx, air, trace)); + } + #[cfg(feature = "parallel")] + let deeps: Vec<(usize, Deep)> = jobs + .into_par_iter() + .map(|(idx, air, trace)| build(idx, air, trace)) + .collect::>()?; + #[cfg(not(feature = "parallel"))] + let deeps: Vec<(usize, Deep)> = jobs + .into_iter() + .map(|(idx, air, trace)| build(idx, air, trace)) + .collect::>()?; + for (idx, deep) in deeps { + fold_one(&mut state, idx, deep); + } + } + + let FoldState { + mut seed, + acc, + order: fold_order, + mut tables, + kept, + } = done.into_inner().expect("fold state"); + if tables.len() != n { + return Err(Error::Prover(format!( + "batched phase: {} tables for a layout of {n}", + tables.len() + ))); + } + + // One FRI per accumulator, and the mapping from table to group. + let mut group_of = vec![usize::MAX; n]; + let sizes: Vec = acc.keys().copied().collect(); + for (idx, table) in tables.iter() { + let lde = table.trace_rows * proof_options.blowup_factor as usize; + let idx = *idx; + group_of[idx] = sizes.iter().position(|s| *s == lde).ok_or_else(|| { + Error::Prover(format!( + "batched phase: table {idx} has no group of size {lde}" + )) + })?; + } + + let any = chunk_airs.get(TableKind::Cpu).as_ref(); + let (mut groups, mut members) = (Vec::new(), Vec::new()); + for (lde_size, (codeword, trace_rows, count)) in acc { + let fri =

>::batch_fri(any, codeword, trace_rows, &mut seed) + .ok_or_else(|| Error::Prover(format!("batched phase: no FRI for size {lde_size}")))?; + groups.push((lde_size, fri)); + members.push(count); + } + + tables.sort_by_key(|(idx, _)| *idx); + let composition_ldes: Vec>> = (0..tables.len()) + .map(|_| std::sync::Mutex::new(None)) + .collect(); + for (idx, lde) in kept { + *composition_ldes[idx].lock().expect("kept composition") = Some(lde); + } + Ok(Batched { + tables: tables.into_iter().map(|(_, t)| t).collect(), + fold_order, + groups, + members, + group_of, + table_counts: challenge.order.counts().clone(), + composition_ldes, + resident, + }) +} + +/// What the Open pass produced: every table's rows at its group's indices. +pub struct Opened { + /// One entry per table, in AIR order. + pub openings: + Vec>, + pub resident: Resident, +} + +type Opens = std::sync::Mutex>; + +struct OpenTables<'a> { + batch: pass::Batched<'a, Item>, +} + +impl<'a> OpenTables<'a> { + fn new( + scope: &'a std::thread::Scope<'a, '_>, + airs: &'a ChunkAirs, + challenge: &'a Challenge, + batched: &'a Batched, + done: &'a Opens, + ) -> Self { + Self { + batch: pass::Batched::new(scope, move |items| { + open_batch(airs, challenge, batched, done, items) + }), + } + } +} + +fn iotas_of(batched: &Batched, idx: usize) -> Result<&[usize], Error> { + let g = *batched + .group_of + .get(idx) + .ok_or_else(|| Error::Prover(format!("open pass: table {idx} has no group")))?; + let (_, fri) = batched + .groups + .get(g) + .ok_or_else(|| Error::Prover(format!("open pass: table {idx} points at group {g}")))?; + Ok(&fri.iotas) +} + +fn open_batch( + airs: &ChunkAirs, + challenge: &Challenge, + batched: &Batched, + done: &Opens, + items: Vec, +) -> Result<(), Error> { + #[cfg(feature = "parallel")] + use rayon::prelude::*; + let order = &challenge.order; + let n = order.len(); + let run_one = |(kind, chunk, mut trace): Item| { + let idx = order.index_of(kind, chunk).ok_or_else(|| { + Error::Prover(format!( + "open pass: {kind:?} chunk {chunk} is not in the layout" + )) + })?; + let mut transcript = fork(&challenge.transcript, idx, n); + let opening = open_of( + airs.get(kind).as_ref(), + &mut trace, + &challenge.challenges, + &mut transcript, + iotas_of(batched, idx)?, + take_kept(batched, idx), + ) + .map_err(|e| Error::Prover(format!("open pass: {kind:?} chunk {chunk}: {e}")))?; + Ok((idx, opening)) + }; + #[cfg(feature = "parallel")] + let built: Result, Error> = items.into_par_iter().map(run_one).collect(); + #[cfg(not(feature = "parallel"))] + let built: Result, Error> = items.into_iter().map(run_one).collect(); + done.lock().expect("openings").extend(built?); + Ok(()) +} + +fn open_of( + air: &dyn stark::traits::AIR< + Field = GoldilocksField, + FieldExtension = GoldilocksExtension, + PublicInputs = (), + >, + trace: &mut TraceTable, + challenges: &[FieldElement], + transcript: &mut DefaultTranscript, + iotas: &[usize], + kept: Option, +) -> Result { + type P = stark::prover::Prover; + match kept { + Some(lde) =>

>::open_for_table_kept( + air, trace, challenges, transcript, iotas, lde, + ), + None =>

>::open_for_table( + air, + &(), + trace, + challenges, + transcript, + iotas, + ), + } + .map_err(|e| format!("{e:?}")) +} + +fn take_kept(batched: &Batched, idx: usize) -> Option { + batched + .composition_ldes + .get(idx) + .and_then(|slot| slot.lock().expect("kept composition").take()) +} + +impl Visitor for OpenTables<'_> { + fn table( + &mut self, + kind: TableKind, + chunk: usize, + trace: TraceTable, + ) -> Result<(), Error> { + self.batch.push((kind, chunk, trace)) + } + + fn flush(&mut self) -> Result<(), Error> { + self.batch.drain() + } +} + +/// Approach 1's fifth pass: walk the execution once more and open every table +/// at the indices its group settled on. +/// +/// The last walk, and the one the spec puts last for a reason — the indices do +/// not exist until the batched FRI is over, so nothing here could have been +/// folded into an earlier pass. +pub fn run_open( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, + proof_options: &ProofOptions, + challenge: &Challenge, + batched: &Batched, +) -> Result { + let chunk_airs = ChunkAirs::new(proof_options); + let done = std::sync::Mutex::new(Vec::new()); + let mut resident = std::thread::scope(|s| { + let mut visitor = OpenTables::new(s, &chunk_airs, challenge, batched, &done); + pass::run(elf, private_input, max_rows, &mut visitor) + })?; + let mut opens = done.into_inner().expect("openings"); + + let order = &challenge.order; + let airs = &challenge.airs; + let n = order.len(); + let build = |idx: usize, + air: &crate::VmAir, + trace: &mut TraceTable| + -> Result<(usize, Open), Error> { + let mut transcript = fork(&challenge.transcript, idx, n); + let opening = open_of( + air.as_ref(), + trace, + &challenge.challenges, + &mut transcript, + iotas_of(batched, idx)?, + take_kept(batched, idx), + ) + .map_err(|e| Error::Prover(format!("open pass: table {idx}: {e}")))?; + Ok((idx, opening)) + }; + let fixed: [( + &crate::VmAir, + &mut TraceTable, + ); NUM_FIXED_AIRS] = [ + (&airs.bitwise, &mut resident.bitwise), + (&airs.decode, &mut resident.decode), + (&airs.keccak_rc, &mut resident.accumulated.keccak_rc), + (&airs.register, &mut resident.register), + ]; + let mut jobs: Vec<( + usize, + &crate::VmAir, + &mut TraceTable, + )> = fixed + .into_iter() + .enumerate() + .map(|(idx, (air, trace))| (idx, air, trace)) + .collect(); + if airs.include_halt { + jobs.push((NUM_FIXED_AIRS, &airs.halt, &mut resident.halt)); + } + // Then the accelerators, in `air_trace_pairs` order, skipping the ones + // the run never called: an absent kind has an empty AIR vec and no + // slot. `accel_index` is the single authority for where each lands. + for (slot, (air_vec, trace)) in [ + (&airs.commits, &mut resident.accumulated.commit), + (&airs.keccaks, &mut resident.accumulated.keccak), + (&airs.keccak_rnds, &mut resident.accumulated.keccak_rnd), + (&airs.ecsms, &mut resident.accumulated.ecsm), + (&airs.ecdases, &mut resident.accumulated.ecdas), + (&airs.hints, &mut resident.accumulated.hint), + ] + .into_iter() + .enumerate() + { + if let (Some(air), Some(idx)) = (air_vec.first(), order.accel_index(slot)) { + jobs.push((idx, air, trace)); + } + } + for (i, (air, trace)) in airs.pages.iter().zip(resident.pages.iter_mut()).enumerate() { + let idx = order + .page_index(i) + .ok_or_else(|| Error::Prover(format!("open pass: page {i} is not in the layout")))?; + jobs.push((idx, air, trace)); + } + { + #[cfg(feature = "parallel")] + use rayon::prelude::*; + #[cfg(feature = "parallel")] + let opened = jobs + .into_par_iter() + .map(|(idx, air, trace)| build(idx, air, trace)) + .collect::, _>>()?; + #[cfg(not(feature = "parallel"))] + let opened = jobs + .into_iter() + .map(|(idx, air, trace)| build(idx, air, trace)) + .collect::, _>>()?; + opens.extend(opened); + } + + opens.sort_by_key(|(idx, _)| *idx); + if opens.len() != n { + return Err(Error::Prover(format!( + "open pass: {} tables for a layout of {n}", + opens.len() + ))); + } + Ok(Opened { + openings: opens.into_iter().map(|(_, o)| o).collect(), + resident, + }) +} + +/// Assemble what the batched and Open passes produced. +/// +/// Takes them rather than running them, so the two walks stay independently +/// testable and a caller can measure either on its own. +pub fn assemble_batched_proof(batched: Batched, opened: Opened) -> Result { + if batched.tables.len() != opened.openings.len() { + return Err(Error::Prover(format!( + "assemble: {} tables against {} openings", + batched.tables.len(), + opened.openings.len() + ))); + } + Ok(BatchedProof { + tables: batched.tables, + table_counts: batched.table_counts, + fold_order: batched.fold_order, + openings: opened.openings, + group_of: batched.group_of, + groups: batched.groups, + public_output: opened.resident.public_output, + page_configs: opened.resident.page_configs, + }) +} diff --git a/prover/src/pass.rs b/prover/src/pass.rs new file mode 100644 index 000000000..fb379fee4 --- /dev/null +++ b/prover/src/pass.rs @@ -0,0 +1,453 @@ +//! The walk Approach 1 makes once per pass. +//! +//! The approach goes through the execution more than once: to commit the main +//! traces, to build the auxiliary columns against the challenge that commit +//! produced, and to open the Merkle trees. The three differ only in what they +//! do with a finished table — the walk itself, the end-of-run finalization and +//! the tables that are not built from an op list are the same every time, and +//! live here once. +//! +//! Trading re-execution for memory is the whole bargain: a pass never keeps a +//! chunk it has dealt with, so what it holds is one table plus the tables that +//! cannot be retired. +//! +//! That bound is not O(1) in the length of the run, and the difference is worth +//! being precise about. Four chunked tables — LT, MUL, DVRM and SHIFT — are +//! deliberately absent from [`crate::tables::trace_builder::CHUNKED_KINDS`] +//! because later derivations keep appending to them (MEMW and HINT feed LT, +//! DVRM feeds LT and MUL, CPU32 feeds all three), so their chunk boundaries are +//! not knowable until the run ends. Their op lists, and the `retired_*` buffers +//! a closing MEMW/MEMW_A/CPU32 chunk converts its rows into, are held whole and +//! chunked only in [`finish`]. So is the walk's BITWISE lookup list. Those are +//! compact routed intermediates — tens of bytes per op against the hundreds a +//! trace row costs — and on the measured mainnet block they are a low single +//! digit percentage of the peak, but they grow with the execution, and a +//! workload that sends many wide memory accesses down the general MEMW path +//! (which emits up to eight LT ops each) grows them faster than this one does. + +use stark::proof::options::ProofOptions; + +use crate::Error; +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::{ + DecodeArtifacts, TableKind, Traces, WalkLeftover, build_initial_image, +}; +use crate::tables::{register, types::*}; +use executor::elf::Elf; +use stark::trace::TraceTable; + +/// What a pass does with each table the walk produces. +/// +/// `chunk` numbers the table within its kind and is continuous across the +/// walk's chunks and the tail the end-of-run step pads, so a visitor can index +/// by `(kind, chunk)` without knowing which of the two produced it. +pub trait Visitor { + fn table( + &mut self, + kind: TableKind, + chunk: usize, + trace: TraceTable, + ) -> Result<(), Error>; + + /// Called once after the last table. A visitor that holds tables back to + /// work on several at a time deals with the remainder here. + fn flush(&mut self) -> Result<(), Error> { + Ok(()) + } +} + +/// How many tables a pass works on at a time. +/// +/// The walk hands tables over one by one, and doing the work right there means +/// one table's worth of parallelism on a machine with far more of it. Holding +/// `k` back costs `k` times one table's working set and no more, which is the +/// bargain the whole approach is built on — bounded residency, not minimal. +/// +/// Measured on the ethrex mainnet block, 96 cores, through the LogUp pass: +/// +/// | k | time | peak | +/// |---|---|---| +/// | 1 | 233.4s | 21550 MB | +/// | 4 | 143.5s | 21554 MB | +/// | 8 | 129.9s | 21550 MB | +/// | 16 | 123.5s | 21550 MB | +/// | 32 | 122.3s | 26640 MB | +/// +/// Up to 16 the peak does not move at all, because it is set at the end of the +/// run by the tables that cannot be retired — BITWISE, DECODE, the pages — and +/// by the op lists the walk holds whole (LT, MUL, DVRM, SHIFT and the walk's +/// BITWISE lookups; see this module's header), against which a batch of chunks +/// is small. At 32 the batch itself becomes the peak (it moves to halfway +/// through the run) and buys 1% of time for 5 GB. +/// +/// So the knee is 16 on that machine, and the bound is memory rather than +/// cores: a smaller run has a smaller resident set for a batch to hide behind. +/// `A1_TABLE_PARALLELISM` overrides it. +pub fn table_parallelism() -> usize { + if let Some(k) = std::env::var("A1_TABLE_PARALLELISM") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|k| *k > 0) + { + return k; + } + std::thread::available_parallelism() + .map(|n| (n.get() / 6).clamp(1, 16)) + .unwrap_or(1) +} + +/// How many full batches may wait in the channel between the walk and the +/// worker, beyond the one being processed. `A1_INFLIGHT`, default 0: the walk +/// hands a batch over and is free the moment the worker takes it, so at most +/// `k` tables are being processed while `k` more are being built. +fn inflight() -> usize { + std::env::var("A1_INFLIGHT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(0) +} + +/// `A1_PIPELINE=0` processes each batch inline, blocking the walk — the +/// pre-pipeline behaviour, kept so the two can be measured against each other +/// in one binary. +fn pipelined() -> bool { + std::env::var("A1_PIPELINE") + .map(|v| v != "0") + .unwrap_or(true) +} + +/// Collects tables until there are `k` of them, then hands the batch to a +/// worker thread and keeps walking. +/// +/// The walk is serial — it re-executes the program — and a batch's proving is +/// not, so the two should overlap: while the worker proves batch N the walk +/// builds batch N+1. Measured at k=1 the walk is ~22s of a 246s pass, and with +/// the old inline processing every one of those seconds was spent with the +/// worker idle and vice versa. The bound on how many tables are alive moves +/// from `k` to `k` times the batches in flight, which is the knob a caller has. +/// +/// The action is boxed so a pass names `Batched<'_, Item>` and not a closure +/// type, and it is `Send` because it runs on the worker. +pub struct Batched<'scope, T> { + batch: Vec, + k: usize, + tx: Option>>, + worker: Option>>, + #[allow(clippy::type_complexity)] + inline: Option) -> Result<(), Error> + Send + 'scope>>, +} + +impl<'scope, T: Send + 'scope> Batched<'scope, T> { + pub fn new( + scope: &'scope std::thread::Scope<'scope, '_>, + run: impl FnMut(Vec) -> Result<(), Error> + Send + 'scope, + ) -> Self { + let k = table_parallelism(); + if !pipelined() { + return Self { + batch: Vec::new(), + k, + tx: None, + worker: None, + inline: Some(Box::new(run)), + }; + } + let (tx, rx) = std::sync::mpsc::sync_channel::>(inflight()); + let worker = scope.spawn(move || { + let mut run = run; + for batch in rx { + run(batch)?; + } + Ok(()) + }); + Self { + batch: Vec::new(), + k, + tx: Some(tx), + worker: Some(worker), + inline: None, + } + } + + pub fn push(&mut self, item: T) -> Result<(), Error> { + self.batch.push(item); + if self.batch.len() >= self.k { + return self.hand_over(); + } + Ok(()) + } + + /// Give the current batch to whoever processes it, without waiting for + /// the result. + fn hand_over(&mut self) -> Result<(), Error> { + if self.batch.is_empty() { + return Ok(()); + } + let batch = std::mem::take(&mut self.batch); + if let Some(run) = self.inline.as_mut() { + return run(batch); + } + match self.tx.as_ref() { + Some(tx) => tx.send(batch).map_err(|_| { + // The worker is gone, which means it failed; the real error is + // what `join` returns. + Error::Prover("batched: the worker stopped early".into()) + }), + None => Err(Error::Prover("batched: pushed after finishing".into())), + } + } + + /// Hand over what is left and wait for the worker to finish everything. + pub fn drain(&mut self) -> Result<(), Error> { + let handed = self.hand_over(); + // Closing the channel is what ends the worker's loop. + drop(self.tx.take()); + let joined = match self.worker.take() { + Some(w) => w + .join() + .unwrap_or_else(|_| Err(Error::Prover("batched: the worker panicked".into()))), + None => Ok(()), + }; + // A send failure only ever means the worker had already failed; report + // the worker's error, which says why. + joined.and(handed) + } +} + +/// The tables a pass cannot retire, still as traces. +/// +/// They are the ones not built from an op list — the preprocessed tables, the +/// accumulators and PAGE — so there is no compact intermediate to rebuild them +/// from and nothing to be gained by dropping them. Every pass gets them back +/// and decides what to do with them. +pub struct Resident { + pub bitwise: TraceTable, + pub decode: TraceTable, + pub halt: TraceTable, + pub register: TraceTable, + pub pages: Vec>, + pub page_configs: Vec, + pub accumulated: crate::tables::trace_builder::AccumulatedTables, + /// The bytes the run committed, which the statement binds into the + /// transcript before any root is absorbed. + pub public_output: Vec, +} + +/// Walk the execution once, handing every chunked table to `visitor`. +/// +/// The walk closes a table the moment it fills; what it could not close — the +/// partial tail of each kind, and every chunk of the kinds CPU32 and DVRM keep +/// feeding — is padded by [`finish`] and handed over the same way, numbered +/// where the walk left off. +pub fn run( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, + visitor: &mut V, +) -> Result { + let walked = walk(elf, private_input, max_rows, visitor)?; + finish(walked, private_input, max_rows, visitor) +} + +/// What the walk leaves behind, and what [`finish`] needs to close it out. +/// +/// The image and the decode artifacts are built once and kept because the +/// end-of-run step reads them: rebuilding them there would walk the ELF a +/// second time for no reason. +pub struct Walked { + /// Everything the walk still held when the execution ended. + pub leftover: WalkLeftover, + artifacts: DecodeArtifacts, + image: std::collections::HashMap, + register_init: Vec, +} + +/// The walk alone. +/// +/// Separate from [`finish`] for the caller that wants the leftover itself — +/// which is what shows the walk is retiring as it goes rather than deferring +/// every chunk to the end. +pub fn walk( + elf: &Elf, + private_input: &[u8], + max_rows: &MaxRowsConfig, + visitor: &mut V, +) -> Result { + let image = build_initial_image(elf, private_input); + let register_init = register::register_init_from_entry_point(elf.entry_point); + let artifacts = DecodeArtifacts::from_elf(elf)?; + + let mut failed: Option = None; + let leftover = Traces::walk_and_emit_chunks( + &artifacts, + elf, + private_input.to_vec(), + &image, + ®ister_init, + max_rows, + |kind, chunk, table| { + if failed.is_none() + && let Err(e) = visitor.table(kind, chunk, table) + { + failed = Some(e); + } + }, + )?; + match failed { + Some(e) => Err(e), + None => Ok(Walked { + leftover, + artifacts, + image, + register_init, + }), + } +} + +/// Pad and hand over what the walk could not close, then build the tables that +/// stay. +/// +/// The spec's step after the walk is "at the end of the execution, the +/// remaining tables are padded and commited to". Finalization comes first, as +/// it does in the ordinary build: HALT appends 33 register MEMW ops, and the +/// MEMW-derived LT ops are collected after them so those accesses get their +/// timestamp checks. +pub fn finish( + walked: Walked, + private_input: &[u8], + max_rows: &MaxRowsConfig, + visitor: &mut V, +) -> Result { + let Walked { + mut leftover, + artifacts, + image, + register_init, + } = walked; + leftover.finalize(max_rows); + + // HALT and REGISTER first: REGISTER's final PC token is derived from the CPU + // padding, and the padding of the tail cannot be counted once the tail has + // been drained into a chunk below. + let (halt, register) = leftover.build_halt_and_register(®ister_init)?; + + for kind in ALL_CHUNKED { + // Chunks the walk already closed keep their numbering; what is left + // continues from there. + let first = leftover.emitted(kind); + for (offset, table) in leftover + .take_remaining(kind, max_rows) + .into_iter() + .enumerate() + { + visitor.table(kind, first + offset, table)?; + } + } + visitor.flush()?; + + let public_output = leftover.public_output_bytes(); + let accumulated = leftover.build_accumulated(); + let decode = leftover.build_decode( + artifacts.decode_trace.clone(), + &artifacts.decode_pc_to_row, + max_rows, + ); + // PAGE last: it owes BITWISE lookups of its own, so BITWISE is written only + // once those are in. + let mut hist = leftover.bitwise_histogram(); + let (pages, page_configs) = leftover.build_pages(&image, private_input, &mut hist); + let bitwise = WalkLeftover::build_bitwise_from(&hist); + + Ok(Resident { + bitwise, + decode, + halt, + register, + pages, + page_configs, + accumulated, + public_output, + }) +} + +/// Every chunked table, closable mid-walk or not. +pub const ALL_CHUNKED: [TableKind; 14] = [ + TableKind::Cpu, + TableKind::Memw, + TableKind::MemwAligned, + TableKind::MemwRegister, + TableKind::Load, + TableKind::Cpu32, + TableKind::Branch, + TableKind::Eq, + TableKind::Bytewise, + TableKind::Store, + TableKind::Lt, + TableKind::Mul, + TableKind::Dvrm, + TableKind::Shift, +]; + +/// The AIRs a pass dispatches a chunk to, one per kind. +/// +/// The per-chunk AIRs differ only by the name used in reports — what a table +/// commits to depends on the trace and the domain — so one AIR per kind serves +/// every chunk of that kind, which is what lets a table be dealt with before +/// the number of chunks is known. +pub struct ChunkAirs { + cpu: crate::VmAir, + memw: crate::VmAir, + memw_aligned: crate::VmAir, + memw_register: crate::VmAir, + load: crate::VmAir, + cpu32: crate::VmAir, + branch: crate::VmAir, + eq: crate::VmAir, + bytewise: crate::VmAir, + store: crate::VmAir, + lt: crate::VmAir, + mul: crate::VmAir, + dvrm: crate::VmAir, + shift: crate::VmAir, +} + +impl ChunkAirs { + pub fn new(proof_options: &ProofOptions) -> Self { + use crate::test_utils::*; + Self { + cpu: Box::new(create_cpu_air(proof_options)), + memw: Box::new(create_memw_air(proof_options)), + memw_aligned: Box::new(create_memw_aligned_air(proof_options)), + memw_register: Box::new(create_memw_register_air(proof_options)), + load: Box::new(create_load_air(proof_options)), + cpu32: Box::new(create_cpu32_air(proof_options)), + branch: Box::new(create_branch_air(proof_options)), + eq: Box::new(create_eq_air(proof_options)), + bytewise: Box::new(create_bytewise_air(proof_options)), + store: Box::new(create_store_air(proof_options)), + lt: Box::new(create_lt_air(proof_options)), + mul: Box::new(create_mul_air(proof_options)), + dvrm: Box::new(create_dvrm_air(proof_options)), + shift: Box::new(create_shift_air(proof_options)), + } + } + + pub fn get(&self, kind: TableKind) -> &crate::VmAir { + match kind { + TableKind::Cpu => &self.cpu, + TableKind::Memw => &self.memw, + TableKind::MemwAligned => &self.memw_aligned, + TableKind::MemwRegister => &self.memw_register, + TableKind::Load => &self.load, + TableKind::Cpu32 => &self.cpu32, + TableKind::Branch => &self.branch, + TableKind::Eq => &self.eq, + TableKind::Bytewise => &self.bytewise, + TableKind::Store => &self.store, + TableKind::Lt => &self.lt, + TableKind::Mul => &self.mul, + TableKind::Dvrm => &self.dvrm, + TableKind::Shift => &self.shift, + } + } +} diff --git a/prover/src/recursion.rs b/prover/src/recursion.rs index efca722c9..39c04edbf 100644 --- a/prover/src/recursion.rs +++ b/prover/src/recursion.rs @@ -193,6 +193,135 @@ pub fn encode_continuation_guest_input( Ok(blob) } +/// [`verify_and_attest_blob`], but the monolithic blob is deserialized into +/// owned values first instead of being read in place. +/// +/// Same proof, same verifier, same attestation — only the read path differs. +/// Zero-copy saves the guest a full owned copy of the proof; what it costs is +/// an archived-pointer dereference on every field access, and the verifier +/// touches most fields many times. This exists to price that trade in guest +/// cycles, which is what an outer prover pays for. +pub fn verify_owned_and_attest( + blob: &[u8], + proof_options: &ProofOptions, +) -> Result>, Error> { + use rkyv::rancor::Error as RkyvError; + + let archive_bytes = crate::recursion_archive_bytes(blob) + .ok_or_else(|| Error::Execution(String::from("recursion blob: bad magic or version")))?; + let mut aligned_fallback = rkyv::util::AlignedVec::<{ crate::RECURSION_INPUT_ALIGN }>::new(); + let archive: &[u8] = + if (archive_bytes.as_ptr() as usize).is_multiple_of(crate::RECURSION_INPUT_ALIGN) { + archive_bytes + } else { + aligned_fallback.extend_from_slice(archive_bytes); + &aligned_fallback + }; + let input = rkyv::from_bytes::(archive) + .map_err(|e| Error::Execution(format!("blob validation failed: {e}")))?; + + let ok = crate::verify_with_options( + &input.vm_proof, + &input.inner_elf, + proof_options, + Some(input.decode_commitment), + Some(&input.page_commitments), + )?; + if !ok { + return Ok(None); + } + + let id = program_id_from_elf( + &input.inner_elf, + &input.decode_commitment, + &input.page_commitments, + )?; + let mut attestation = id.to_vec(); + attestation.extend_from_slice(&input.vm_proof.public_output); + Ok(Some(attestation)) +} + +/// The batched guest's private-input layout (the `batched` guest feature). +/// Mirrors [`crate::GuestInput`] with the monolithic proof replaced by a +/// [`BatchedProof`]: one FRI per domain height instead of one per table. +/// Rkyv-archived on the same magic-prefixed wire format as the other two +/// blobs; the guest is feature-pinned to one layout, and a blob of another +/// kind fails the bytecheck validation. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct BatchedGuestInput { + pub proof: crate::batched_proof::BatchedProof, + pub inner_elf: Vec, + pub decode_commitment: Commitment, + pub page_commitments: Vec<(u64, Commitment)>, +} + +/// Build the batched guest's private-input blob for `proof` of `inner_elf`. +pub fn encode_batched_guest_input( + proof: crate::batched_proof::BatchedProof, + inner_elf: &[u8], + opts: &ProofOptions, +) -> Result, Error> { + let (decode_commitment, page_commitments) = precomputed_commitments(inner_elf, opts)?; + let input = BatchedGuestInput { + proof, + inner_elf: inner_elf.to_vec(), + decode_commitment, + page_commitments, + }; + let archive = rkyv::to_bytes::(&input) + .map_err(|e| Error::Execution(format!("rkyv encode failed: {e}")))?; + let mut blob = Vec::with_capacity(crate::RECURSION_INPUT_PREFIX_LEN + archive.len()); + blob.extend_from_slice(&crate::RECURSION_INPUT_MAGIC); + blob.extend_from_slice(&crate::RECURSION_INPUT_VERSION.to_le_bytes()); + blob.extend_from_slice(&[0u8; 4]); // reserved + blob.extend_from_slice(&archive); + Ok(blob) +} + +/// [`verify_and_attest_blob`]'s logic for a batched proof: verify it against +/// the supplied roots and attest `program_id(elf, roots) || public_output`. +/// The batched verifier works on owned values, so the archive is deserialized +/// once rather than read in place. +pub fn verify_batched_and_attest( + blob: &[u8], + proof_options: &ProofOptions, +) -> Result>, Error> { + use rkyv::rancor::Error as RkyvError; + + let archive_bytes = crate::recursion_archive_bytes(blob).ok_or_else(|| { + Error::Execution(String::from("batched recursion blob: bad magic or version")) + })?; + let mut aligned_fallback = rkyv::util::AlignedVec::<{ crate::RECURSION_INPUT_ALIGN }>::new(); + let archive: &[u8] = + if (archive_bytes.as_ptr() as usize).is_multiple_of(crate::RECURSION_INPUT_ALIGN) { + archive_bytes + } else { + aligned_fallback.extend_from_slice(archive_bytes); + &aligned_fallback + }; + let input = rkyv::from_bytes::(archive) + .map_err(|e| Error::Execution(format!("batched blob validation failed: {e}")))?; + + if !crate::batched_verifier::verify_with_precomputed( + &input.proof, + &input.inner_elf, + proof_options, + Some(input.decode_commitment), + Some(&input.page_commitments), + )? { + return Ok(None); + } + + let id = program_id_from_elf( + &input.inner_elf, + &input.decode_commitment, + &input.page_commitments, + )?; + let mut attestation = id.to_vec(); + attestation.extend_from_slice(&input.proof.public_output); + Ok(Some(attestation)) +} + /// Domain tag for [`program_id`]. const PROGRAM_ID_TAG: &[u8] = b"LAMBDAVM_PROGRAM_ID_V1"; diff --git a/prover/src/streaming.rs b/prover/src/streaming.rs new file mode 100644 index 000000000..10740036e --- /dev/null +++ b/prover/src/streaming.rs @@ -0,0 +1,257 @@ +//! On-demand trace source for the streaming prover. +//! +//! Approach 1 step C.2b: the chunked tables are built as empty placeholders and +//! their rows live only as the routed op lists they came from. The prover asks +//! this provider for a trace at the two points it needs one — the Round 1 main +//! commit, and the table's fused chain — and drops it again after each. + +use std::collections::HashMap; +use std::sync::Mutex; + +use stark::prover::TraceProvider; +use stark::trace::TraceTable; + +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::{CollectedOps, TableKind, Traces}; +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; + +/// The groups of chunked tables, in the order `VmAirs::air_trace_pairs` emits +/// them. `None` marks a group that stays resident (PAGE), which still consumes +/// AIR indices and so must be walked over. +pub(crate) const GROUP_ORDER: [Option; 15] = [ + Some(TableKind::Cpu), + Some(TableKind::Lt), + Some(TableKind::Shift), + Some(TableKind::Memw), + Some(TableKind::MemwAligned), + Some(TableKind::Load), + Some(TableKind::Mul), + Some(TableKind::Dvrm), + Some(TableKind::Branch), + None, // PAGE — built from the ELF image, not from an op list + Some(TableKind::MemwRegister), + Some(TableKind::Eq), + Some(TableKind::Bytewise), + Some(TableKind::Store), + Some(TableKind::Cpu32), +]; + +/// Number of singleton tables emitted before HALT: BITWISE, DECODE, KECCAK_RC, +/// REGISTER. +/// +/// The six accelerators used to sit here too. #977 moved them after HALT, as +/// groups reporting 0 or 1 — `TableCounts::validate` rejects more — so a run +/// that never calls one pays no table for it. `accel_lengths` is where that +/// order lives now. +pub(crate) const NUM_FIXED_AIRS: usize = 4; + +/// The accelerator counts in the order `VmAirs::air_trace_pairs` emits them, +/// between HALT and the chunked groups. The order is the protocol. +pub(crate) fn accel_lengths(counts: &crate::TableCounts) -> [usize; 6] { + [ + counts.commit, + counts.keccak, + counts.keccak_rnd, + counts.ecsm, + counts.ecdas, + counts.hint, + ] +} + +/// Where each table sits in `VmAirs::air_trace_pairs`. +/// +/// The order is the protocol — the transcript absorbs roots in it, and each +/// table's own fork is domain-separated by its index — so every pass has to +/// agree on it. A pass that walks the execution produces chunks in the order +/// they close, which is not this order, so it needs to be able to ask. +/// +/// Knowable before the second walk because the first one already counted the +/// chunks. +pub(crate) struct AirOrder { + counts: crate::TableCounts, + include_halt: bool, + num_pages: usize, +} + +impl AirOrder { + pub(crate) fn new(counts: crate::TableCounts, include_halt: bool, num_pages: usize) -> Self { + Self { + counts, + include_halt, + num_pages, + } + } + + /// The index of the first chunked table, after the fixed ones and HALT. + fn first_chunked(&self) -> usize { + NUM_FIXED_AIRS + + usize::from(self.include_halt) + + accel_lengths(&self.counts).iter().sum::() + } + + /// The AIR index of accelerator `slot` (0..6, in `accel_lengths` order), or + /// `None` when the run produced no table for it. + pub(crate) fn accel_index(&self, slot: usize) -> Option { + let lens = accel_lengths(&self.counts); + if lens.get(slot).copied().unwrap_or(0) == 0 { + return None; + } + let before: usize = lens[..slot].iter().sum(); + Some(NUM_FIXED_AIRS + usize::from(self.include_halt) + before) + } + + fn group_len(&self, group: Option) -> usize { + match group { + None => self.num_pages, + Some(kind) => crate::challenge_phase::count_for(&self.counts, kind), + } + } + + /// The AIR index of a chunk, or `None` when the layout has no such chunk. + pub(crate) fn index_of(&self, kind: TableKind, chunk: usize) -> Option { + let mut idx = self.first_chunked(); + for group in GROUP_ORDER { + let len = self.group_len(group); + if group == Some(kind) { + return (chunk < len).then_some(idx + chunk); + } + idx += len; + } + None + } + + /// The AIR index of the `i`th PAGE table. + pub(crate) fn page_index(&self, i: usize) -> Option { + let mut idx = self.first_chunked(); + for group in GROUP_ORDER { + if group.is_none() { + return (i < self.num_pages).then_some(idx + i); + } + idx += self.group_len(group); + } + None + } + + pub fn counts(&self) -> &crate::TableCounts { + &self.counts + } + + /// How many tables the layout has in total. + pub(crate) fn len(&self) -> usize { + self.first_chunked() + + GROUP_ORDER + .iter() + .map(|g| self.group_len(*g)) + .sum::() + } +} + +pub(crate) struct StreamingProvider { + routed: CollectedOps, + max_rows: MaxRowsConfig, + /// AIR index -> the chunk that rebuilds it, or `None` when it is resident. + slots: Vec>, + /// Memoized `(rows, main_columns)` per retired AIR index. Only the shape is + /// cached — caching the trace would give back the memory this mode exists + /// to save. + shapes: Mutex>, +} + +impl StreamingProvider { + /// Walk the AIR order and record which index each retired chunk answers to. + /// + /// `include_halt` is not a detail: HALT sits between the fixed tables and + /// the chunked groups and is emitted only for a final epoch, so getting it + /// wrong shifts every slot by one and hands each table the trace of its + /// neighbour. It is taken from the caller's `VmAirs` rather than assumed. + pub(crate) fn new( + routed: CollectedOps, + max_rows: MaxRowsConfig, + traces: &Traces, + include_halt: bool, + ) -> Self { + let group_lengths = [ + traces.cpus.len(), + traces.lts.len(), + traces.shifts.len(), + traces.memws.len(), + traces.memw_aligneds.len(), + traces.loads.len(), + traces.muls.len(), + traces.dvrms.len(), + traces.branches.len(), + traces.pages.len(), + traces.memw_registers.len(), + traces.eqs.len(), + traces.bytewises.len(), + traces.stores.len(), + traces.cpu32s.len(), + ]; + + // The fixed singletons, HALT, then one slot per accelerator the run + // produced: all resident, so all `None`, but they shift every chunked + // slot after them. + let accel_slots = traces.commits.len() + + traces.keccaks.len() + + traces.keccak_rnds.len() + + traces.ecsms.len() + + traces.ecdases.len() + + traces.hints.len(); + let mut slots = vec![None; NUM_FIXED_AIRS + usize::from(include_halt) + accel_slots]; + for (kind, len) in GROUP_ORDER.iter().zip(group_lengths.iter()) { + for chunk in 0..*len { + slots.push(kind.map(|k| (k, chunk))); + } + } + + Self { + routed, + max_rows, + slots, + shapes: Mutex::new(HashMap::new()), + } + } + + fn slot(&self, idx: usize) -> Option<(TableKind, usize)> { + self.slots.get(idx).copied().flatten() + } + + /// Rows and width of a retired chunk, without building it. + /// + /// `chunk_shape` derives both from the op counts and the table's constant + /// width, so the pre-pass and the memory estimates no longer pay a full + /// trace generation each just to learn a row count. Still memoized: for the + /// deduplicating tables it is a counting pass, not free. + fn shape(&self, idx: usize) -> (usize, usize) { + if let Some(hit) = self.shapes.lock().unwrap().get(&idx) { + return *hit; + } + let (kind, chunk) = self.slot(idx).expect("shape asked for a resident table"); + let shape = self.routed.chunk_shape(kind, chunk, &self.max_rows); + #[cfg(feature = "instruments")] + stark::instruments::count_retired_shape_query(); + self.shapes.lock().unwrap().insert(idx, shape); + shape + } +} + +impl TraceProvider for StreamingProvider { + fn is_retired(&self, idx: usize) -> bool { + self.slot(idx).is_some() + } + + fn num_rows(&self, idx: usize) -> usize { + self.shape(idx).0 + } + + fn num_main_columns(&self, idx: usize) -> usize { + self.shape(idx).1 + } + + fn build_main(&self, idx: usize) -> TraceTable { + let (kind, chunk) = self.slot(idx).expect("build asked for a resident table"); + #[cfg(feature = "instruments")] + stark::instruments::count_retired_trace_build(); + self.routed.build_chunk(kind, chunk, &self.max_rows) + } +} diff --git a/prover/src/tables/bitwise.rs b/prover/src/tables/bitwise.rs index c73e1e341..6e921c016 100644 --- a/prover/src/tables/bitwise.rs +++ b/prover/src/tables/bitwise.rs @@ -523,6 +523,7 @@ const _: () = { /// [`update_multiplicities`] produces (both just sum the same lookups per cell). /// /// Memory: `NUM_ROWS * NUM_LOOKUP_TYPES * 8` bytes = 2^20 * 10 * 8 = 80 MiB. +#[derive(PartialEq, Eq)] pub(crate) struct BitwiseHistogram { counters: Box<[u64]>, } diff --git a/prover/src/tables/branch.rs b/prover/src/tables/branch.rs index 0d3c2e206..24776caca 100644 --- a/prover/src/tables/branch.rs +++ b/prover/src/tables/branch.rs @@ -106,7 +106,7 @@ const MASK_254: u64 = 254; /// A single BRANCH operation to be added to the trace. /// /// Derives Hash and Eq so it can be used as a HashMap key for deduplication. -#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct BranchOperation { /// Current program counter (64-bit) pub pc: u64, @@ -163,7 +163,12 @@ pub fn generate_branch_trace( *op_map.entry(op.clone()).or_insert(0) += 1; } - let unique_ops: Vec<_> = op_map.into_iter().collect(); + // Sorted, not `HashMap` order: std randomizes iteration per instance, so two + // builds of the same logs produced the same rows in a different order. Harmless + // for a single build, fatal for rebuilding a retired trace — the rebuilt table + // must hash to the root its first build committed. + let mut unique_ops: Vec<_> = op_map.into_iter().collect(); + unique_ops.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let num_rows = unique_ops.len().next_power_of_two().max(4); let mut trace = TraceTable::new_main( crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), diff --git a/prover/src/tables/bytewise.rs b/prover/src/tables/bytewise.rs index 2808365c6..a608a6a80 100644 --- a/prover/src/tables/bytewise.rs +++ b/prover/src/tables/bytewise.rs @@ -47,7 +47,7 @@ pub mod cols { // ========================================================================= /// A single BYTEWISE operation. `op` is an [`alu_op`] opcode in {AND, OR, XOR}. -#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct BytewiseOperation { pub a: u64, pub b: u64, @@ -104,7 +104,12 @@ pub fn generate_bytewise_trace( *op_map.entry(op.clone()).or_insert(0) += 1; } - let unique_ops: Vec<_> = op_map.into_iter().collect(); + // Sorted, not `HashMap` order: std randomizes iteration per instance, so two + // builds of the same logs produced the same rows in a different order. Harmless + // for a single build, fatal for rebuilding a retired trace — the rebuilt table + // must hash to the root its first build committed. + let mut unique_ops: Vec<_> = op_map.into_iter().collect(); + unique_ops.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let num_rows = unique_ops.len().next_power_of_two().max(4); let mut trace = TraceTable::new_main( crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), diff --git a/prover/src/tables/decode.rs b/prover/src/tables/decode.rs index bfd1ddb90..df197de27 100644 --- a/prover/src/tables/decode.rs +++ b/prover/src/tables/decode.rs @@ -172,6 +172,27 @@ pub fn generate_decode_trace( (trace, pc_to_row) } +/// Add `count` lookups of `pc` at once. +/// +/// The per-lookup form needs one entry per executed cycle, which a prover that +/// walks the execution and drops what it has proved cannot keep. Counting by pc +/// costs one entry per distinct program counter instead — bounded by the +/// program, not by how long it runs. +pub fn add_multiplicities( + trace: &mut TraceTable, + pc_to_row: &PcToRow, + counts: &std::collections::HashMap, +) { + for (pc, count) in counts { + if let Some(&row_idx) = pc_to_row.get(pc) { + let current = trace.main_table.get(row_idx, cols::MU); + trace + .main_table + .set_fe(row_idx, cols::MU, current + FE::from(*count)); + } + } +} + /// Updates multiplicities in the DECODE trace table. /// /// For each PC in `lookups`, increments the MU column in the corresponding row. diff --git a/prover/src/tables/dvrm.rs b/prover/src/tables/dvrm.rs index c499a72bf..32175166f 100644 --- a/prover/src/tables/dvrm.rs +++ b/prover/src/tables/dvrm.rs @@ -152,7 +152,7 @@ const SIGN_FILL: u64 = 0xFFFF; /// A single DVRM operation to be added to the trace. /// /// Derives Hash and Eq for HashMap-based deduplication. -#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct DvrmOperation { /// Numerator (64-bit) pub n: u64, @@ -295,7 +295,12 @@ pub fn generate_dvrm_trace( } } - let unique_ops: Vec<_> = op_map.into_iter().collect(); + // Sorted, not `HashMap` order: std randomizes iteration per instance, so two + // builds of the same logs produced the same rows in a different order. Harmless + // for a single build, fatal for rebuilding a retired trace — the rebuilt table + // must hash to the root its first build committed. + let mut unique_ops: Vec<_> = op_map.into_iter().collect(); + unique_ops.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let num_rows = unique_ops.len().next_power_of_two().max(4); let mut trace = TraceTable::new_main( crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), diff --git a/prover/src/tables/eq.rs b/prover/src/tables/eq.rs index f967becf4..bc6046f6e 100644 --- a/prover/src/tables/eq.rs +++ b/prover/src/tables/eq.rs @@ -64,7 +64,7 @@ pub mod cols { // ========================================================================= /// A single EQ operation. -#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct EqOperation { /// First operand (64-bit) pub a: u64, @@ -125,7 +125,12 @@ pub fn generate_eq_trace( *op_map.entry(op.clone()).or_insert(0) += 1; } - let unique_ops: Vec<_> = op_map.into_iter().collect(); + // Sorted, not `HashMap` order: std randomizes iteration per instance, so two + // builds of the same logs produced the same rows in a different order. Harmless + // for a single build, fatal for rebuilding a retired trace — the rebuilt table + // must hash to the root its first build committed. + let mut unique_ops: Vec<_> = op_map.into_iter().collect(); + unique_ops.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let num_rows = unique_ops.len().next_power_of_two().max(4); let mut trace = TraceTable::new_main( crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), diff --git a/prover/src/tables/lt.rs b/prover/src/tables/lt.rs index fb7d34267..aad3e0796 100644 --- a/prover/src/tables/lt.rs +++ b/prover/src/tables/lt.rs @@ -104,7 +104,7 @@ pub mod cols { /// from the inverted form (`BGE[U]`). /// /// Derives Hash and Eq so it can be used as a HashMap key for deduplication. -#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct LtOperation { /// Left operand (64-bit value) pub lhs: u64, @@ -165,7 +165,12 @@ pub fn generate_lt_trace( *op_map.entry(op.clone()).or_insert(0) += 1; } - let unique_ops: Vec<_> = op_map.into_iter().collect(); + // Sorted, not `HashMap` order: std randomizes iteration per instance, so two + // builds of the same logs produced the same rows in a different order. Harmless + // for a single build, fatal for rebuilding a retired trace — the rebuilt table + // must hash to the root its first build committed. + let mut unique_ops: Vec<_> = op_map.into_iter().collect(); + unique_ops.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let num_rows = unique_ops.len().next_power_of_two().max(4); let mut trace = TraceTable::new_main( crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), diff --git a/prover/src/tables/mul.rs b/prover/src/tables/mul.rs index a615f74df..48e3a48c2 100644 --- a/prover/src/tables/mul.rs +++ b/prover/src/tables/mul.rs @@ -144,7 +144,7 @@ const SIGN_FILL: u64 = 0xFFFF; /// the sender's `flags` byte at lookup time. /// /// Derives Hash and Eq for HashMap-based deduplication. -#[derive(Debug, Clone, Hash, PartialEq, Eq)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct MulOperation { /// Left operand (64-bit) pub lhs: u64, @@ -303,7 +303,12 @@ pub fn generate_mul_trace( } } - let unique_ops: Vec<_> = op_map.into_iter().collect(); + // Sorted, not `HashMap` order: std randomizes iteration per instance, so two + // builds of the same logs produced the same rows in a different order. Harmless + // for a single build, fatal for rebuilding a retired trace — the rebuilt table + // must hash to the root its first build committed. + let mut unique_ops: Vec<_> = op_map.into_iter().collect(); + unique_ops.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); let num_rows = unique_ops.len().next_power_of_two().max(4); let mut trace = TraceTable::new_main( crate::tables::types::zeroed_fe_vec(num_rows * cols::NUM_COLUMNS), diff --git a/prover/src/tables/page.rs b/prover/src/tables/page.rs index 6788bee08..6c2425839 100644 --- a/prover/src/tables/page.rs +++ b/prover/src/tables/page.rs @@ -111,7 +111,7 @@ pub struct FinalByteState { pub type FinalStateMap = HashMap; /// Configuration for a single PAGE table instance. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] pub struct PageConfig { /// Base address of this page (must be page-aligned). pub page_base: u64, diff --git a/prover/src/tables/trace_builder.rs b/prover/src/tables/trace_builder.rs index c3b695a80..fdc441ec9 100644 --- a/prover/src/tables/trace_builder.rs +++ b/prover/src/tables/trace_builder.rs @@ -81,7 +81,7 @@ type MemoryCell = (u8, u64); type RegisterCell = (u64, u64); /// Memory state tracker for generating MEMW/LOAD traces. -struct MemoryState { +pub(crate) struct MemoryState { /// Per byte-address `(value, timestamp)`, as a dense per-page store. This is /// the hot structure — `read_byte`/`write_byte` hit it on every memory access /// during the replay, and it's rebuilt each epoch — so a per-page array (small @@ -154,7 +154,7 @@ impl MemoryState { } /// Register state tracker for generating MEMW register traces. -struct RegisterState { +pub(crate) struct RegisterState { /// Register file: (value, last_write_timestamp) regs: [RegisterCell; 32], /// Synthetic x254 commit index register: (value, last_write_timestamp) @@ -336,6 +336,11 @@ fn pack_register_value(value: u64) -> [u32; 8] { fn collect_cpu_ops( logs: &[Log], instructions: &U64HashMap, + // Index of `logs[0]` within the whole execution. Zero when the caller holds + // every log; the running count when it is walking the execution in pieces, + // since the timestamp comes from the cycle's position and restarting it per + // piece would silently rewind time. + first_cycle: usize, ) -> Result, Error> { let mut cpu_ops = Vec::with_capacity(logs.len()); @@ -345,7 +350,7 @@ fn collect_cpu_ops( // Exactly 4 so that inline PC's prev_ts = timestamp - 3 = 1 on the first row, // matching the REGISTER table's initial PC token at timestamp 1 (per spec/memory.typ). for (i, log) in logs.iter().enumerate() { - let timestamp = (i as u64) * 4 + 4; + let timestamp = ((first_cycle + i) as u64) * 4 + 4; let instruction = instructions .get(&log.current_pc) .copied() @@ -409,6 +414,15 @@ struct MemwBuckets { } impl MemwBuckets { + /// Append another segment's buckets. Order is preserved, so collecting an + /// execution in pieces and appending them yields exactly what collecting it + /// whole would have. + fn append(&mut self, mut other: Self) { + self.register_rows.append(&mut other.register_rows); + self.aligned.append(&mut other.aligned); + self.general.append(&mut other.general); + } + fn with_register_capacity(n: usize) -> Self { Self { register_rows: Vec::with_capacity(n), @@ -1235,9 +1249,415 @@ fn collect_cpu32_bitwise(c: &cpu32::Cpu32Operation) -> Vec { ops } +/// What a Commit-phase walk still holds when the execution ends. +/// +/// The chunks it closed are gone — committed and dropped as they filled. This +/// is the rest: every table's partial tail, the tables the walk cannot close, +/// and the end state the finalization needs. The spec's "remaining tables are +/// padded and committed" operates on exactly this. +pub struct WalkLeftover { + /// Ops not yet turned into a committed chunk, including the lists the walk + /// never closes: the accumulators, and the tables CPU32 and DVRM still feed. + pub(crate) tail: CollectedOps, + /// Chunks already emitted per kind, indexed like [`CHUNKED_KINDS`], so the + /// tail's chunk numbering continues where the walk stopped. + pub(crate) emitted: [usize; CHUNKED_KINDS.len()], + /// BITWISE lookups owed by the chunks the walk closed and dropped. The + /// end-of-run phase folds the rest in on top of this. + pub(crate) retired_bitwise: bitwise::BitwiseHistogram, + /// DECODE lookups per program counter, counted rather than listed. + pub(crate) decode_counts: HashMap, + /// Padding rows the CPU chunks closed so far added, each of which looks + /// DECODE up at the padding pc. + pub(crate) padding_rows: usize, + /// CPU padding rows over the whole run, frozen by `finalize` while the tail + /// is still intact. + pub(crate) total_cpu_padding: Option, + /// Timestamp and next pc of the run's last ECALL, which HALT is built from. + pub(crate) last_ecall: Option<(u64, u64)>, + /// Memory at the last cycle, which the PAGE build reads. + pub(crate) memory_state: MemoryState, + /// Register state at the last cycle. The end-of-run finalization is driven + /// from it — HALT appends 33 register MEMW ops at `u64::MAX` — so the phase + /// that pads and commits the tails needs nothing else from the run. + /// + /// The memory state is not carried: the only thing that reads it is the + /// PAGE build, which is part of the preprocessed step that does not exist + /// yet, and a field nobody reads is a field that quietly goes wrong. + pub(crate) register_state: RegisterState, + /// Cycles executed. + pub(crate) cycles: usize, +} + +impl WalkLeftover { + /// Apply the end-of-run finalization and the routing that depends on it. + /// + /// HALT appends 33 register MEMW ops at `u64::MAX`, and those need their + /// timestamp checks like any other access, so the MEMW-derived LT ops are + /// collected after them — the same order `build_traces` uses, where + /// finalization runs before phase 3. + pub(crate) fn finalize(&mut self, max_rows: &super::MaxRowsConfig) { + // Freeze the CPU padding here, while the tail is still whole. Both HALT's + // register token and DECODE's padding lookups are derived from it, and + // both are built after the tails have been drained into chunks — reading + // it then would count a tail that no longer exists. + let tail = self.tail.cpu_ops.len(); + self.total_cpu_padding = Some(if tail == 0 && self.emitted(TableKind::Cpu) > 0 { + // An empty tail is not a chunk: `ops.chunks(n)` over a length that + // divides evenly yields no trailing empty one. + self.padding_rows + } else { + self.padding_rows + tail.next_power_of_two().max(4) - tail + }); + + let halt = collect_halt_ops(&mut self.register_state); + let mut buckets = MemwBuckets::with_register_capacity(halt.len()); + buckets.extend_ops(halt); + self.tail.memw_register_rows.extend(buckets.register_rows); + self.tail.memw_aligned_ops.extend(buckets.aligned); + self.tail.memw_ops.extend(buckets.general); + + // What the ordinary build derives after the CPU pass, in its order. CPU32 + // rows dispatch to SHIFT, MUL and DVRM: the retired chunks' first, then the + // tail's, after everything the CPU itself sent. + let tail_cpu32 = self.tail.cpu32_ops.len(); + for c in &self.tail.cpu32_ops[..tail_cpu32] { + cpu32_chip_op( + c, + &mut self.tail.retired_cpu32_shift, + &mut self.tail.retired_cpu32_mul, + &mut self.tail.retired_cpu32_dvrm, + ); + } + let shift = std::mem::take(&mut self.tail.retired_cpu32_shift); + self.tail.shift_ops.extend(shift); + let mul = std::mem::take(&mut self.tail.retired_cpu32_mul); + self.tail.mul_ops.extend(mul); + let dvrm = std::mem::take(&mut self.tail.retired_cpu32_dvrm); + self.tail.dvrm_ops.extend(dvrm); + // Every DVRM op owes LT |r| < |d| and MUL d * q, lo and hi. + for (op, _wants_remainder) in &self.tail.dvrm_ops { + self.tail + .lt_ops + .push(LtOperation::new(op.abs_r(), op.abs_d(), false)); + } + for (op, _wants_remainder) in &self.tail.dvrm_ops { + let mul_op = MulOperation::new(op.d, op.signed, op.compute_quotient(), op.sign_q()); + self.tail.mul_ops.push((mul_op.clone(), false)); + self.tail.mul_ops.push((mul_op, true)); + } + + // MEMW's timestamp checks are LT rows. The chunks retired during the + // walk left theirs behind; the tail's are derived now. Same order as the + // ordinary build: every general MEMW op, then every aligned one. + let retired = std::mem::take(&mut self.tail.retired_memw_lt); + self.tail.lt_ops.extend(retired); + self.tail + .lt_ops + .extend(collect_lt_from_memw(&self.tail.memw_ops)); + let retired = std::mem::take(&mut self.tail.retired_memw_aligned_lt); + self.tail.lt_ops.extend(retired); + self.tail + .lt_ops + .extend(collect_lt_from_memw_aligned(&self.tail.memw_aligned_ops)); + // HINT's range checks, last of all: selector and both address low limbs. + self.tail + .lt_ops + .extend(self.tail.hint_ops.iter().flat_map(|op| { + [ + LtOperation::new(op.hint_id, hint::HINT_SELECTOR_BOUND, false), + LtOperation::new(op.in_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), + LtOperation::new(op.out_addr & 0xFFFF_FFFF, hint::HINT_ADDR_LIMB_BOUND, false), + ] + })); + + // Fold the tail's own BITWISE lookups in now, while the tail is whole. + // The retired chunks contributed theirs as they closed; from here the + // histogram is complete and draining the tail cannot change it. + let mut hist = + std::mem::replace(&mut self.retired_bitwise, bitwise::BitwiseHistogram::new()); + for kind in [ + TableKind::MemwAligned, + TableKind::MemwRegister, + TableKind::Branch, + TableKind::Bytewise, + TableKind::Eq, + TableKind::Store, + TableKind::Cpu32, + ] { + let n = self.tail.buffered(kind); + self.tail.fold_bitwise_from_front(kind, n, &mut hist); + } + // The sources nothing ever retires, so their whole list is here. + hist.add_ops(&collect_bitwise_from_lt(&self.tail.lt_ops)); + hist.add_ops(&collect_bitwise_from_mul(&self.tail.mul_ops, max_rows.mul)); + hist.add_ops(&collect_bitwise_from_dvrm( + &self.tail.dvrm_ops, + max_rows.dvrm, + )); + hist.add_ops(&shift::collect_bitwise_from_shift(&self.tail.shift_ops)); + hist.add_ops(&collect_bitwise_from_commit(&self.tail.commit_ops)); + hist.add_ops(&collect_bitwise_from_keccak(&self.tail.keccak_ops)); + hist.add_ops(&collect_bitwise_from_ecsm(&self.tail.ecsm_ops)); + hist.add_ops(&collect_bitwise_from_ecdas(&self.tail.ecdas_ops)); + hist.add_ops(&collect_bitwise_from_hint(&self.tail.hint_ops)); + // CPU padding rows send ARE_BYTES with all-zero values. + add_padding_byte_checks(&mut hist, self.cpu_padding_rows()); + // The lookups the walk itself collected while routing. + hist.add_ops(&self.tail.bitwise_ops); + self.retired_bitwise = hist; + } + + /// Build every chunk still held for `kind`, draining it. + /// + /// An empty list still yields one padded chunk when the walk never closed + /// any, matching `chunk_and_generate`: the table exists in the proof with + /// the shape the verifier expects. + pub(crate) fn take_remaining( + &mut self, + kind: TableKind, + max_rows: &super::MaxRowsConfig, + ) -> Vec> { + let limit = max_rows_for(kind, max_rows); + let mut out = Vec::new(); + while self.tail.buffered(kind) > limit { + out.push(self.tail.take_front(kind, limit, max_rows)); + } + let left = self.tail.buffered(kind); + // A kind the run never used gets no table at all since #977: the ordinary + // build's `chunk_and_generate_optional` emits zero chunks for an empty op + // list, and the two sides have to declare the same table set. CPU and + // MEMW_R are the exception — they are structurally required, so they keep + // the padded chunk. `skips_when_empty` is the single predicate, shared + // with `num_chunks` and `build_table`. + let padded_chunk = + out.is_empty() && self.emitted(kind) == 0 && !CollectedOps::skips_when_empty(kind); + if left > 0 || padded_chunk { + out.push(self.tail.take_front(kind, left, max_rows)); + } + out + } + + /// Build the BITWISE table from what the run owes it. + /// + /// The lookups of the chunks the walk retired were folded in as they were + /// dropped; the tables still held contribute here. BITWISE is a fixed table + /// whose rows are the lookup space — only its multiplicity columns depend + /// on the run — so it is built once, at the end, and never chunked. + /// + /// Every BITWISE lookup the run owes, from the retired chunks and from the + /// tail — `finalize` folded both in, so this is complete whatever has been + /// drained since. PAGE's own lookups are added by `build_pages`. + pub(crate) fn bitwise_histogram(&self) -> bitwise::BitwiseHistogram { + let mut hist = bitwise::BitwiseHistogram::new(); + hist.merge(&self.retired_bitwise); + hist + } + + /// Fill BITWISE's multiplicity columns from a histogram. + /// + /// Taken separately from [`bitwise_histogram`](Self::bitwise_histogram) so + /// PAGE — which owes BITWISE its own lookups and is built later, from the + /// memory image — can fold them in before the table is written. + pub(crate) fn build_bitwise_from( + hist: &bitwise::BitwiseHistogram, + ) -> TraceTable { + let mut table = bitwise::generate_bitwise_trace(); + hist.fill_multiplicities(&mut table); + table + } + + /// Build the tables that are a function of one accumulated op list. + /// + /// COMMIT, KECCAK and its two round tables, and the three accelerator + /// tables. None of them is ever closed mid-walk — they are written once, at + /// the end, from everything the run produced — so this is where they + /// belong rather than in the chunk machinery. + /// The committed public output, in the order the run wrote it. + /// + /// COMMIT is an accumulator — the walk never closes it — so every op is + /// still here at the end of the run, and this is the same fold over the + /// same list that the ordinary build does. The statement absorbed into the + /// transcript carries these bytes, so the Challenge phase cannot sample + /// without them. + pub(crate) fn public_output_bytes(&self) -> Vec { + self.tail + .commit_ops + .iter() + .filter(|op| !op.end) + .map(|op| op.value) + .collect() + } + + pub(crate) fn build_accumulated(&self) -> AccumulatedTables { + let keccak_rnd_ops: Vec = self + .tail + .keccak_ops + .iter() + .map(|op| KeccakRoundOperation { + timestamp: op.timestamp, + input: op.input, + output: op.output, + }) + .collect(); + let mut keccak_rc = keccak_rc::generate_keccak_rc_trace(); + keccak_rc::update_multiplicities(&mut keccak_rc, self.tail.keccak_ops.len()); + + AccumulatedTables { + present: [ + !self.tail.commit_ops.is_empty(), + !self.tail.keccak_ops.is_empty(), + !keccak_rnd_ops.is_empty(), + !self.tail.ecsm_ops.is_empty(), + !self.tail.ecdas_ops.is_empty(), + !self.tail.hint_ops.is_empty(), + ], + commit: commit::generate_commit_trace(&self.tail.commit_ops), + keccak: keccak::generate_keccak_trace(&self.tail.keccak_ops), + keccak_rnd: keccak_rnd::generate_keccak_rnd_trace(&keccak_rnd_ops), + keccak_rc, + ecsm: ecsm::generate_ecsm_trace(&self.tail.ecsm_ops), + ecdas: ecdas::generate_ecdas_trace(&self.tail.ecdas_ops), + hint: hint::generate_hint_trace(&self.tail.hint_ops), + } + } + + /// Build the DECODE table for the run. + /// + /// One lookup per executed cycle at that cycle's pc, plus one per padding + /// row at the padding pc. The walk counted both as it went — the cycles + /// because their CPU ops are long gone, the padding because each chunk's + /// share is known when the chunk closes — so this only has to add the tail's + /// own cycles and its padding. + /// + /// `decode_trace` is the pristine table from the ELF; the multiplicities are + /// the only part that depends on the run. + pub(crate) fn build_decode( + &self, + decode_trace: TraceTable, + pc_to_row: &decode::PcToRow, + max_rows: &super::MaxRowsConfig, + ) -> TraceTable { + let mut counts = self.decode_counts.clone(); + let padding = self.cpu_padding_rows(); + let _ = max_rows; + *counts.entry(cpu::CPU_PADDING_PC).or_insert(0) += padding as u64; + + let mut decode = decode_trace; + decode::add_multiplicities(&mut decode, pc_to_row, &counts); + decode + } + + /// Total padding rows the CPU table adds, over the closed chunks and the + /// tail. + fn cpu_padding_rows(&self) -> usize { + self.total_cpu_padding + .expect("finalize must run before the end-of-run tables are built") + } + + /// Build HALT and REGISTER, in that order because the second depends on the + /// first. + /// + /// HALT comes from the run's terminating ECALL. REGISTER then has to finalize + /// the PC: the CPU padding rows chain inline-PC tokens at a +4 cadence from + /// the HALT chip's emit at `halt_timestamp + 1`, so the last write lands at + /// `halt_timestamp + 4 * padding + 1` and REGISTER's final token must match + /// it or the memory argument does not balance. Both numbers were counted + /// during the walk, since the ops that carry them are long dropped. + pub(crate) fn build_halt_and_register( + &mut self, + register_init: &[u32], + ) -> Result { + let (halt_timestamp, halt_next_pc) = self.last_ecall.ok_or(Error::MissingHaltEcall)?; + let padding = self.cpu_padding_rows(); + self.register_state + .write_pc(1, halt_timestamp + 4 * padding as u64 + 1); + let register_final_state = self.register_state.to_final_state_map(); + + Ok(( + halt::generate_halt_trace(halt_timestamp, halt_next_pc), + register::generate_register_trace(®ister_final_state, register_init), + )) + } + + /// Build the PAGE tables from the run's end memory. + /// + /// PAGE also owes BITWISE its lookups, so they are folded into `hist` here + /// rather than left for a caller to remember. + pub(crate) fn build_pages( + &self, + initial_image: &I, + private_input: &[u8], + hist: &mut bitwise::BitwiseHistogram, + ) -> ( + Vec>, + Vec, + ) { + let (tables, configs) = + generate_page_tables(initial_image, &self.memory_state, private_input, false); + collect_bitwise_from_page(initial_image, &self.memory_state, false, hist); + (tables, configs) + } + + /// Cycles the walk executed. + pub fn cycles(&self) -> usize { + self.cycles + } + + /// Ops still held for `kind` — the tail that the end-of-run phase pads and + /// commits. + pub fn buffered(&self, kind: TableKind) -> usize { + self.tail.buffered(kind) + } + + /// Chunks the walk closed for `kind`, so the tail can be numbered after + /// them. + pub fn emitted(&self, kind: TableKind) -> usize { + CHUNKED_KINDS + .iter() + .position(|k| *k == kind) + .map_or(0, |slot| self.emitted[slot]) + } +} + +/// HALT and REGISTER, which are built together because the second depends on +/// the first. +pub type HaltAndRegister = ( + TraceTable, + TraceTable, +); + +/// The tables built once, at the end, from an accumulated op list. +pub struct AccumulatedTables { + /// Whether the run produced ops for each accelerator, in the order + /// `air_trace_pairs` emits them: COMMIT, KECCAK, KECCAK_RND, ECSM, ECDAS, + /// HINT. + /// + /// Taken before generating, because `generate_*` pads: an accelerator the + /// run never called still comes back with rows, so the table cannot answer + /// this afterwards. `generate_optional` asks the same question of the same + /// lists, which is what keeps the two table sets identical. + pub present: [bool; 6], + pub commit: TraceTable, + pub keccak: TraceTable, + pub keccak_rnd: TraceTable, + pub keccak_rc: TraceTable, + pub ecsm: TraceTable, + pub ecdas: TraceTable, + pub hint: TraceTable, +} + +/// The tables `cpu32_chip_op` appends to. +/// +/// Kept beside it because it is load-bearing elsewhere: a table listed here is +/// NOT final when a segment ends, so the Commit-phase walk must not close it +/// early — `cpu32_appends_are_excluded_from_early_closing` enforces that. If +/// this function starts feeding another table, add it here. +pub const CPU32_APPENDS_TO: [TableKind; 3] = [TableKind::Shift, TableKind::Mul, TableKind::Dvrm]; + +#[allow(clippy::type_complexity)] /// The ALU-chip op a word ALU instruction dispatches (SHIFT/MUL/DVRM). ADDW/SUBW /// are the CPU32 ADD/SUB fast-path (no external chip), returning `None`. -#[allow(clippy::type_complexity)] fn cpu32_chip_op( c: &cpu32::Cpu32Operation, shift_ops: &mut Vec, @@ -2101,6 +2521,45 @@ fn private_input_bytes(private_input: &[u8]) -> Vec { .collect() } +/// Run-length encode the runtime (non-ELF) page bases into `(base, count)`. +/// +/// Zero-init pages are the runtime ones, so `init_values == None` identifies +/// them without rescanning the ELF segments. The result goes into the statement +/// the transcript absorbs, which is why it takes the configs rather than a +/// built `Traces`: the Commit phase has the configs and no `Traces`. +pub(crate) fn runtime_page_ranges( + page_configs: &[page::PageConfig], +) -> Vec { + let page_size = page::DEFAULT_PAGE_SIZE as u64; + + let runtime_bases: Vec = page_configs + .iter() + .filter(|config| config.init_values.is_none()) + .map(|config| config.page_base) + .collect(); + + let mut ranges = Vec::new(); + if runtime_bases.is_empty() { + return ranges; + } + + let mut start = runtime_bases[0]; + let mut count = 1u64; + + for &base in &runtime_bases[1..] { + if base == start + count * page_size { + count += 1; + } else { + ranges.push(crate::RuntimePageRange { base: start, count }); + start = base; + count = 1; + } + } + ranges.push(crate::RuntimePageRange { base: start, count }); + + ranges +} + /// Build the initial-memory image (byte address -> value) from the ELF segments /// and the private-input region. Single source of "what memory starts as", read /// by both `MemoryState` seeding and PAGE/bitwise init. @@ -2161,7 +2620,7 @@ pub(crate) fn epoch_touched_cells( ) -> Result, Error> { let instructions = decode::instructions_from_elf(elf) .map_err(|e| Error::Execution(format!("Failed to parse instructions: {e}")))?; - let cpu_ops = collect_cpu_ops(logs, &instructions)?; + let cpu_ops = collect_cpu_ops(logs, &instructions, 0)?; let mut memory_state = MemoryState::from_image(initial_image); let mut register_state = RegisterState::from_init(register_init); @@ -2747,9 +3206,9 @@ fn generate_page_tables( /// build ([`Traces::from_image_and_logs_with_decode`]) instead of re-parsing /// the ELF and regenerating the trace per epoch. pub struct DecodeArtifacts { - instructions: U64HashMap, - decode_trace: TraceTable, - decode_pc_to_row: decode::PcToRow, + pub(crate) instructions: U64HashMap, + pub(crate) decode_trace: TraceTable, + pub(crate) decode_pc_to_row: decode::PcToRow, } impl DecodeArtifacts { @@ -2784,6 +3243,12 @@ pub struct CollectedEpoch { } impl CollectedEpoch { + /// Ops collected for `kind`. Mirrors [`CollectedOps::buffered`] so a walk's + /// output and a finished run's can be compared on the same footing. + pub fn op_count(&self, kind: TableKind) -> usize { + self.ops.buffered(kind) + } + /// The epoch's touched memory cells (sorted by address): the exact values /// `build_traces` later stores in `Traces::touched_memory_cells` (both are /// [`touched_cells_from_memory_state`] over the same immutable @@ -2904,95 +3369,656 @@ pub struct Traces { /// Intermediate state from Phase 2: all ops collected from CPU, ready for /// Phases 3-5 (LT extension, bitwise, trace generation). -struct CollectedOps { - cpu_ops: Vec, - memw_ops: Vec, - memw_aligned_ops: Vec, +impl CollectedOps { + /// Rows and main columns of one chunk, without building it. + /// + /// Every generator pads to `count.next_power_of_two().max(4)`, where `count` + /// is the chunk's op count — or, for the six tables that deduplicate, the + /// number of DISTINCT ops in it. The width is a per-table constant. So the + /// shape needs a counting pass at worst, never a trace. + /// + /// `chunk_shape_matches_the_built_chunk` pins this against real builds for + /// every kind; it is what catches a generator that changes its padding. + pub(crate) fn chunk_shape( + &self, + kind: TableKind, + chunk: usize, + max_rows: &super::MaxRowsConfig, + ) -> (usize, usize) { + use std::collections::HashSet; + + macro_rules! slice_of { + ($ops:expr, $limit:expr) => {{ + let ops = $ops; + let slice: &[_] = if ops.is_empty() { + &[] + } else { + ops.chunks($limit).nth(chunk).unwrap_or(&[]) + }; + slice + }}; + } + // One row per op. + macro_rules! plain { + ($ops:expr, $limit:expr, $cols:expr) => { + (slice_of!($ops, $limit).len(), $cols) + }; + } + // One row per DISTINCT op. + macro_rules! dedup { + ($ops:expr, $limit:expr, $cols:expr) => { + ( + slice_of!($ops, $limit).iter().collect::>().len(), + $cols, + ) + }; + } + // Same, where the op list pairs each op with a flag the dedup folds in. + macro_rules! dedup_tagged { + ($ops:expr, $limit:expr, $cols:expr) => { + ( + slice_of!($ops, $limit) + .iter() + .map(|(op, _)| op) + .collect::>() + .len(), + $cols, + ) + }; + } + + let (count, cols) = match kind { + TableKind::Cpu => plain!(&self.cpu_ops, max_rows.cpu, cpu::cols::NUM_COLUMNS), + TableKind::Memw => plain!(&self.memw_ops, max_rows.memw, memw::cols::NUM_COLUMNS), + TableKind::MemwAligned => plain!( + &self.memw_aligned_ops, + max_rows.memw_aligned, + memw_aligned::cols::NUM_COLUMNS + ), + TableKind::MemwRegister => plain!( + &self.memw_register_rows, + max_rows.memw_register, + memw_register::cols::NUM_COLUMNS + ), + TableKind::Load => plain!(&self.load_ops, max_rows.load, load::cols::NUM_COLUMNS), + TableKind::Shift => plain!(&self.shift_ops, max_rows.shift, shift::cols::NUM_COLUMNS), + TableKind::Store => plain!(&self.store_ops, max_rows.store, store::cols::NUM_COLUMNS), + TableKind::Cpu32 => plain!(&self.cpu32_ops, max_rows.cpu32, cpu32::cols::NUM_COLUMNS), + TableKind::Lt => dedup!(&self.lt_ops, max_rows.lt, lt::cols::NUM_COLUMNS), + TableKind::Branch => { + dedup!(&self.branch_ops, max_rows.branch, branch::cols::NUM_COLUMNS) + } + TableKind::Eq => dedup!(&self.eq_ops, max_rows.eq, eq::cols::NUM_COLUMNS), + TableKind::Bytewise => dedup!( + &self.bytewise_ops, + max_rows.bytewise, + bytewise::cols::NUM_COLUMNS + ), + TableKind::Mul => dedup_tagged!(&self.mul_ops, max_rows.mul, mul::cols::NUM_COLUMNS), + TableKind::Dvrm => { + dedup_tagged!(&self.dvrm_ops, max_rows.dvrm, dvrm::cols::NUM_COLUMNS) + } + }; + (count.next_power_of_two().max(4), cols) + } + + /// Build exactly one chunk of one table. + /// + /// Byte-identical to `build_table(kind)[chunk]`: same op slice into the same + /// generator. That equality is the whole point — it is what lets the fused + /// chain rebuild a trace the Round 1 commit already hashed. + pub(crate) fn build_chunk( + &self, + kind: TableKind, + chunk: usize, + max_rows: &super::MaxRowsConfig, + ) -> TraceTable { + macro_rules! chunk_of { + ($ops:expr, $limit:expr, $f:path) => {{ + let ops = $ops; + let slice: &[_] = if ops.is_empty() { + &[] + } else { + ops.chunks($limit).nth(chunk).unwrap_or(&[]) + }; + $f(slice) + }}; + } + match kind { + TableKind::Cpu => chunk_of!(&self.cpu_ops, max_rows.cpu, cpu::generate_cpu_trace), + TableKind::Memw => chunk_of!(&self.memw_ops, max_rows.memw, memw::generate_memw_trace), + TableKind::MemwAligned => chunk_of!( + &self.memw_aligned_ops, + max_rows.memw_aligned, + memw_aligned::generate_memw_aligned_trace + ), + TableKind::MemwRegister => chunk_of!( + &self.memw_register_rows, + max_rows.memw_register, + memw_register::generate_memw_register_trace_from_rows + ), + TableKind::Load => chunk_of!(&self.load_ops, max_rows.load, load::generate_load_trace), + TableKind::Lt => chunk_of!(&self.lt_ops, max_rows.lt, lt::generate_lt_trace), + TableKind::Shift => { + chunk_of!(&self.shift_ops, max_rows.shift, shift::generate_shift_trace) + } + TableKind::Mul => chunk_of!(&self.mul_ops, max_rows.mul, mul::generate_mul_trace), + TableKind::Dvrm => chunk_of!(&self.dvrm_ops, max_rows.dvrm, dvrm::generate_dvrm_trace), + TableKind::Branch => chunk_of!( + &self.branch_ops, + max_rows.branch, + branch::generate_branch_trace + ), + TableKind::Eq => chunk_of!(&self.eq_ops, max_rows.eq, eq::generate_eq_trace), + TableKind::Bytewise => chunk_of!( + &self.bytewise_ops, + max_rows.bytewise, + bytewise::generate_bytewise_trace + ), + TableKind::Store => { + chunk_of!(&self.store_ops, max_rows.store, store::generate_store_trace) + } + TableKind::Cpu32 => { + chunk_of!(&self.cpu32_ops, max_rows.cpu32, cpu32::generate_cpu32_trace) + } + } + } + + /// Build every chunk of one table. Byte-identical whenever it is called, + /// which is what lets a retired trace be rebuilt against the root its first + /// build committed. + pub(crate) fn build_table( + &self, + kind: TableKind, + max_rows: &super::MaxRowsConfig, + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + ) -> Result>, Error> { + macro_rules! build { + ($ops:expr, $limit:expr, $f:path) => { + chunk_and_generate( + $ops, + $limit, + $f, + #[cfg(feature = "disk-spill")] + storage_mode, + ) + }; + } + // Empty op list yields zero chunks, not one padded chunk (#977). Which + // kinds take this arm is `skips_when_empty`; `num_chunks` reads the same + // predicate, and the placeholder path depends on the two agreeing. + macro_rules! build_optional { + ($ops:expr, $limit:expr, $f:path) => { + chunk_and_generate_optional( + $ops, + $limit, + $f, + #[cfg(feature = "disk-spill")] + storage_mode, + ) + }; + } + match kind { + TableKind::Cpu => build!(&self.cpu_ops, max_rows.cpu, cpu::generate_cpu_trace), + TableKind::Memw => { + build_optional!(&self.memw_ops, max_rows.memw, memw::generate_memw_trace) + } + TableKind::MemwAligned => build_optional!( + &self.memw_aligned_ops, + max_rows.memw_aligned, + memw_aligned::generate_memw_aligned_trace + ), + TableKind::MemwRegister => build!( + &self.memw_register_rows, + max_rows.memw_register, + memw_register::generate_memw_register_trace_from_rows + ), + TableKind::Load => { + build_optional!(&self.load_ops, max_rows.load, load::generate_load_trace) + } + TableKind::Lt => build_optional!(&self.lt_ops, max_rows.lt, lt::generate_lt_trace), + TableKind::Shift => { + build_optional!(&self.shift_ops, max_rows.shift, shift::generate_shift_trace) + } + TableKind::Mul => build_optional!(&self.mul_ops, max_rows.mul, mul::generate_mul_trace), + TableKind::Dvrm => { + build_optional!(&self.dvrm_ops, max_rows.dvrm, dvrm::generate_dvrm_trace) + } + TableKind::Branch => build_optional!( + &self.branch_ops, + max_rows.branch, + branch::generate_branch_trace + ), + TableKind::Eq => build_optional!(&self.eq_ops, max_rows.eq, eq::generate_eq_trace), + TableKind::Bytewise => build_optional!( + &self.bytewise_ops, + max_rows.bytewise, + bytewise::generate_bytewise_trace + ), + TableKind::Store => { + build_optional!(&self.store_ops, max_rows.store, store::generate_store_trace) + } + TableKind::Cpu32 => { + build_optional!(&self.cpu32_ops, max_rows.cpu32, cpu32::generate_cpu32_trace) + } + } + } + + /// Fold the BITWISE lookups the first `n` ops of `kind` imply into `hist`. + /// + /// Must run before those ops are drained. BITWISE accumulates across the + /// whole run from tables the Commit phase retires, so a chunk's + /// contribution has to be taken while the chunk still exists — otherwise + /// the table it lands in comes out short and the bus does not balance. + /// + /// Only three of the kinds the walk closes feed BITWISE; the others have no + /// collector and contribute nothing. + pub(crate) fn fold_bitwise_from_front( + &self, + kind: TableKind, + n: usize, + hist: &mut bitwise::BitwiseHistogram, + ) { + match kind { + TableKind::MemwAligned => hist.add_ops(&collect_bitwise_from_memw_aligned( + &self.memw_aligned_ops[..n], + )), + TableKind::MemwRegister => memw_register::collect_bitwise_from_memw_register( + &self.memw_register_rows[..n], + hist, + ), + TableKind::Branch => hist.add_ops(&collect_bitwise_from_branch(&self.branch_ops[..n])), + TableKind::Bytewise => { + for op in &self.bytewise_ops[..n] { + hist.add_ops(&op.collect_bitwise_ops()); + } + } + TableKind::Eq => { + for op in &self.eq_ops[..n] { + hist.add_ops(&op.collect_bitwise_ops()); + } + } + TableKind::Store => { + for op in &self.store_ops[..n] { + hist.add_ops(&op.collect_bitwise_ops()); + } + } + TableKind::Cpu32 => { + for c in &self.cpu32_ops[..n] { + hist.add_ops(&collect_cpu32_bitwise(c)); + } + } + _ => {} + } + } + + /// Build a trace from the first `n` buffered ops of `kind` and drop them. + /// + /// Draining is the point: this is what keeps the walk's buffers from + /// growing with the run. + fn take_front( + &mut self, + kind: TableKind, + n: usize, + max_rows: &super::MaxRowsConfig, + ) -> TraceTable { + macro_rules! drain { + ($ops:expr, $f:path) => {{ + let front: Vec<_> = $ops.drain(..n).collect(); + $f(&front) + }}; + } + let _ = max_rows; + match kind { + TableKind::Cpu => drain!(self.cpu_ops, cpu::generate_cpu_trace), + TableKind::Memw => drain!(self.memw_ops, memw::generate_memw_trace), + TableKind::MemwAligned => { + drain!( + self.memw_aligned_ops, + memw_aligned::generate_memw_aligned_trace + ) + } + TableKind::MemwRegister => drain!( + self.memw_register_rows, + memw_register::generate_memw_register_trace_from_rows + ), + TableKind::Load => drain!(self.load_ops, load::generate_load_trace), + TableKind::Cpu32 => drain!(self.cpu32_ops, cpu32::generate_cpu32_trace), + TableKind::Branch => drain!(self.branch_ops, branch::generate_branch_trace), + TableKind::Eq => drain!(self.eq_ops, eq::generate_eq_trace), + TableKind::Bytewise => drain!(self.bytewise_ops, bytewise::generate_bytewise_trace), + TableKind::Store => drain!(self.store_ops, store::generate_store_trace), + // Not closable mid-walk, but the end-of-run phase builds them the + // same way once nothing can append to them any more. + TableKind::Lt => drain!(self.lt_ops, lt::generate_lt_trace), + TableKind::Mul => drain!(self.mul_ops, mul::generate_mul_trace), + TableKind::Dvrm => drain!(self.dvrm_ops, dvrm::generate_dvrm_trace), + TableKind::Shift => drain!(self.shift_ops, shift::generate_shift_trace), + } + } + + /// How many chunks `build_table` would produce for `kind`. + /// + /// Mirrors `chunk_and_generate`: an empty op list still yields one (padded) + /// chunk, so the table exists in the proof with the shape the verifier + /// expects. + /// Whether an empty op list for `kind` yields zero chunks instead of one + /// padded one. The authority for which arm `build_table` takes: the two must + /// agree, or the placeholder path emits a chunk the real build never makes. + pub(crate) fn skips_when_empty(kind: TableKind) -> bool { + !matches!(kind, TableKind::Cpu | TableKind::MemwRegister) + } + + pub(crate) fn num_chunks(&self, kind: TableKind, max_rows: &super::MaxRowsConfig) -> usize { + let (len, limit) = self.shape_of(kind, max_rows); + if len == 0 { + usize::from(!Self::skips_when_empty(kind)) + } else { + len.div_ceil(limit) + } + } + + /// Op count and chunk limit for `kind`. + fn shape_of(&self, kind: TableKind, max_rows: &super::MaxRowsConfig) -> (usize, usize) { + match kind { + TableKind::Cpu => (self.cpu_ops.len(), max_rows.cpu), + TableKind::Memw => (self.memw_ops.len(), max_rows.memw), + TableKind::MemwAligned => (self.memw_aligned_ops.len(), max_rows.memw_aligned), + TableKind::MemwRegister => (self.memw_register_rows.len(), max_rows.memw_register), + TableKind::Load => (self.load_ops.len(), max_rows.load), + TableKind::Lt => (self.lt_ops.len(), max_rows.lt), + TableKind::Shift => (self.shift_ops.len(), max_rows.shift), + TableKind::Mul => (self.mul_ops.len(), max_rows.mul), + TableKind::Dvrm => (self.dvrm_ops.len(), max_rows.dvrm), + TableKind::Branch => (self.branch_ops.len(), max_rows.branch), + TableKind::Eq => (self.eq_ops.len(), max_rows.eq), + TableKind::Bytewise => (self.bytewise_ops.len(), max_rows.bytewise), + TableKind::Store => (self.store_ops.len(), max_rows.store), + TableKind::Cpu32 => (self.cpu32_ops.len(), max_rows.cpu32), + } + } + + /// Ops collected for `kind`, for the kinds a Commit-phase walk can close. + pub(crate) fn buffered(&self, kind: TableKind) -> usize { + match kind { + TableKind::Cpu => self.cpu_ops.len(), + TableKind::Memw => self.memw_ops.len(), + TableKind::MemwAligned => self.memw_aligned_ops.len(), + TableKind::MemwRegister => self.memw_register_rows.len(), + TableKind::Load => self.load_ops.len(), + TableKind::Cpu32 => self.cpu32_ops.len(), + TableKind::Branch => self.branch_ops.len(), + TableKind::Eq => self.eq_ops.len(), + TableKind::Bytewise => self.bytewise_ops.len(), + TableKind::Store => self.store_ops.len(), + TableKind::Shift => self.shift_ops.len(), + TableKind::Lt => self.lt_ops.len(), + TableKind::Mul => self.mul_ops.len(), + TableKind::Dvrm => self.dvrm_ops.len(), + } + } +} + +#[derive(Default)] +pub(crate) struct CollectedOps { + pub(crate) cpu_ops: Vec, + /// LT rows owed by the MEMW chunks already retired, kept apart so LT gets + /// them in the ordinary build's order once the tail's are known. + pub(crate) retired_memw_lt: Vec, + pub(crate) retired_memw_aligned_lt: Vec, + /// SHIFT, MUL and DVRM rows the retired CPU32 chunks dispatched, likewise. + pub(crate) retired_cpu32_shift: Vec, + pub(crate) retired_cpu32_mul: Vec<(MulOperation, bool)>, + pub(crate) retired_cpu32_dvrm: Vec<(DvrmOperation, bool)>, + pub(crate) memw_ops: Vec, + pub(crate) memw_aligned_ops: Vec, /// Direct-fill MEMW_R rows (register fast path). - memw_register_rows: Vec, - load_ops: Vec, - lt_ops: Vec, - shift_ops: Vec, - bitwise_ops: Vec, - branch_ops: Vec, - mul_ops: Vec<(MulOperation, bool)>, - dvrm_ops: Vec<(DvrmOperation, bool)>, - commit_ops: Vec, - keccak_ops: Vec, + pub(crate) memw_register_rows: Vec, + pub(crate) load_ops: Vec, + pub(crate) lt_ops: Vec, + pub(crate) shift_ops: Vec, + pub(crate) bitwise_ops: Vec, + pub(crate) branch_ops: Vec, + pub(crate) mul_ops: Vec<(MulOperation, bool)>, + pub(crate) dvrm_ops: Vec<(DvrmOperation, bool)>, + pub(crate) commit_ops: Vec, + pub(crate) keccak_ops: Vec, // Auxiliary ALU / memory / CPU32 dispatch chips (driven by the CPU ALU/MEMORY dispatch). - eq_ops: Vec, - bytewise_ops: Vec, - store_ops: Vec, - cpu32_ops: Vec, + pub(crate) eq_ops: Vec, + pub(crate) bytewise_ops: Vec, + pub(crate) store_ops: Vec, + pub(crate) cpu32_ops: Vec, // EC scalar-multiplication accelerator chips. - ecsm_ops: Vec, - ecdas_ops: Vec, + pub(crate) ecsm_ops: Vec, + pub(crate) ecdas_ops: Vec, // Non-constraining hint ecall. - hint_ops: Vec, + pub(crate) hint_ops: Vec, } -/// Chunk raw ops and generate one trace table per chunk, padding an empty `ops` -/// to a single chunk so the table is always present in the proof. +/// One log-derived, chunked table — the ones whose trace is a function of a +/// single routed op list, so it can be rebuilt on demand long after the routing +/// that produced it. /// -/// For tables that may be omitted entirely, use [`chunk_and_generate_optional`]. -fn chunk_and_generate( - ops: &[T], - max_rows: usize, - generate: impl Fn(&[T]) -> TraceTable + Send + Sync, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, -) -> Result>, Error> { - let op_chunks: Vec<&[T]> = if ops.is_empty() { - vec![&[][..]] - } else { - ops.chunks(max_rows).collect() - }; - generate_chunks( - op_chunks, - generate, - #[cfg(feature = "disk-spill")] - storage_mode, - ) +/// The preprocessed tables (BITWISE, DECODE, REGISTER, HALT, COMMIT, KECCAK*) +/// and PAGE are deliberately absent: they are not driven by one op list and the +/// streaming prover keeps them resident. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum TableKind { + Cpu, + Memw, + MemwAligned, + MemwRegister, + Load, + Lt, + Shift, + Mul, + Dvrm, + Branch, + Eq, + Bytewise, + Store, + Cpu32, } -/// Like [`chunk_and_generate`], but an empty `ops` yields no table at all: the -/// chip is left out of the proof instead of costing a padded sub-proof. +/// The tables that are a pure per-op function of the CPU ops, with no later +/// source appending to them. /// -/// The empty case short-circuits rather than falling through to `ops.chunks`, -/// which panics on a zero chunk size even for an empty slice. `max_rows` comes -/// from a caller-supplied [`MaxRowsConfig`] whose fields are public. +/// Extracted so the all-at-once path and the Commit-phase walk derive them with +/// the same code: a table closed mid-walk has to be the table the finished run +/// would have produced, and two copies of a filter+map drift. /// -/// Sound because a chip contributes to the run only through its LogUp bus, and a -/// chip with no rows contributes zero. A prover that omits a table whose ops did -/// execute leaves its counterparty's sends unmatched, and the bus-balance check -/// over the tables that *are* present rejects the proof. See -/// `TableCounts::validate`. -fn chunk_and_generate_optional( - ops: &[T], - max_rows: usize, - generate: impl Fn(&[T]) -> TraceTable + Send + Sync, - #[cfg(feature = "disk-spill")] storage_mode: StorageMode, -) -> Result>, Error> { - let op_chunks: Vec<&[T]> = if ops.is_empty() { - vec![] - } else { - ops.chunks(max_rows).collect() - }; - generate_chunks( - op_chunks, - generate, - #[cfg(feature = "disk-spill")] - storage_mode, - ) +/// DVRM, MUL and LT are deliberately not here. Each takes ops from more than +/// one source — CPU32 appends to DVRM and MUL, DVRM appends to MUL and LT — and +/// the finished run concatenates those sources whole, so deriving them per +/// segment would interleave them differently and cut the chunks elsewhere. +struct DerivedFromCpu { + branch_ops: Vec, + eq_ops: Vec, + bytewise_ops: Vec, + store_ops: Vec, + mul_ops: Vec<(MulOperation, bool)>, + dvrm_ops: Vec<(DvrmOperation, bool)>, } -/// Generate a single trace table for `ops`, or none at all when `ops` is empty. -/// -/// The accelerator chips are not chunked: one call means one table. What they do -/// share with the chunked chips is that an empty op list should cost nothing, so -/// this returns an empty `Vec` and the table drops out of the proof. Soundness -/// rests on the same LogUp argument as [`chunk_and_generate_optional`]. -fn generate_optional( - ops: &[T], +fn derive_from_cpu(cpu_ops: &[CpuOperation]) -> DerivedFromCpu { + // BRANCH: CPU ops where branch_cond = true. + let branch_ops: Vec = cpu_ops + .iter() + .filter(|op| op.branch_cond) + .map(|op| { + BranchOperation::new( + op.decode.pc, + op.decode.imm, // offset as full 64-bit DWordWL (already sign-extended) + op.rv1, // register value must match the CPU's BRANCH bus signature + op.decode.fields.jalr(), + ) + }) + .collect(); + // EQ: BEQ/BNE (invert = alu_flags bit 6). + let eq_ops: Vec = cpu_ops + .iter() + .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_eq()) + .map(|op| eq::EqOperation::new(op.rv1, op.arg2, op.decode.fields.alu_signed2_or_invert())) + .collect(); + // BYTEWISE: AND/OR/XOR (op = alu_op). + let bytewise_ops: Vec = cpu_ops + .iter() + .filter(|op| { + let f = &op.decode.fields; + !f.word_instr && (f.is_and() || f.is_or() || f.is_xor()) + }) + .map(|op| bytewise::BytewiseOperation::new(op.rv1, op.arg2, op.decode.fields.alu_op())) + .collect(); + // STORE: receives MEMORY(memory_op=1) from the CPU and sends the MEMW write + // at timestamp+1 (mirrors `collect_store_op_from_cpu`, which records the MEMW + // table row). The MEMORY bus and the STORE chip's MEMW write share the base + // timestamp (spec store.toml uses one `timestamp` for both). + let store_ops: Vec = cpu_ops + .iter() + .filter(|op| op.decode.fields.is_store()) + .map(|op| { + store::StoreOperation::new( + op.res, + op.timestamp, + op.rv2, + op.decode.fields.mem_bytes() as u8, + ) + }) + .collect(); + + // MUL: non-word MUL instructions. lhs_signed = `signed` (alu_flags bit 5); + // rhs_signed = `signed2` (bit 6); wants_hi = `muldiv` (bit 7). + let mul_ops: Vec<(MulOperation, bool)> = cpu_ops + .iter() + .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_mul()) + .map(|op| { + let f = op.decode.fields; + ( + MulOperation::new(op.rv1, f.alu_signed(), op.arg2, f.alu_signed2_or_invert()), + f.alu_muldiv(), + ) + }) + .collect(); + // DVRM: non-word DIV/REM instructions. + let dvrm_ops: Vec<(DvrmOperation, bool)> = cpu_ops + .iter() + .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_divrem()) + .map(|op| { + let f = op.decode.fields; + ( + DvrmOperation::new(op.rv1, op.arg2, f.alu_signed()), + f.alu_muldiv(), + ) + }) + .collect(); + + DerivedFromCpu { + branch_ops, + eq_ops, + bytewise_ops, + store_ops, + mul_ops, + dvrm_ops, + } +} + +/// The tables this walk can close mid-execution: their ops come straight out of +/// `collect_ops_from_cpu` and nothing appends to them afterwards. +pub const CHUNKED_KINDS: [TableKind; 10] = [ + TableKind::Cpu, + TableKind::Memw, + TableKind::MemwAligned, + TableKind::MemwRegister, + TableKind::Load, + TableKind::Cpu32, + TableKind::Branch, + TableKind::Eq, + TableKind::Bytewise, + TableKind::Store, +]; + +/// Chunk limit for one kind. +pub fn max_rows_for(kind: TableKind, max_rows: &super::MaxRowsConfig) -> usize { + match kind { + TableKind::Cpu => max_rows.cpu, + TableKind::Memw => max_rows.memw, + TableKind::MemwAligned => max_rows.memw_aligned, + TableKind::MemwRegister => max_rows.memw_register, + TableKind::Load => max_rows.load, + TableKind::Shift => max_rows.shift, + TableKind::Mul => max_rows.mul, + TableKind::Dvrm => max_rows.dvrm, + TableKind::Branch => max_rows.branch, + TableKind::Lt => max_rows.lt, + TableKind::Eq => max_rows.eq, + TableKind::Bytewise => max_rows.bytewise, + TableKind::Store => max_rows.store, + TableKind::Cpu32 => max_rows.cpu32, + } +} + +/// Chunk raw ops and generate one trace table per chunk, padding an empty `ops` +/// to a single chunk so the table is always present in the proof. +/// +/// For tables that may be omitted entirely, use [`chunk_and_generate_optional`]. +fn chunk_and_generate( + ops: &[T], + max_rows: usize, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, +) -> Result>, Error> { + let op_chunks: Vec<&[T]> = if ops.is_empty() { + vec![&[][..]] + } else { + ops.chunks(max_rows).collect() + }; + generate_chunks( + op_chunks, + generate, + #[cfg(feature = "disk-spill")] + storage_mode, + ) +} + +/// Like [`chunk_and_generate`], but an empty `ops` yields no table at all: the +/// chip is left out of the proof instead of costing a padded sub-proof. +/// +/// The empty case short-circuits rather than falling through to `ops.chunks`, +/// which panics on a zero chunk size even for an empty slice. `max_rows` comes +/// from a caller-supplied [`MaxRowsConfig`] whose fields are public. +/// +/// Sound because a chip contributes to the run only through its LogUp bus, and a +/// chip with no rows contributes zero. A prover that omits a table whose ops did +/// execute leaves its counterparty's sends unmatched, and the bus-balance check +/// over the tables that *are* present rejects the proof. See +/// `TableCounts::validate`. +fn chunk_and_generate_optional( + ops: &[T], + max_rows: usize, + generate: impl Fn(&[T]) -> TraceTable + Send + Sync, + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, +) -> Result>, Error> { + let op_chunks: Vec<&[T]> = if ops.is_empty() { + vec![] + } else { + ops.chunks(max_rows).collect() + }; + generate_chunks( + op_chunks, + generate, + #[cfg(feature = "disk-spill")] + storage_mode, + ) +} + +/// Generate a single trace table for `ops`, or none at all when `ops` is empty. +/// +/// The accelerator chips are not chunked: one call means one table. What they do +/// share with the chunked chips is that an empty op list should cost nothing, so +/// this returns an empty `Vec` and the table drops out of the proof. Soundness +/// rests on the same LogUp argument as [`chunk_and_generate_optional`]. +fn generate_optional( + ops: &[T], generate: impl Fn(&[T]) -> TraceTable + Send + Sync, #[cfg(feature = "disk-spill")] storage_mode: StorageMode, ) -> Result>, Error> { @@ -3075,79 +4101,14 @@ fn collect_all_ops( general: memw_ops, } = memw; - // Collect BRANCH operations from CPU ops where branch_cond = true - let branch_ops: Vec = cpu_ops - .iter() - .filter(|op| op.branch_cond) - .map(|op| { - BranchOperation::new( - op.decode.pc, - op.decode.imm, // offset as full 64-bit DWordWL (already sign-extended) - op.rv1, // register value must match the CPU's BRANCH bus signature - op.decode.fields.jalr(), - ) - }) - .collect(); - - // Collect MUL operations from non-word MUL instructions. lhs_signed = `signed` - // (alu_flags bit 5); rhs_signed = `signed2` (bit 6); wants_hi = `muldiv` (bit 7). - let mut mul_ops: Vec<(MulOperation, bool)> = cpu_ops - .iter() - .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_mul()) - .map(|op| { - let f = op.decode.fields; - ( - MulOperation::new(op.rv1, f.alu_signed(), op.arg2, f.alu_signed2_or_invert()), - f.alu_muldiv(), - ) - }) - .collect(); - - // Collect DVRM operations from non-word DIV/REM instructions. - let mut dvrm_ops: Vec<(DvrmOperation, bool)> = cpu_ops - .iter() - .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_divrem()) - .map(|op| { - let f = op.decode.fields; - ( - DvrmOperation::new(op.rv1, op.arg2, f.alu_signed()), - f.alu_muldiv(), - ) - }) - .collect(); - - // Collect the ALU/MEMORY chip ops (non-word rows). - // EQ: BEQ/BNE (invert = alu_flags bit 6). BYTEWISE: AND/OR/XOR (op = alu_op). - let eq_ops: Vec = cpu_ops - .iter() - .filter(|op| !op.decode.fields.word_instr && op.decode.fields.is_eq()) - .map(|op| eq::EqOperation::new(op.rv1, op.arg2, op.decode.fields.alu_signed2_or_invert())) - .collect(); - let bytewise_ops: Vec = cpu_ops - .iter() - .filter(|op| { - let f = &op.decode.fields; - !f.word_instr && (f.is_and() || f.is_or() || f.is_xor()) - }) - .map(|op| bytewise::BytewiseOperation::new(op.rv1, op.arg2, op.decode.fields.alu_op())) - .collect(); - // STORE: receives MEMORY(memory_op=1) from the CPU and sends the MEMW write - // at timestamp+1 (mirrors `collect_store_op_from_cpu`, which records the MEMW - // table row). - let store_ops: Vec = cpu_ops - .iter() - .filter(|op| op.decode.fields.is_store()) - .map(|op| { - // The MEMORY bus and the STORE chip's MEMW write share the base - // timestamp (spec store.toml uses one `timestamp` for both). - store::StoreOperation::new( - op.res, - op.timestamp, - op.rv2, - op.decode.fields.mem_bytes() as u8, - ) - }) - .collect(); + let DerivedFromCpu { + branch_ops, + eq_ops, + bytewise_ops, + store_ops, + mut mul_ops, + mut dvrm_ops, + } = derive_from_cpu(&cpu_ops); // CPU32 (word `*W`) dispatch: each CPU32 row that uses the full ALU sends to // the SHIFT/MUL/DVRM chips (ADDW/SUBW are the CPU32 ADD/SUB fast-path). These @@ -3175,6 +4136,11 @@ fn collect_all_ops( } CollectedOps { + retired_memw_lt: Vec::new(), + retired_memw_aligned_lt: Vec::new(), + retired_cpu32_shift: Vec::new(), + retired_cpu32_mul: Vec::new(), + retired_cpu32_dvrm: Vec::new(), cpu_ops, memw_ops, memw_aligned_ops, @@ -3217,8 +4183,16 @@ fn build_traces( private_input: &[u8], is_final: bool, l2g_memory_bookend: bool, -) -> Result { + // `true` builds the chunked tables as empty placeholders, leaving the + // returned `CollectedOps` as the only way to get their rows. + retire_chunked: bool, +) -> Result<(Traces, CollectedOps), Error> { let CollectedOps { + retired_memw_lt: _, + retired_memw_aligned_lt: _, + retired_cpu32_shift: _, + retired_cpu32_mul: _, + retired_cpu32_dvrm: _, cpu_ops, memw_ops, memw_aligned_ops, @@ -3388,7 +4362,10 @@ fn build_traces( { base.add_ops(&bitwise_ops); memw_register::collect_bitwise_from_memw_register(&memw_register_rows, &mut base); - for f in &collectors { + // By value, like the parallel arm's `units.extend(collectors)`: these + // closures borrow the op lists, and the lists are moved into + // `CollectedOps` below, so the collectors have to be dropped here. + for f in collectors { f(&mut base); } } @@ -3432,137 +4409,70 @@ fn build_traces( // Each build below reads disjoint op lists and writes its own table, so // they all run in one rayon scope. Disk-spill stays sequential: its // generate→spill order keeps trace memory bounded. - let cpu_ops_ref = &cpu_ops; - let gen_cpus = || { - chunk_and_generate( - cpu_ops_ref, - max_rows.cpu, - cpu::generate_cpu_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_memws = || { - chunk_and_generate_optional( - &memw_ops, - max_rows.memw, - memw::generate_memw_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_memw_aligneds = || { - chunk_and_generate_optional( - &memw_aligned_ops, - max_rows.memw_aligned, - memw_aligned::generate_memw_aligned_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_memw_registers = || { - // Direct-to-column fill from compact RegRows — the register fast path never - // materializes a `Vec`. - chunk_and_generate( - &memw_register_rows, - max_rows.memw_register, - memw_register::generate_memw_register_trace_from_rows, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_loads = || { - chunk_and_generate_optional( - &load_ops, - max_rows.load, - load::generate_load_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_lts = || { - chunk_and_generate_optional( - <_ops, - max_rows.lt, - lt::generate_lt_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_shifts = || { - chunk_and_generate_optional( - &shift_ops, - max_rows.shift, - shift::generate_shift_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_muls = || { - chunk_and_generate_optional( - &mul_ops, - max_rows.mul, - mul::generate_mul_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_dvrms = || { - chunk_and_generate_optional( - &dvrm_ops, - max_rows.dvrm, - dvrm::generate_dvrm_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_branches = || { - chunk_and_generate_optional( - &branch_ops, - max_rows.branch, - branch::generate_branch_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - // Auxiliary ALU / memory / CPU32 dispatch chips, each filtered out of the CPU - // ops above. - let gen_eqs = || { - chunk_and_generate_optional::( - &eq_ops, - max_rows.eq, - eq::generate_eq_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_bytewises = || { - chunk_and_generate_optional::( - &bytewise_ops, - max_rows.bytewise, - bytewise::generate_bytewise_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_stores = || { - chunk_and_generate_optional::( - &store_ops, - max_rows.store, - store::generate_store_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) - }; - let gen_cpu32s = || { - chunk_and_generate_optional::( - &cpu32_ops, - max_rows.cpu32, - cpu32::generate_cpu32_trace, - #[cfg(feature = "disk-spill")] - storage_mode, - ) + // Phases 3-4 are settled, so every cross-table coupling is already folded in + // and each of these tables is now a pure function of one routed op list. + // Pack them into `CollectedOps`: the same intermediate builds them here and can + // rebuild any one of them later, which is what a retired trace needs. + let routed = CollectedOps { + cpu_ops, + memw_ops, + memw_aligned_ops, + memw_register_rows, + load_ops, + lt_ops, + shift_ops, + branch_ops, + mul_ops, + dvrm_ops, + eq_ops, + bytewise_ops, + store_ops, + cpu32_ops, + // The accumulators stay with the phase-5 closures below: this value + // exists to rebuild the chunked tables, which do not read them. + ..Default::default() }; + let cpu_ops_ref = &routed.cpu_ops; + + // Each build below reads disjoint op lists and writes its own table, so + // they all run in one rayon scope. Disk-spill stays sequential: its + // generate→spill order keeps trace memory bounded. + macro_rules! gen_of { + ($kind:ident) => { + || { + if retire_chunked { + // Placeholders: the right number of chunks, none of the rows. + // `table_counts` (and so the AIR layout) only reads the chunk + // count, and the prover asks the provider for every shape it + // needs before a trace exists. + return Ok((0..routed.num_chunks(TableKind::$kind, max_rows)) + .map(|_| TraceTable::from_columns_main(Vec::new(), 1)) + .collect()); + } + routed.build_table( + TableKind::$kind, + max_rows, + #[cfg(feature = "disk-spill")] + storage_mode, + ) + } + }; + } + let gen_cpus = gen_of!(Cpu); + let gen_memws = gen_of!(Memw); + let gen_memw_aligneds = gen_of!(MemwAligned); + let gen_memw_registers = gen_of!(MemwRegister); + let gen_loads = gen_of!(Load); + let gen_lts = gen_of!(Lt); + let gen_shifts = gen_of!(Shift); + let gen_muls = gen_of!(Mul); + let gen_dvrms = gen_of!(Dvrm); + let gen_branches = gen_of!(Branch); + let gen_eqs = gen_of!(Eq); + let gen_bytewises = gen_of!(Bytewise); + let gen_stores = gen_of!(Store); + let gen_cpu32s = gen_of!(Cpu32); + let gen_bitwise = || { let mut bitwise = bitwise::generate_bitwise_trace(); // Fill the MU columns (11..=20) from the accumulated histogram. @@ -3811,7 +4721,7 @@ fn build_traces( #[cfg(feature = "instruments")] drop(__sp); - Ok(Traces { + let traces = Traces { cpus, bitwise, lts, @@ -3842,7 +4752,9 @@ fn build_traces( bytewises, stores, cpu32s, - }) + }; + + Ok((traces, routed)) } /// Padded row count after chunking, for a table that is always present: an @@ -4647,37 +5559,7 @@ impl Traces { /// Runtime (non-ELF) pages are identified by `init_values == None` /// (zero-init), avoiding a redundant ELF segment scan. pub fn runtime_page_ranges(&self) -> Vec { - let page_size = page::DEFAULT_PAGE_SIZE as u64; - - // Collect sorted non-ELF page bases (zero-init pages are runtime pages) - let runtime_bases: Vec = self - .page_configs - .iter() - .filter(|config| config.init_values.is_none()) - .map(|config| config.page_base) - .collect(); - - // Run-length encode contiguous pages into (base, count) ranges - let mut ranges = Vec::new(); - if runtime_bases.is_empty() { - return ranges; - } - - let mut start = runtime_bases[0]; - let mut count = 1u64; - - for &base in &runtime_bases[1..] { - if base == start + count * page_size { - count += 1; - } else { - ranges.push(crate::RuntimePageRange { base: start, count }); - start = base; - count = 1; - } - } - ranges.push(crate::RuntimePageRange { base: start, count }); - - ranges + runtime_page_ranges(&self.page_configs) } /// Generates all traces from ELF and execution logs using phased collection. @@ -4712,6 +5594,43 @@ impl Traces { ) } + /// `from_elf_and_logs`, retiring the chunked tables. + /// + /// The returned `Traces` carries an empty placeholder per chunk — the right + /// count, none of the rows — and the `CollectedOps` beside it is what rebuilds + /// any of them on demand. + pub(crate) fn from_elf_and_logs_streaming( + elf: &Elf, + max_rows: &super::MaxRowsConfig, + private_input: &[u8], + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + ) -> Result<(Self, CollectedOps), Error> { + let initial_image = build_initial_image(elf, private_input); + let register_init = register::register_init_from_entry_point(elf.entry_point); + let artifacts = DecodeArtifacts::from_elf(elf)?; + // Drives its own executor: no caller holds the logs for it. + let collected = Self::collect_epoch_streaming( + &artifacts, + elf, + private_input.to_vec(), + &initial_image, + ®ister_init, + true, + )?; + Self::build_from_collected_streaming( + &artifacts, + collected, + Some(&initial_image), + ®ister_init, + max_rows, + private_input, + true, + false, + #[cfg(feature = "disk-spill")] + storage_mode, + ) + } + /// Build traces for one execution epoch starting from an explicit /// initial-memory image (the epoch's starting memory) rather than the ELF /// image. `elf` is still used for the program code (DECODE) and entry point. @@ -4785,6 +5704,253 @@ impl Traces { ) } + /// Walk the execution and hand each chunked table's chunk to `on_chunk` as + /// soon as it is full, dropping its ops right after. + /// + /// This is Approach 1's Commit phase seen from the producer side: the spec + /// has the prover commit tables "once the memory pressure becomes too + /// large" and drop them, which it can only do if the tables arrive while + /// the execution is still being walked. The buffers of the kinds listed in + /// [`CHUNKED_KINDS`] never exceed one chunk each; the rest — the four the + /// paragraph below names, the `retired_*` rows a closing chunk converts its + /// ops into, the accumulators and the walk's own BITWISE lookups — are held + /// whole and grow with the run. See `crate::pass`'s header for the size of + /// that term. + /// + /// The chunks come out exactly as `ops.chunks(max_rows)` would cut them and + /// each is built by the same generator, so a consumer sees byte-identical + /// traces to the all-at-once path — `commit_walk_emits_the_same_chunks` + /// pins that. + /// + /// Only the tables whose ops are final when the segment ends are emitted. + /// SHIFT is not one of them despite coming out of `collect_ops_from_cpu`: + /// `cpu32_chip_op` appends to it for every word instruction, so a program + /// with `*W` ops would have its SHIFT chunks cut elsewhere than the + /// finished run cuts them. + /// LT and MUL are not among them — later derivations append to both (DVRM + /// contributes range checks to LT and a product to MUL) — and neither are + /// the tables `collect_all_ops` derives from the CPU ops, nor the + /// accumulators (BITWISE), nor PAGE/DECODE/REGISTER, which are only final + /// once the run is over. Those are the spec's "remaining tables are padded + /// and committed" at the end; extracting the per-op derivations so they can + /// be emitted mid-walk too is the next step, not a guess to make here. + pub fn walk_and_emit_chunks( + artifacts: &DecodeArtifacts, + elf: &Elf, + private_input: Vec, + initial_image: &impl ImageSource, + register_init: &[u32], + max_rows: &super::MaxRowsConfig, + mut on_chunk: impl FnMut(TableKind, usize, TraceTable), + ) -> Result { + let mut executor = executor::vm::execution::Executor::new(elf, private_input) + .map_err(|e| Error::Prover(format!("executor: {e}")))?; + let mut memory_state = MemoryState::from_image(initial_image); + let mut register_state = RegisterState::from_init(register_init); + + let mut buf = CollectedOps::default(); + let mut bitwise_hist = bitwise::BitwiseHistogram::new(); + let mut decode_counts: HashMap = HashMap::new(); + let mut padding_rows = 0usize; + // HALT is built from the run's terminating ECALL, and the CPU ops that + // carry it are dropped as their chunk closes, so it is noted in passing. + let mut last_ecall: Option<(u64, u64)> = None; + let mut emitted = [0usize; CHUNKED_KINDS.len()]; + let mut cycles_so_far = 0usize; + + while let Some(logs) = executor + .resume() + .map_err(|e| Error::Prover(format!("executor: {e}")))? + { + let cpu = collect_cpu_ops(logs, &artifacts.instructions, cycles_so_far)?; + cycles_so_far += cpu.len(); + let (memw, ld, lt, sh, bw, cm, kc, c32, ec, ed, hn) = + collect_ops_from_cpu(&cpu, &mut memory_state, &mut register_state); + // Derived from THIS segment's ops, before they are moved into the + // buffer — the buffer is drained as chunks close, so it is not the + // segment. + // DECODE counts one lookup per executed cycle. Counting by pc keeps + // that bounded by the program instead of by the run, which is what + // lets the CPU ops be dropped at all. + for op in &cpu { + *decode_counts.entry(op.decode.pc).or_insert(0) += 1; + if op.decode.fields.ecall { + last_ecall = Some((op.timestamp, op.next_pc)); + } + } + let derived = derive_from_cpu(&cpu); + buf.branch_ops.extend(derived.branch_ops); + buf.eq_ops.extend(derived.eq_ops); + buf.bytewise_ops.extend(derived.bytewise_ops); + buf.store_ops.extend(derived.store_ops); + buf.mul_ops.extend(derived.mul_ops); + buf.dvrm_ops.extend(derived.dvrm_ops); + buf.cpu_ops.extend(cpu); + buf.memw_register_rows.extend(memw.register_rows); + buf.memw_aligned_ops.extend(memw.aligned); + buf.memw_ops.extend(memw.general); + buf.load_ops.extend(ld); + buf.lt_ops.extend(lt); + buf.shift_ops.extend(sh); + buf.cpu32_ops.extend(c32); + // Never closed mid-walk: these are accumulators or accelerator + // tables built once, at the end, from the whole run. + buf.bitwise_ops.extend(bw); + buf.commit_ops.extend(cm); + buf.keccak_ops.extend(kc); + buf.ecsm_ops.extend(ec); + buf.ecdas_ops.extend(ed); + buf.hint_ops.extend(hn); + + // Emit every chunk that is now full, and only those: a partial chunk + // may still grow, so it waits for the end. + for (slot, kind) in CHUNKED_KINDS.iter().enumerate() { + while buf.buffered(*kind) >= max_rows_for(*kind, max_rows) { + let limit = max_rows_for(*kind, max_rows); + // Before the ops go: BITWISE counts them across the whole + // run, and this chunk is about to stop existing. + buf.fold_bitwise_from_front(*kind, limit, &mut bitwise_hist); + // So does LT, which owes a row to every MEMW timestamp check + // and is never closed mid-walk itself. + match kind { + TableKind::Memw => { + let derived = collect_lt_from_memw(&buf.memw_ops[..limit]); + buf.retired_memw_lt.extend(derived); + } + TableKind::MemwAligned => { + let derived = + collect_lt_from_memw_aligned(&buf.memw_aligned_ops[..limit]); + buf.retired_memw_aligned_lt.extend(derived); + } + // CPU32 rows dispatch to SHIFT, MUL and DVRM. + TableKind::Cpu32 => { + for c in &buf.cpu32_ops[..limit] { + cpu32_chip_op( + c, + &mut buf.retired_cpu32_shift, + &mut buf.retired_cpu32_mul, + &mut buf.retired_cpu32_dvrm, + ); + } + } + _ => {} + } + if *kind == TableKind::Cpu { + // Each CPU chunk pads to a power of two, and every + // padding row looks DECODE up at the padding pc. + padding_rows += limit.next_power_of_two().max(4) - limit; + } + let table = buf.take_front(*kind, limit, max_rows); + on_chunk(*kind, emitted[slot], table); + emitted[slot] += 1; + } + } + } + + // The tail is deliberately NOT emitted. End-of-run finalization still + // appends to these op lists — the terminating ECALL's register writes + // land in MEMW — so a partial chunk is not final until the execution + // is over. That is the spec's own split: full tables are committed and + // retired during the walk, and "at the end of the execution, the + // remaining tables are padded and committed". What is left goes back to + // the caller so it can do exactly that. + Ok(WalkLeftover { + tail: buf, + retired_bitwise: bitwise_hist, + decode_counts, + padding_rows, + total_cpu_padding: None, + last_ecall, + memory_state, + emitted, + register_state, + cycles: cycles_so_far, + }) + } + + /// `collect_epoch`, driving the executor itself and consuming its logs one + /// chunk at a time instead of taking them all at once. + /// + /// Approach 1's Commit phase walks the execution and retires what it has + /// finished with; it cannot start by materializing every log. Phases 1-3 are + /// segment-local given the carried state — `MemoryState` and `RegisterState` + /// thread through, and the LT ops a MEMW access implies come from the + /// timestamps that access already carries, not from a global ordering — so + /// the same ops come out in the same order, and only one chunk of logs is + /// ever resident. + /// + /// `collect_streaming_matches_collect_epoch` pins that equality. + pub(crate) fn collect_epoch_streaming( + artifacts: &DecodeArtifacts, + elf: &Elf, + private_input: Vec, + initial_image: &I, + register_init: &[u32], + is_final: bool, + ) -> Result { + let mut executor = executor::vm::execution::Executor::new(elf, private_input) + .map_err(|e| Error::Prover(format!("executor: {e}")))?; + + let mut memory_state = MemoryState::from_image(initial_image); + let mut register_state = RegisterState::from_init(register_init); + + let mut cpu_ops: Vec = Vec::new(); + let mut memw = MemwBuckets::with_register_capacity(0); + let (mut load_ops, mut lt_ops, mut shift_ops, mut bitwise_ops) = + (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + let (mut commit_ops, mut keccak_ops, mut cpu32_ops) = (Vec::new(), Vec::new(), Vec::new()); + let (mut ecsm_ops, mut ecdas_ops, mut hint_ops) = (Vec::new(), Vec::new(), Vec::new()); + + let mut cycles_so_far = 0usize; + while let Some(chunk) = executor + .resume() + .map_err(|e| Error::Prover(format!("executor: {e}")))? + { + if !is_final && chunk.iter().any(|log| log.next_pc == 0) { + return Err(Error::HaltInNonFinalEpoch); + } + let chunk_cpu = collect_cpu_ops(chunk, &artifacts.instructions, cycles_so_far)?; + let (m, ld, lt, sh, bw, cm, kc, c32, ec, ed, hn) = + collect_ops_from_cpu(&chunk_cpu, &mut memory_state, &mut register_state); + memw.append(m); + load_ops.extend(ld); + lt_ops.extend(lt); + shift_ops.extend(sh); + bitwise_ops.extend(bw); + commit_ops.extend(cm); + keccak_ops.extend(kc); + cpu32_ops.extend(c32); + ecsm_ops.extend(ec); + ecdas_ops.extend(ed); + hint_ops.extend(hn); + cycles_so_far += chunk_cpu.len(); + cpu_ops.extend(chunk_cpu); + } + + let ops = collect_all_ops( + cpu_ops, + memw, + load_ops, + lt_ops, + shift_ops, + bitwise_ops, + commit_ops, + keccak_ops, + cpu32_ops, + ecsm_ops, + ecdas_ops, + hint_ops, + &mut register_state, + is_final, + ); + + Ok(CollectedEpoch { + ops, + memory_state, + register_state, + }) + } + /// The sequential-critical half of an epoch's trace build: log collection /// and op routing (Phases 1-2), which read the pre-epoch memory image and /// produce the epoch's memory/register end state. This must run in epoch @@ -4809,7 +5975,7 @@ impl Traces { // Phase 1: Logs → CPU operations #[cfg(feature = "instruments")] let __sp = stark::instruments::span("p1_cpu_ops"); - let cpu_ops = collect_cpu_ops(logs, &artifacts.instructions)?; + let cpu_ops = collect_cpu_ops(logs, &artifacts.instructions, 0)?; #[cfg(feature = "instruments")] drop(__sp); @@ -4868,6 +6034,36 @@ impl Traces { /// producer collects the next epoch. `initial_image` is only used for PAGE /// tables and their bitwise lookups, both skipped in continuation mode /// (`l2g_memory_bookend`), where callers pass `None`. + #[allow(clippy::too_many_arguments)] + /// `build_from_collected`, retiring the chunked tables: they come back as + /// empty placeholders and the returned `CollectedOps` is what rebuilds them. + #[allow(clippy::too_many_arguments)] + pub(crate) fn build_from_collected_streaming( + artifacts: &DecodeArtifacts, + collected: CollectedEpoch, + initial_image: Option<&I>, + register_init: &[u32], + max_rows: &super::MaxRowsConfig, + private_input: &[u8], + is_final: bool, + l2g_memory_bookend: bool, + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + ) -> Result<(Self, CollectedOps), Error> { + Self::build_from_collected_inner( + artifacts, + collected, + initial_image, + register_init, + max_rows, + private_input, + is_final, + l2g_memory_bookend, + #[cfg(feature = "disk-spill")] + storage_mode, + true, + ) + } + #[allow(clippy::too_many_arguments)] pub fn build_from_collected( artifacts: &DecodeArtifacts, @@ -4880,6 +6076,35 @@ impl Traces { l2g_memory_bookend: bool, #[cfg(feature = "disk-spill")] storage_mode: StorageMode, ) -> Result { + Self::build_from_collected_inner( + artifacts, + collected, + initial_image, + register_init, + max_rows, + private_input, + is_final, + l2g_memory_bookend, + #[cfg(feature = "disk-spill")] + storage_mode, + false, + ) + .map(|(traces, _routed)| traces) + } + + #[allow(clippy::too_many_arguments)] + fn build_from_collected_inner( + artifacts: &DecodeArtifacts, + collected: CollectedEpoch, + initial_image: Option<&I>, + register_init: &[u32], + max_rows: &super::MaxRowsConfig, + private_input: &[u8], + is_final: bool, + l2g_memory_bookend: bool, + #[cfg(feature = "disk-spill")] storage_mode: StorageMode, + retire_chunked: bool, + ) -> Result<(Self, CollectedOps), Error> { // Phase 0 (cached): the pristine DECODE trace is cloned so // `build_traces` can fill this epoch's multiplicities. #[cfg(feature = "instruments")] @@ -4906,6 +6131,7 @@ impl Traces { private_input, is_final, l2g_memory_bookend, + retire_chunked, ); #[cfg(feature = "instruments")] drop(__sp); @@ -4924,7 +6150,7 @@ impl Traces { max_rows: &super::MaxRowsConfig, ) -> Result { // Phase 1: Logs → CPU operations - let cpu_ops = collect_cpu_ops(logs, &instructions)?; + let cpu_ops = collect_cpu_ops(logs, &instructions, 0)?; // Phase 2: Collect + route all ops let mut memory_state = MemoryState::new(); @@ -4980,6 +6206,8 @@ impl Traces { &[], true, false, + false, ) + .map(|(traces, _routed)| traces) } } diff --git a/prover/src/tests/auto_storage_tests.rs b/prover/src/tests/auto_storage_tests.rs index e26674d27..bf6cd36af 100644 --- a/prover/src/tests/auto_storage_tests.rs +++ b/prover/src/tests/auto_storage_tests.rs @@ -135,3 +135,39 @@ fn unbounded_k_inflates_peak_bytes_on_many_page_shapes() { "expected >20 % inflation, got {bounded} -> {unbounded}" ); } + +#[test] +fn retire_lde_off_when_estimate_below_threshold() { + // 10 GB estimated, 32 GB available → threshold 28.8 GB → the main LDE is + // affordable, so do not pay ~11% of prove time to retire it. + assert!(!crate::auto_storage::retire_lde_for(10 * GB, Some(32 * GB))); +} + +#[test] +fn retire_lde_on_when_estimate_exceeds_threshold() { + // 30 GB estimated, 32 GB available → threshold 28.8 GB → about to swap. + assert!(crate::auto_storage::retire_lde_for(30 * GB, Some(32 * GB))); +} + +#[test] +fn retire_lde_on_when_available_ram_is_unknown() { + assert!(crate::auto_storage::retire_lde_for(10 * GB, None)); +} + +/// The documented policy: retire-LDE and disk-spill share one trigger, so a +/// change to the storage threshold can never silently desynchronize them. +#[test] +fn retire_lde_agrees_with_disk_spill_on_every_side_of_the_threshold() { + for (estimated, available) in [ + (10 * GB, Some(32 * GB)), + (30 * GB, Some(32 * GB)), + (GB, Some(2 * GB)), + (10 * GB, None), + ] { + assert_eq!( + crate::auto_storage::retire_lde_for(estimated, available), + select_storage_mode(estimated, available) == StorageMode::Disk, + "policy drifted for estimated={estimated} available={available:?}" + ); + } +} diff --git a/prover/src/tests/batched_fri_tests.rs b/prover/src/tests/batched_fri_tests.rs new file mode 100644 index 000000000..6df85c4f9 --- /dev/null +++ b/prover/src/tests/batched_fri_tests.rs @@ -0,0 +1,730 @@ +//! The batched FRI, pinned at the one size where it has to agree with the +//! unbatched one. + +use crate::tables::MaxRowsConfig; +use crate::tables::types::{GoldilocksExtension, GoldilocksField}; +use executor::elf::Elf; +use stark::prover::IsStarkProver; +use stark::prover::TableDeep; + +/// A batch of one must reproduce the proof's FRI exactly. +/// +/// `Σ αᵏ·deepₖ` over a single member is `deepₖ`, so the batched path and the +/// per-table one fold the same codeword over the same domain. If their layer +/// roots differ, the difference is in the codeword or in the domain — which is +/// the whole substance of the batching, and worth catching before any group has +/// more than one member in it. +#[test] +fn a_batch_of_one_matches_the_unbatched_fri() { + type P = stark::prover::Prover; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let vm_proof = crate::prove_with_options_and_inputs(&elf_bytes, &[], &proof_options, &max_rows) + .expect("ordinary prove"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + + let page_configs = crate::tables::trace_builder::Traces::page_configs_from_elf_and_runtime( + &elf, + &vm_proof.runtime_page_ranges, + vm_proof.num_private_input_pages, + vm_proof.proof.proofs.len(), + ) + .expect("page configs"); + let airs = crate::VmAirs::new( + &elf, + &proof_options, + false, + &page_configs, + &vm_proof.table_counts, + None, + true, + None, + None, + None, + ); + let mut resident = crate::logup_phase::walk_only(&elf, &[], &max_rows).expect("resident"); + + // BITWISE is table 0 and the largest resident one, so it exercises a real + // domain rather than a one-row corner. + let idx = 0usize; + let n = challenge.roots.len(); + let mut fork = crate::logup_phase::fork_for(&challenge, idx, n); + let deep =

>::deep_for_table( + airs.bitwise.as_ref(), + &(), + &mut resident.bitwise, + &challenge.challenges, + &mut fork, + None, + false, + ) + .expect("deep"); + + // A batch of one: the accumulator is the codeword itself, folded with a + // coefficient of one. + let one = math::field::element::FieldElement::::one(); + let mut acc = Vec::new(); +

>::accumulate(&mut acc, &one, &deep.deep); + let fri =

>::batch_fri( + airs.bitwise.as_ref(), + acc, + deep.trace_rows, + &mut fork, + ) + .expect("batched fri"); + + // Everything round 4 would have produced for this table on its own: the + // layers, the final polynomial, the ground nonce, and the queries its + // openings answer. + let want = &vm_proof.proof.proofs[idx]; + assert_eq!( + fri.layer_roots, want.fri_layers_merkle_roots, + "a batch of one folded to a different FRI than the proof carries" + ); + assert_eq!( + fri.final_poly_coeffs, want.fri_final_poly_coeffs, + "a batch of one folded to a different final polynomial" + ); + // And there the comparison stops. Grinding searches for a nonce in parallel + // and finds whichever one it finds first, so two runs over the same + // transcript state produce different valid nonces — and the queries are + // sampled after the nonce is absorbed, so they differ with it. What is + // deterministic is everything up to that point, which is what is checked + // above; the queries are checked instead by the count they produce. + assert_eq!( + fri.query_list.len(), + want.query_list.len(), + "a batch of one answered a different number of queries" + ); + assert_eq!( + fri.iotas.len(), + want.query_list.len(), + "the group sampled a different number of query indices than it decommitted" + ); +} + +/// Every table's coefficient must depend on every table folded before it. +/// +/// The coefficients are what make a batched fold binding: if one could be drawn +/// without some table's round-3 data, that table could be swapped after the +/// coefficient was fixed and the fold would still check out. So the property to +/// pin is not that the derivation runs — it is that moving any single field any +/// table contributes moves the sequence. +/// +/// Drawn one per table, after that table's data goes in, which is what the spec +/// asks for and what lets a codeword be folded and dropped as it is produced +/// instead of every codeword being held to the end. +/// +/// Built from data rather than from a proof on purpose: this is about the byte +/// order, and a synthetic table exercises every field including the ones a real +/// fixture might leave empty. +#[test] +fn alpha_moves_when_any_table_moves() { + use crypto::fiat_shamir::default_transcript::DefaultTranscript; + use math::field::element::FieldElement; + use stark::table::Table; + type P = stark::prover::Prover; + type E = GoldilocksExtension; + + let table = |seed: u64| TableDeep:: { + lde_size: 8, + trace_rows: 4, + deep: Vec::new(), + bus_contribution: Some(FieldElement::::from(seed)), + main_roots: stark::prover::MainRoots { + precomputed: None, + main: [0u8; 32], + }, + aux_root: None, + bus_public_inputs: None, + composition_poly_root: [seed as u8; 32], + trace_ood: Table::new(vec![FieldElement::::from(seed + 1)], 1), + trace_ood_next: Table::new(vec![FieldElement::::from(seed + 2)], 1), + parts_ood: vec![FieldElement::::from(seed + 3)], + composition_lde: None, + }; + + let pre_fork = DefaultTranscript::::new(&[7, 7, 7]); + let base = vec![table(1), table(2)]; + let alphas =

>::fold_coefficients(&pre_fork, &base); + + // Every field of every table, one at a time. + type Mutation = (&'static str, Box>)>); + let mutate: Vec = vec![ + ( + "bus", + Box::new(|t: &mut Vec>| { + t[0].bus_contribution = Some(FieldElement::::from(99)) + }), + ), + ( + "root", + Box::new(|t: &mut Vec>| t[1].composition_poly_root = [9u8; 32]), + ), + ( + "ood", + Box::new(|t: &mut Vec>| { + t[0].trace_ood = Table::new(vec![FieldElement::::from(99)], 1) + }), + ), + ( + "ood_next", + Box::new(|t: &mut Vec>| { + t[1].trace_ood_next = Table::new(vec![FieldElement::::from(99)], 1) + }), + ), + ( + "parts", + Box::new(|t: &mut Vec>| { + t[0].parts_ood = vec![FieldElement::::from(99)] + }), + ), + ]; + for (what, f) in mutate { + let mut moved = base.clone(); + f(&mut moved); + assert_ne!( + alphas, +

>::fold_coefficients(&pre_fork, &moved), + "the coefficients ignore {what}, so that data is not bound to the fold" + ); + } + + // And order is part of it: the same tables the other way round are a + // different batch. + let swapped = vec![base[1].clone(), base[0].clone()]; + assert_ne!( + alphas, +

>::fold_coefficients(&pre_fork, &swapped), + "the coefficients ignore the table order, which the verifier replays" + ); +} + +/// The driver must fold every table, and fold them by domain. +/// +/// Two things can silently go wrong at once here. A table can go missing — the +/// walk produces the chunked ones and the end-of-run step the rest, and a batch +/// that skips one is not a smaller batch, it is a wrong one. And tables of +/// different domains can end up in the same group, which the fold cannot +/// express: it squares the coset offset each layer, so a short codeword never +/// lines up with a tall fold. +/// +/// So this checks the count against a real proof's table count, and that every +/// group is one domain with at least one member, and that the collapse actually +/// happened. +#[test] +fn the_driver_folds_every_table_grouped_by_domain() { + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let vm_proof = crate::prove_with_options_and_inputs(&elf_bytes, &[], &proof_options, &max_rows) + .expect("ordinary prove"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + let batched = crate::logup_phase::run_batched(&elf, &[], &max_rows, &proof_options, &challenge) + .expect("batched phase"); + + let folded: usize = batched.members.iter().sum(); + assert_eq!( + folded, + vm_proof.proof.proofs.len(), + "the driver folded {folded} tables but the proof has {}", + vm_proof.proof.proofs.len() + ); + assert!( + batched.groups.len() < folded, + "{} groups for {folded} tables is no collapse at all", + batched.groups.len() + ); + // A group commits layers exactly when there is something to fold. FRI stops + // at the final polynomial, whose CODEWORD is the blowup times its degree + // bound — so with blowup 2 and a degree bound of 2^7, a 256-long codeword is + // already terminal and folds zero times. The short tables (one row blown up + // to two) are terminal for the same reason. An empty group there is correct, + // not a group that failed. + let terminal = + (1usize << proof_options.fri_final_poly_log_degree) * proof_options.blowup_factor as usize; + for ((lde_size, fri), count) in batched.groups.iter().zip(batched.members.iter()) { + assert!(*count > 0, "a group of {lde_size} folded nothing"); + assert_eq!( + !fri.layer_roots.is_empty(), + *lde_size > terminal, + "a group of {lde_size} committed {} layers against a terminal of {terminal}", + fri.layer_roots.len() + ); + assert!( + !fri.iotas.is_empty(), + "a group of {lde_size} sampled no queries for its members to open at" + ); + } + // Every table knows its group, and it is the group of its own domain. The + // Open pass reads this to take a table's openings at the right indices, so + // a table pointing at the wrong group opens against a FRI that never folded + // it. + assert_eq!(batched.group_of.len(), folded, "a table has no group"); + for (idx, g) in batched.group_of.iter().enumerate() { + assert!( + *g < batched.groups.len(), + "table {idx} points at group {g}, past the {} there are", + batched.groups.len() + ); + } + let mut per_group = vec![0usize; batched.groups.len()]; + for g in &batched.group_of { + per_group[*g] += 1; + } + assert_eq!( + per_group, batched.members, + "the tables' groups disagree with what each group says it folded" + ); + + // Domains are distinct: a repeated one would mean two groups that should + // have been one, which is a fold that did not happen. + let mut sizes: Vec = batched.groups.iter().map(|(s, _)| *s).collect(); + let before = sizes.len(); + sizes.dedup(); + assert_eq!(before, sizes.len(), "two groups share a domain"); +} + +/// The Open pass must serve every table, at its own group's indices. +/// +/// This is Approach 1's fifth pass and the last one: the query indices do not +/// exist until the batched FRI is over, so it could not have been folded into +/// an earlier walk. What it produces is what a verifier will authenticate, so +/// the two things that matter are that no table is missing and that each opened +/// as many rows as its group asked for — a table opening the wrong count is a +/// table answering a different FRI. +#[test] +fn the_open_pass_serves_every_table_at_its_group() { + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + let batched = crate::logup_phase::run_batched(&elf, &[], &max_rows, &proof_options, &challenge) + .expect("batched phase"); + let opened = + crate::logup_phase::run_open(&elf, &[], &max_rows, &proof_options, &challenge, &batched) + .expect("open pass"); + + assert_eq!( + opened.openings.len(), + batched.group_of.len(), + "the Open pass served a different number of tables than the batch folded" + ); + for (idx, opening) in opened.openings.iter().enumerate() { + let g = batched.group_of[idx]; + let wanted = batched.groups[g].1.iotas.len(); + assert_eq!( + opening.len(), + wanted, + "table {idx} opened {} rows for a group that asked {wanted}", + opening.len() + ); + } + + // The count above is weaker than it looks: the number of queries is a + // global option, so every group asks for the same number and a table given + // the wrong group's indices still opens the right count. What separates the + // groups is the indices themselves, which are sampled from different + // transcript states and — this is the part that must hold — addressed + // against different domains. An index from a taller group is simply not a + // row a shorter one has. + for (lde_size, fri) in batched.groups.iter() { + for iota in fri.iotas.iter() { + assert!( + iota < lde_size, + "a group of {lde_size} sampled index {iota}, which is not a row it has" + ); + } + } +} + +/// The batched proof must carry, per table, what the per-table proof carries — +/// minus exactly the FRI. +/// +/// The claim behind the whole format is that nothing is lost by moving the FRI +/// to the group: a table still answers for its own roots, its own out-of-domain +/// values and its own rows. So every one of those is compared against a real +/// per-table proof, table by table. What is absent is the four things that +/// became the group's, and those are what the size saving is made of. +#[test] +fn the_batched_proof_keeps_everything_but_the_fri() { + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let vm_proof = crate::prove_with_options_and_inputs(&elf_bytes, &[], &proof_options, &max_rows) + .expect("ordinary prove"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + let batched = crate::logup_phase::run_batched(&elf, &[], &max_rows, &proof_options, &challenge) + .expect("batched phase"); + let opened = + crate::logup_phase::run_open(&elf, &[], &max_rows, &proof_options, &challenge, &batched) + .expect("open pass"); + let proof = crate::logup_phase::assemble_batched_proof(batched, opened).expect("assemble"); + + assert_eq!( + proof.tables.len(), + vm_proof.proof.proofs.len(), + "the batched proof covers a different number of tables" + ); + for (idx, (got, want)) in proof + .tables + .iter() + .zip(vm_proof.proof.proofs.iter()) + .enumerate() + { + assert_eq!( + got.main_root, want.lde_trace_main_merkle_root, + "table {idx}: a different main root" + ); + assert_eq!( + got.aux_root, want.lde_trace_aux_merkle_root, + "table {idx}: a different auxiliary root" + ); + assert_eq!( + got.precomputed_root, want.lde_trace_precomputed_merkle_root, + "table {idx}: a different precomputed root" + ); + assert_eq!( + got.composition_poly_root, want.composition_poly_root, + "table {idx}: a different composition root" + ); + assert_eq!( + got.parts_ood, want.composition_poly_parts_ood_evaluation, + "table {idx}: different composition parts at z" + ); + assert_eq!( + (got.trace_ood.width, got.trace_ood.columns()), + ( + want.trace_ood_evaluations.width, + want.trace_ood_evaluations.columns() + ), + "table {idx}: different out-of-domain evaluations at z" + ); + assert_eq!( + got.trace_rows, want.trace_length, + "table {idx}: a different trace length" + ); + } + + // And the FRI is where it should be: nowhere per table, once per domain. + assert!( + proof.groups.len() < proof.tables.len(), + "{} groups for {} tables is no collapse", + proof.groups.len(), + proof.tables.len() + ); +} + +/// The verifier must derive the prover's challenges from the proof alone. +/// +/// This is the floor everything else stands on. A verifier that asks different +/// questions than the prover answered rejects valid proofs and, worse, may +/// accept invalid ones — the later checks all assume the challenges are the +/// ones the data was bound to. So the replay is pinned against the prover's own +/// values, not against a second implementation of the same idea. +/// +/// Note what it is NOT: deriving the right challenges does not make a proof +/// valid. It makes the questions right. Whether the answers are is what the +/// pieces after this decide. +#[test] +fn the_verifier_replays_the_provers_challenges() { + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + let batched = crate::logup_phase::run_batched(&elf, &[], &max_rows, &proof_options, &challenge) + .expect("batched phase"); + let counts = challenge.order.counts().clone(); + let opened = + crate::logup_phase::run_open(&elf, &[], &max_rows, &proof_options, &challenge, &batched) + .expect("open pass"); + let proof = crate::logup_phase::assemble_batched_proof(batched, opened).expect("assemble"); + + let replay = crate::batched_verifier::replay(&proof, &elf_bytes, &counts, &proof_options) + .expect("replay"); + + assert_eq!( + replay.logup, challenge.challenges, + "the verifier sampled a different shared challenge than the prover did" + ); + assert_eq!( + replay.coefficients.len(), + proof.tables.len(), + "the verifier derived a coefficient for a different number of tables" + ); + // Every coefficient distinct: two equal ones would mean two tables absorbed + // into the same seed state, which is the fold losing its binding. + let mut seen = replay.coefficients.clone(); + seen.sort_by_key(|c| format!("{c:?}")); + seen.dedup_by_key(|c| format!("{c:?}")); + assert_eq!( + seen.len(), + replay.coefficients.len(), + "two tables got the same fold coefficient" + ); +} + +fn batched_proof_of_fib() -> ( + Vec, + stark::proof::options::ProofOptions, + crate::logup_phase::BatchedProof, +) { + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + let batched = crate::logup_phase::run_batched(&elf, &[], &max_rows, &proof_options, &challenge) + .expect("batched phase"); + let opened = + crate::logup_phase::run_open(&elf, &[], &max_rows, &proof_options, &challenge, &batched) + .expect("open pass"); + let proof = crate::logup_phase::assemble_batched_proof(batched, opened).expect("assemble"); + (elf_bytes, proof_options, proof) +} + +/// The whole thing, from nothing but the proof, the program and the options. +#[test] +fn the_batched_proof_verifies() { + let (elf_bytes, proof_options, proof) = batched_proof_of_fib(); + assert!( + crate::batched_verifier::verify(&proof, &elf_bytes, &proof_options).expect("verify"), + "the batched proof does not verify" + ); +} + +/// Each part the verifier reads, changed on its own, is caught — and the +/// untouched proof still passes afterwards, so it is the change that was caught. +#[test] +fn a_tampered_batched_proof_is_rejected() { + let (elf_bytes, proof_options, mut proof) = batched_proof_of_fib(); + let one = math::field::element::FieldElement::::one(); + let rejected = |proof: &crate::logup_phase::BatchedProof, what: &str| { + assert!( + !matches!( + crate::batched_verifier::verify(proof, &elf_bytes, &proof_options), + Ok(true) + ), + "{what} was not caught" + ); + }; + let accepted = |proof: &crate::logup_phase::BatchedProof| { + assert!( + crate::batched_verifier::verify(proof, &elf_bytes, &proof_options).expect("verify"), + "the untouched proof no longer verifies" + ); + }; + + // A composition part at z. + let orig = proof.tables[0].parts_ood[0]; + proof.tables[0].parts_ood[0] = &orig + &one; + rejected(&proof, "a composition part at z"); + proof.tables[0].parts_ood[0] = orig; + accepted(&proof); + + // An opened value. + let orig = proof.openings[0][0].composition_poly.evaluations[0]; + proof.openings[0][0].composition_poly.evaluations[0] = &orig + &one; + rejected(&proof, "an opened composition value"); + proof.openings[0][0].composition_poly.evaluations[0] = orig; + accepted(&proof); + + // The fold order. + proof.fold_order.swap(0, 1); + rejected(&proof, "a swapped fold order"); + proof.fold_order.swap(0, 1); + accepted(&proof); + + // A group's final polynomial. + let orig = proof.groups[0].1.final_poly_coeffs[0]; + proof.groups[0].1.final_poly_coeffs[0] = &orig + &one; + rejected(&proof, "a final polynomial coefficient"); + proof.groups[0].1.final_poly_coeffs[0] = orig; + accepted(&proof); + + // A group's query index. + proof.groups[0].1.iotas[0] ^= 1; + rejected(&proof, "a query index"); + proof.groups[0].1.iotas[0] ^= 1; + accepted(&proof); + + // The public output, which the statement binds. + proof.public_output.push(0); + rejected(&proof, "a longer public output"); + proof.public_output.pop(); + accepted(&proof); + + // The layout. + proof.table_counts.cpu += 1; + rejected(&proof, "a layout with one more CPU chunk"); + proof.table_counts.cpu -= 1; + accepted(&proof); + + // A group's FRI layer commitment. This is the one thing batching actually + // relocated — per table before, per group now — so it is the commitment most + // worth pinning. It is bound only indirectly: the seed absorbs each layer + // root, so changing one moves the group's query indices. + // + // Groups at or below the terminal size commit no layers at all, so pick one + // that has some. Asserting that such a group exists keeps this from becoming + // a tamper that silently does nothing if the fixture's shape changes. + let with_layers = proof + .groups + .iter() + .position(|(_, fri)| !fri.layer_roots.is_empty()) + .expect("the fixture must produce at least one group that commits FRI layers"); + proof.groups[with_layers].1.layer_roots[0][0] ^= 1; + rejected(&proof, "a group's FRI layer root"); + proof.groups[with_layers].1.layer_roots[0][0] ^= 1; + accepted(&proof); + + // A table's round-1 main commitment. + proof.tables[0].main_root[0] ^= 1; + rejected(&proof, "a main trace root"); + proof.tables[0].main_root[0] ^= 1; + accepted(&proof); + + // A table's composition commitment, which the fold seed absorbs directly. + proof.tables[0].composition_poly_root[0] ^= 1; + rejected(&proof, "a composition root"); + proof.tables[0].composition_poly_root[0] ^= 1; + accepted(&proof); + + // An out-of-domain value, the other thing the fold seed binds. + let orig = *proof.tables[0].trace_ood.get(0, 0); + proof.tables[0].trace_ood.set(0, 0, &orig + &one); + rejected(&proof, "an out-of-domain trace value"); + proof.tables[0].trace_ood.set(0, 0, orig); + accepted(&proof); + + // A block whose advertised dimensions disagree with its data length. The + // verifier must REJECT this, not panic: the fold seed indexes both + // out-of-domain blocks at the dimensions the proof declares, so the shape + // has to be pinned to the AIR before that read rather than after it. + let orig_width = proof.tables[0].trace_ood.width; + proof.tables[0].trace_ood.width += 1; + rejected(&proof, "an out-of-domain block with a lying width"); + proof.tables[0].trace_ood.width = orig_width; + accepted(&proof); +} + +/// The lying width again, delivered the way a guest gets it: as bytes. +/// +/// `a_tampered_batched_proof_is_rejected` mounts this on a proof in memory, +/// which was the only way to mount it while `BatchedProof` had no derives — +/// the reason the guard that rejects it was documented as unreachable from +/// bytes. Serializing the format is the point of the batched path, so the +/// shape has to be pinned on the way out of the archive too. +/// +/// Rejection may come from rkyv's validation or from the verifier's own shape +/// check; which one catches it is not the property under test. What is: a +/// blob whose advertised width disagrees with its data yields no attestation, +/// and does not panic. +#[test] +fn a_lying_width_is_rejected_when_the_proof_arrives_as_bytes() { + let (elf_bytes, proof_options, mut proof) = batched_proof_of_fib(); + + // The honest proof attests after the round trip, so what the second half + // catches is the tamper and not the encoding. + let blob = + crate::recursion::encode_batched_guest_input(proof.clone(), &elf_bytes, &proof_options) + .expect("encode the untouched proof"); + assert!( + crate::recursion::verify_batched_and_attest(&blob, &proof_options) + .expect("verify the untouched blob") + .is_some(), + "the untouched proof does not attest after a round trip through bytes" + ); + + proof.tables[0].trace_ood.width += 1; + let blob = crate::recursion::encode_batched_guest_input(proof, &elf_bytes, &proof_options) + .expect("encode the tampered proof"); + assert!( + !matches!( + crate::recursion::verify_batched_and_attest(&blob, &proof_options), + Ok(Some(_)) + ), + "a block with a lying width attested after a round trip through bytes" + ); +} diff --git a/prover/src/tests/challenge_phase_tests.rs b/prover/src/tests/challenge_phase_tests.rs new file mode 100644 index 000000000..957167c79 --- /dev/null +++ b/prover/src/tests/challenge_phase_tests.rs @@ -0,0 +1,341 @@ +//! The Challenge phase must reconstruct the ordinary prover's transcript. + +use crate::tables::MaxRowsConfig; +use crate::tables::trace_builder::Traces; +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use executor::elf::Elf; +use stark::proof::view::MultiProofView; + +/// Approach 1's second pass must draw the very challenge the tables were built +/// against. +/// +/// This is the whole claim of the single-challenge design: the roots the Commit +/// phase produced, absorbed in AIR order on top of the bound statement, +/// reproduce the transcript the production prover builds. Anything that shifts +/// the order, omits a preprocessed table's precomputed root, or commits a table +/// differently moves `(z, alpha)`, and a challenge that differs from the +/// prover's is a proof that does not verify. +/// +/// The roots are checked in position first, so a mismatch names the table +/// rather than surfacing only as a different field element at the end. The +/// challenge itself is then compared against the *verifier's* replay, rebuilt +/// from the proof's own statement fields, so the comparison does not run +/// through `challenge_phase` twice. +#[test] +fn challenge_matches_the_ordinary_prover() { + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + // Small enough that the walk closes several chunks and leaves tails: the + // order only gets exercised when a group has more than one member. + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let vm_proof = crate::prove_with_options_and_inputs(&elf_bytes, &[], &proof_options, &max_rows) + .expect("ordinary prove"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + + assert_eq!( + challenge.roots.len(), + vm_proof.proof.proofs.len(), + "the Commit phase accounted for a different number of tables than the proof has" + ); + let mut preprocessed = 0usize; + for (idx, (got, want)) in challenge + .roots + .iter() + .zip(vm_proof.proof.proofs.iter()) + .enumerate() + { + assert_eq!( + got.main, want.lde_trace_main_merkle_root, + "table {idx}: committed under a different root than the proof carries" + ); + if got.precomputed.is_some() { + preprocessed += 1; + } + } + assert!( + preprocessed >= 4, + "the fixture must cover the preprocessed tables (BITWISE, DECODE, KECCAK_RC, \ + REGISTER and the pages); saw {preprocessed}" + ); + + // The verifier's path: statement from the proof, AIRs reconstructed the way + // verification reconstructs them. + let page_configs = Traces::page_configs_from_elf_and_runtime( + &elf, + &vm_proof.runtime_page_ranges, + vm_proof.num_private_input_pages, + vm_proof.proof.proofs.len(), + ) + .expect("page configs"); + let verifier_airs = crate::VmAirs::new( + &elf, + &proof_options, + false, + &page_configs, + &vm_proof.table_counts, + None, + true, + None, + None, + None, + ); + let mut transcript = DefaultTranscript::new(&[]); + crate::statement::absorb_statement( + &mut transcript, + crate::statement::StatementKind::Monolithic, + &elf_bytes, + &vm_proof.public_output, + &vm_proof.table_counts, + vm_proof.num_private_input_pages, + &vm_proof.runtime_page_ranges, + proof_options.fri_final_poly_log_degree, + ); + let (z, alpha) = crate::replay_transcript_phase_a_view( + &verifier_airs.air_refs(), + MultiProofView::Owned(&vm_proof.proof), + &mut transcript, + ); + + assert_eq!( + challenge.challenges, + vec![z, alpha], + "the Challenge phase sampled a different challenge from the same execution" + ); +} + +/// The LogUp pass must commit the auxiliary columns the proof carries. +/// +/// The pass rebuilds every table from scratch — the ones the Commit phase +/// dropped no longer exist — and builds their auxiliary columns against the +/// challenge that phase produced. Three separate things have to hold for the +/// root to land: the rebuild is byte-identical to what was committed, the +/// challenge is the prover's, and the table is in the slot the proof expects. +/// Any one of them failing changes the bus the verifier checks, so all three +/// are pinned here at once against a real proof. +#[test] +fn logup_matches_the_ordinary_prover() { + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + // Small enough that MEMW and MEMW_A chunks close mid-walk: their timestamp + // checks are LT rows, and a walk that retires a chunk must still hand LT + // the rows the chunk owes it. + let max_rows = MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + memw_aligned: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let vm_proof = crate::prove_with_options_and_inputs(&elf_bytes, &[], &proof_options, &max_rows) + .expect("ordinary prove"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + let logup = crate::logup_phase::run(&elf, &[], &max_rows, &proof_options, &challenge) + .expect("logup phase"); + + assert_eq!( + format!("{:?}", challenge.order.counts()), + format!("{:?}", vm_proof.table_counts), + "the pass laid the tables out differently from the ordinary prover" + ); + assert_eq!( + logup.tables.len(), + vm_proof.proof.proofs.len(), + "the pass accounted for a different number of tables than the proof has" + ); + for (idx, (got, want)) in logup + .tables + .iter() + .zip(vm_proof.proof.proofs.iter()) + .enumerate() + { + assert_eq!( + got.lde_trace_aux_merkle_root, want.lde_trace_aux_merkle_root, + "table {idx}: auxiliary trace committed under a different root" + ); + assert_eq!( + got.composition_poly_root, want.composition_poly_root, + "table {idx}: composition polynomial committed under a different root" + ); + assert_eq!( + got.fri_layers_merkle_roots, want.fri_layers_merkle_roots, + "table {idx}: a different FRI commitment" + ); + assert_eq!( + got.fri_final_poly_coeffs, want.fri_final_poly_coeffs, + "table {idx}: a different FRI final polynomial" + ); + } + + // The decisive one: the pass's own proof, verified. Everything above says + // it matches the ordinary prover piece by piece; this says the assembled + // whole is a proof. + let rebuilt = crate::logup_phase::assemble_vm_proof(logup, &challenge); + assert!( + crate::verify(&rebuilt, &elf_bytes).expect("verify"), + "the proof the pass assembled does not verify" + ); +} + +/// Many chunks of every kind, on a program that works memory. The +/// small program above has one chunk per kind, so it cannot tell per-chunk +/// accounting from whole-table accounting; ethrex could, and did not verify. +#[test] +fn a1_verifies_with_many_chunks() { + let elf_bytes = { + let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let path = root.parent().expect("workspace root").join(format!( + "executor/program_artifacts/rust/{}.elf", + std::env::var("A1_DIAG_ELF").unwrap_or_else(|_| "vector".into()) + )); + std::fs::read(&path).unwrap_or_else(|_| panic!("Failed to read ELF: {}", path.display())) + }; + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = MaxRowsConfig { + cpu: 1 << 11, + memw: 1 << 9, + memw_aligned: 1 << 9, + dvrm: 1 << 9, + mul: 1 << 9, + lt: 1 << 9, + shift: 1 << 9, + load: 1 << 9, + branch: 1 << 9, + memw_register: 1 << 9, + eq: 1 << 9, + bytewise: 1 << 9, + store: 1 << 9, + cpu32: 1 << 9, + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let vm_proof = crate::prove_with_options_and_inputs(&elf_bytes, &[], &proof_options, &max_rows) + .expect("ordinary prove"); + + let committed = crate::commit_phase::run_to_end(&elf, &[], &max_rows, &proof_options) + .expect("commit phase"); + let challenge = crate::challenge_phase::run(&committed, &elf, &elf_bytes, &proof_options) + .expect("challenge phase"); + drop(committed); + let logup = crate::logup_phase::run(&elf, &[], &max_rows, &proof_options, &challenge) + .expect("logup phase"); + let rebuilt = crate::logup_phase::assemble_vm_proof(logup, &challenge); + + eprintln!( + "tables: pass {} vs ordinary {}; counts pass {:?} vs ordinary {:?}", + rebuilt.proof.proofs.len(), + vm_proof.proof.proofs.len(), + rebuilt.table_counts, + vm_proof.table_counts + ); + let differing: Vec = rebuilt + .proof + .proofs + .iter() + .zip(vm_proof.proof.proofs.iter()) + .enumerate() + .filter(|(_, (a, b))| a.lde_trace_main_merkle_root != b.lde_trace_main_merkle_root) + .map(|(i, _)| i) + .collect(); + eprintln!("tables whose main root differs from the ordinary prover's: {differing:?}"); + assert!( + crate::verify(&rebuilt, &elf_bytes).expect("verify"), + "the proof the pass assembled does not verify" + ); +} + +/// Diagnostic: where A1's BITWISE multiplicities depart from the ordinary build's. +#[test] +#[allow(clippy::needless_range_loop)] +fn bitwise_multiplicities_match_the_ordinary_build() { + let elf_bytes = { + let root = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let name = std::env::var("A1_DIAG_ELF").unwrap_or_else(|_| "vector".into()); + std::fs::read( + root.parent() + .unwrap() + .join(format!("executor/program_artifacts/rust/{name}.elf")), + ) + .expect("ELF") + }; + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = if std::env::var("A1_DIAG_DEFAULT_ROWS").is_ok() { + MaxRowsConfig::default() + } else { + MaxRowsConfig { + cpu: 1 << 11, + memw: 1 << 9, + memw_aligned: 1 << 9, + dvrm: 1 << 9, + mul: 1 << 9, + lt: 1 << 9, + shift: 1 << 9, + load: 1 << 9, + branch: 1 << 9, + memw_register: 1 << 9, + eq: 1 << 9, + bytewise: 1 << 9, + store: 1 << 9, + cpu32: 1 << 9, + } + }; + let input = std::env::var("A1_DIAG_INPUT") + .ok() + .map(|path| std::fs::read(path).expect("private input")) + .unwrap_or_default(); + let ordinary = crate::commit_phase::build_resident(&elf, &input, &max_rows).expect("ordinary"); + let walked = crate::logup_phase::walk_only(&elf, &input, &max_rows).expect("walk"); + eprintln!( + "cpu rows: {}", + ordinary.cpus.iter().map(|t| t.num_rows()).sum::() + ); + let (a, b) = (&walked.bitwise, &ordinary.bitwise); + assert_eq!(a.num_rows(), b.num_rows()); + assert_eq!(a.num_cols(), b.num_cols()); + let mut shown = 0; + let mut per_col = vec![0usize; a.num_cols()]; + for row in 0..a.num_rows() { + for col in 0..a.num_cols() { + if a.get_main(row, col) != b.get_main(row, col) { + per_col[col] += 1; + if shown < 25 { + eprintln!( + "row {row} (x={:?} y={:?}) col {col}: walk {:?} vs ordinary {:?}", + a.get_main(row, 0).value(), + a.get_main(row, 1).value(), + a.get_main(row, col).value(), + b.get_main(row, col).value() + ); + shown += 1; + } + } + } + } + eprintln!("differing cells per column: {per_col:?}"); + assert!( + per_col.iter().all(|&n| n == 0), + "BITWISE multiplicities differ" + ); +} diff --git a/prover/src/tests/mod.rs b/prover/src/tests/mod.rs index 73ff6ee45..8deb9e601 100644 --- a/prover/src/tests/mod.rs +++ b/prover/src/tests/mod.rs @@ -1,6 +1,8 @@ #[cfg(all(test, feature = "disk-spill"))] pub mod auto_storage_tests; #[cfg(test)] +pub mod batched_fri_tests; +#[cfg(test)] pub mod bitwise_bus_tests; #[cfg(test)] pub mod bitwise_tests; @@ -11,6 +13,8 @@ pub mod branch_constraints_tests; #[cfg(test)] pub mod bytewise_tests; #[cfg(test)] +pub mod challenge_phase_tests; +#[cfg(test)] pub mod commit_tests; #[cfg(test)] pub mod compute_commit_bus_offset_tests; diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index d1fd2009e..335c491bf 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -1036,6 +1036,67 @@ fn test_dump_recursion_input() { // public output), computed here while the `ContinuationProof` bundle // still exists (`encode_continuation_guest_input` consumes it) — lets a // consumer check the pre-proved fixture without re-deriving it. + // `RECURSION_DUMP_BATCHED=1` proves the inner program with the + // prove-and-retire pipeline's batched path and dumps a `BatchedProof` + // blob for the `batched` guest, so the two proof layouts can be compared + // on guest cycles. + if std::env::var("RECURSION_DUMP_BATCHED").is_ok() { + let opts = preset.options(); + let elf = executor::elf::Elf::load(&inner_elf_bytes).expect("load inner ELF"); + let max_rows = crate::tables::MaxRowsConfig::default(); + eprintln!( + "[dump-input] proving inner batched (blowup={}, fri_queries={}) ...", + opts.blowup_factor, opts.fri_number_of_queries + ); + let committed = crate::commit_phase::run_to_end(&elf, &inner_input, &max_rows, &opts) + .expect("commit pass"); + let challenge = crate::challenge_phase::run(&committed, &elf, &inner_elf_bytes, &opts) + .expect("challenge pass"); + drop(committed); + let batched = + crate::logup_phase::run_batched(&elf, &inner_input, &max_rows, &opts, &challenge) + .expect("batched pass"); + let opened = crate::logup_phase::run_open( + &elf, + &inner_input, + &max_rows, + &opts, + &challenge, + &batched, + ) + .expect("open pass"); + let proof = crate::logup_phase::assemble_batched_proof(batched, opened) + .expect("assemble batched proof"); + assert!( + crate::batched_verifier::verify(&proof, &inner_elf_bytes, &opts) + .expect("batched verify errored"), + "batched proof must verify on host before dumping" + ); + let public_output = proof.public_output.clone(); + let (decode, pages) = + crate::recursion::precomputed_commitments(&inner_elf_bytes, &opts).expect("roots"); + let id = crate::recursion::program_id_from_elf(&inner_elf_bytes, &decode, &pages) + .expect("program id"); + let blob = crate::recursion::encode_batched_guest_input(proof, &inner_elf_bytes, &opts) + .expect("encode batched guest input"); + eprintln!("[dump-input] batched blob bytes: {}", blob.len()); + assert!( + blob.len() <= executor::vm::memory::MAX_PRIVATE_INPUT_SIZE as usize, + "batched recursion input exceeds MAX_PRIVATE_INPUT_SIZE" + ); + let path = "/tmp/recursion_input.bin"; + std::fs::write(path, &blob).expect("write blob"); + let mut sidecar = id.to_vec(); + sidecar.extend_from_slice(&public_output); + std::fs::write(format!("{path}.expected"), &sidecar).expect("write sidecar"); + eprintln!( + "[dump-input] preset={} inner={inner_label} wrote {} bytes to {path}", + preset.name(), + blob.len() + ); + return; + } + let (blob, expected_sidecar) = match std::env::var("RECURSION_DUMP_EPOCH_LOG2") { Ok(s) => { // No recursion-cont-blowup8.elf is built (RECURSION_CONT_PRESETS diff --git a/prover/src/tests/trace_builder_tests.rs b/prover/src/tests/trace_builder_tests.rs index 428fd4700..2a1b3d068 100644 --- a/prover/src/tests/trace_builder_tests.rs +++ b/prover/src/tests/trace_builder_tests.rs @@ -1099,3 +1099,1264 @@ fn test_local_to_global_traces_from_real_execution() { assert_eq!(trace.num_rows(), expected_rows); } } + +/// Two builds of the same logs must produce byte-identical traces. +/// +/// The dedup'd tables (LT, MUL, DVRM, BRANCH, EQ, BYTEWISE) collect their rows +/// out of a `HashMap`, whose iteration order std randomizes per instance — so +/// the rows were identical in content but arbitrary in order. Harmless while a +/// trace is built once, fatal for rebuilding a retired one: the rebuild has to +/// hash to the root the first build committed. +/// +/// Fails if any of the `sort_unstable_by` calls after those dedups is removed. +#[test] +fn trace_build_is_deterministic_across_builds() { + type TT = stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >; + + // Several DISTINCT ops per table, so each dedup'd `unique_ops` holds more + // than one element and its order can actually vary. + let mut logs = vec![ + make_slt_log(0x1000, 5, 10, 1), + make_slt_log(0x1004, 200, 7, 0), + make_slt_log(0x1008, 42, 42, 0), + make_slt_log(0x100c, 1, 999, 1), + make_blt_log(0x1010, 3, 4, true), + make_blt_log(0x1014, 50, 9, false), + make_blt_log(0x1018, 77, 77, false), + ]; + let mut instrs = vec![ + Instruction::Arith { + dst: 1, + src1: 2, + src2: 3, + op: ArithOp::SetLessThan, + }, + Instruction::Arith { + dst: 1, + src1: 2, + src2: 3, + op: ArithOp::SetLessThan, + }, + Instruction::Arith { + dst: 1, + src1: 2, + src2: 3, + op: ArithOp::SetLessThan, + }, + Instruction::Arith { + dst: 1, + src1: 2, + src2: 3, + op: ArithOp::SetLessThan, + }, + Instruction::Branch { + src1: 2, + src2: 3, + cond: Comparison::LessThan, + offset: 8, + }, + Instruction::Branch { + src1: 2, + src2: 3, + cond: Comparison::LessThan, + offset: 8, + }, + Instruction::Branch { + src1: 2, + src2: 3, + cond: Comparison::LessThan, + offset: 8, + }, + ]; + append_ecall(&mut logs, &mut instrs); + let instructions = make_instructions(&logs, &instrs); + let max_rows = Default::default(); + + let a = Traces::from_logs(&logs, instructions.clone(), &max_rows).unwrap(); + let b = Traces::from_logs(&logs, instructions, &max_rows).unwrap(); + + fn flat(t: &TT) -> Vec { + let (data, _cols) = t.main_data_row_major(); + data.iter().map(|fe| *fe.value()).collect() + } + fn eq_chunks(x: &[TT], y: &[TT], name: &str) { + assert_eq!( + x.len(), + y.len(), + "{name}: chunk count differs across builds" + ); + for (i, (s, m)) in x.iter().zip(y.iter()).enumerate() { + assert_eq!( + flat(s), + flat(m), + "{name} chunk {i}: trace data differs across builds (non-deterministic order)" + ); + } + } + eq_chunks(&a.lts, &b.lts, "LT"); + eq_chunks(&a.muls, &b.muls, "MUL"); + eq_chunks(&a.dvrms, &b.dvrms, "DVRM"); + eq_chunks(&a.branches, &b.branches, "BRANCH"); + eq_chunks(&a.eqs, &b.eqs, "EQ"); + eq_chunks(&a.bytewises, &b.bytewises, "BYTEWISE"); + eq_chunks(&a.cpus, &b.cpus, "CPU"); + eq_chunks(&a.memws, &b.memws, "MEMW"); + eq_chunks(&a.shifts, &b.shifts, "SHIFT"); + eq_chunks(&a.loads, &b.loads, "LOAD"); +} + +/// `build_chunk(kind, i)` must equal `build_table(kind)[i]`, byte for byte. +/// +/// This is the equality the streaming prover rests on: Round 1 commits the +/// table built one way, and the fused chain rebuilds the chunk it needs the +/// other way. If they ever diverge, the rebuilt trace hashes to a root the +/// verifier will not accept. +#[test] +fn build_chunk_matches_the_full_table_build() { + use crate::tables::trace_builder::{CollectedOps, TableKind}; + + // More ops than the chunk limit below, so several chunks exist and the + // last one is short. + let lt_ops: Vec<_> = (0..10u64) + .map(|i| crate::tables::lt::LtOperation::new(i, i * 7 + 1, false)) + .collect(); + let routed = CollectedOps { + lt_ops, + ..Default::default() + }; + + let max_rows = crate::tables::MaxRowsConfig { + lt: 4, + ..Default::default() + }; + + let whole = routed + .build_table( + TableKind::Lt, + &max_rows, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("full build"); + assert_eq!( + whole.len(), + routed.num_chunks(TableKind::Lt, &max_rows), + "num_chunks disagrees with what build_table produced" + ); + + for (i, expected) in whole.iter().enumerate() { + let one = routed.build_chunk(TableKind::Lt, i, &max_rows); + let (a, _) = expected.main_data_row_major(); + let (b, _) = one.main_data_row_major(); + assert_eq!( + a.iter().map(|fe| *fe.value()).collect::>(), + b.iter().map(|fe| *fe.value()).collect::>(), + "chunk {i}: on-demand build differs from the full build" + ); + } +} + +/// `chunk_shape` must agree with the chunk it declines to build, for every kind. +/// +/// It reads the shape off op counts and a constant width instead of generating +/// a trace, which is only valid while every generator pads to +/// `count.next_power_of_two().max(4)`. This is the test that fails if one of +/// them ever stops. +#[test] +fn chunk_shape_matches_the_built_chunk() { + use crate::tables::trace_builder::{CollectedOps, TableKind}; + + // Ops with deliberate repeats, so the deduplicating kinds and the plain ones + // disagree on count and the distinction is actually exercised. + // 8 ops, 3 distinct: the deduplicating rule pads to 4 rows where counting + // them raw would pad to 8. Without that gap the test would pass even if + // `chunk_shape` ignored deduplication entirely. + let lt_ops: Vec<_> = (0..8u64) + .map(|i| crate::tables::lt::LtOperation::new(i % 3, i % 3 + 1, false)) + .collect(); + // SHIFT is a plain kind (no dedup): 20 ops over a limit of 8 give chunks of + // 8/8/4, i.e. row counts of 8/8/4. Sizes above 4 matter — an empty op list + // pads to 4, so a fixture whose chunks all land on 4 compares the padding + // floor against itself and would pass even if the row rule were wrong. + let shift_ops: Vec<_> = (0..20u64) + .map(|i| crate::tables::shift::ShiftOperation::new(i, i % 5, false, false, false)) + .collect(); + // MUL deduplicates like LT, and its rows are keyed on the op with the lo/hi + // flag folded into the multiplicity: 12 entries, 3 distinct. + let mul_ops: Vec<_> = (0..6u64) + .flat_map(|i| { + let op = crate::tables::mul::MulOperation::new(i % 3, false, i % 3 + 1, false); + [(op.clone(), false), (op, true)] + }) + .collect(); + let routed = CollectedOps { + lt_ops, + shift_ops, + mul_ops, + ..Default::default() + }; + + let max_rows = crate::tables::MaxRowsConfig { + lt: 16, + shift: 8, + mul: 16, + ..Default::default() + }; + + assert_eq!( + routed.chunk_shape(TableKind::Lt, 0, &max_rows).0, + 4, + "the LT fixture must exercise deduplication (8 ops, 3 distinct)" + ); + assert_eq!( + routed.chunk_shape(TableKind::Mul, 0, &max_rows).0, + 4, + "the MUL fixture must exercise deduplication (12 entries, 3 distinct)" + ); + assert_eq!( + routed.num_chunks(TableKind::Shift, &max_rows), + 3, + "the SHIFT fixture must split into several chunks, or the plain path is \ + only ever checked on one" + ); + assert_eq!( + routed.chunk_shape(TableKind::Shift, 0, &max_rows).0, + 8, + "the SHIFT fixture must produce chunks wider than the 4-row padding floor" + ); + + let mut populated = 0usize; + for kind in [ + TableKind::Cpu, + TableKind::Memw, + TableKind::MemwAligned, + TableKind::MemwRegister, + TableKind::Load, + TableKind::Lt, + TableKind::Shift, + TableKind::Mul, + TableKind::Dvrm, + TableKind::Branch, + TableKind::Eq, + TableKind::Bytewise, + TableKind::Store, + TableKind::Cpu32, + ] { + for chunk in 0..routed.num_chunks(kind, &max_rows) { + let built = routed.build_chunk(kind, chunk, &max_rows); + assert_eq!( + routed.chunk_shape(kind, chunk, &max_rows), + (built.num_rows(), built.num_main_columns), + "{kind:?} chunk {chunk}: declared shape differs from the built one" + ); + if routed.buffered(kind) > 0 { + populated += 1; + } + } + } + // A kind with no ops pads to the same 4 rows on both sides, so it pins the + // column width and nothing else. Without at least a few populated kinds this + // whole loop is a constant compared against itself. + assert!( + populated >= 3, + "the fixture must give several kinds real ops, or the row half of the \ + comparison is vacuous (populated chunks: {populated})" + ); +} + +/// Collecting an execution chunk by chunk must produce exactly what collecting +/// it all at once produces. +/// +/// This is what lets the prover walk an execution instead of starting from a +/// materialized log of the whole thing. It holds because phases 1-3 are +/// segment-local once `MemoryState` and `RegisterState` are carried: the LT ops +/// a memory access implies come from the timestamps that access already +/// carries, not from an ordering over the whole run. +/// +/// Compared through the built traces rather than the op lists, since that is +/// what the commitment is taken over. +#[test] +fn collect_streaming_matches_collect_epoch() { + use crate::tables::register::register_init_from_entry_point; + use crate::tables::trace_builder::{DecodeArtifacts, Traces as T, build_initial_image}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let artifacts = DecodeArtifacts::from_elf(&elf).expect("decode artifacts"); + let image = build_initial_image(&elf, &[]); + let register_init = register_init_from_entry_point(elf.entry_point); + let max_rows = crate::tables::MaxRowsConfig::default(); + + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let at_once = + T::collect_epoch(&artifacts, &image, ®ister_init, &logs, true).expect("collect at once"); + let streamed = + T::collect_epoch_streaming(&artifacts, &elf, vec![], &image, ®ister_init, true) + .expect("collect streaming"); + + let build = |collected| { + T::build_from_collected( + &artifacts, + collected, + Some(&image), + ®ister_init, + &max_rows, + &[], + true, + false, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("build") + }; + let a = build(at_once); + let b = build(streamed); + + let flat = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| { + let (data, _) = t.main_data_row_major(); + data.iter().map(|fe| *fe.value()).collect::>() + }; + let same = |x: &[_], y: &[_], name: &str| { + assert_eq!(x.len(), y.len(), "{name}: chunk count differs"); + for (i, (p, q)) in x.iter().zip(y.iter()).enumerate() { + assert_eq!(flat(p), flat(q), "{name} chunk {i} differs"); + } + }; + same(&a.cpus, &b.cpus, "CPU"); + same(&a.memws, &b.memws, "MEMW"); + same(&a.lts, &b.lts, "LT"); + same(&a.loads, &b.loads, "LOAD"); + same(&a.shifts, &b.shifts, "SHIFT"); + same(&a.branches, &b.branches, "BRANCH"); + same(&a.memw_registers, &b.memw_registers, "MEMW_R"); + assert_eq!(flat(&a.bitwise), flat(&b.bitwise), "BITWISE differs"); + assert_eq!(flat(&a.register), flat(&b.register), "REGISTER differs"); +} + +/// The Commit-phase walk must hand out exactly the chunks the all-at-once build +/// produces, in the same order. +/// +/// It is the same equality `build_chunk` rests on, one level up: a table closed +/// mid-execution and a table built from the finished op list have to be the +/// same table, or committing early means committing something else. +fn assert_walk_matches(fixture: &str, max_rows: crate::tables::MaxRowsConfig, min_split: usize) { + use crate::tables::register::register_init_from_entry_point; + use crate::tables::trace_builder::{ + DecodeArtifacts, TableKind, Traces as T, build_initial_image, + }; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf_bytes = crate::test_utils::asm_elf_bytes(fixture); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let artifacts = DecodeArtifacts::from_elf(&elf).expect("decode artifacts"); + let image = build_initial_image(&elf, &[]); + let register_init = register_init_from_entry_point(elf.entry_point); + // Small enough that the walk closes several chunks before the run ends, + // which is the case that matters — a single tail chunk would prove nothing. + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let collected = + T::collect_epoch(&artifacts, &image, ®ister_init, &logs, true).expect("collect"); + let expected = T::build_from_collected( + &artifacts, + collected, + Some(&image), + ®ister_init, + &max_rows, + &[], + true, + false, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("build"); + + let mut seen: Vec<(TableKind, usize, Vec)> = Vec::new(); + T::walk_and_emit_chunks( + &artifacts, + &elf, + vec![], + &image, + ®ister_init, + &max_rows, + |kind, chunk, table| { + let (data, _) = table.main_data_row_major(); + seen.push((kind, chunk, data.iter().map(|fe| *fe.value()).collect())); + }, + ) + .expect("walk"); + + let flat = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| { + let (data, _) = t.main_data_row_major(); + data.iter().map(|fe| *fe.value()).collect::>() + }; + let check = |kind: TableKind, + want: &[stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >]| { + let got: Vec<_> = seen.iter().filter(|(k, _, _)| *k == kind).collect(); + // The walk emits only the chunks that filled up during the run; the + // partial tail waits for the end-of-run finalization, which still + // appends to these lists. So the emitted chunks are a prefix. + assert_eq!( + got.len(), + want.len().saturating_sub(1), + "{kind:?}: the walk should emit every chunk but the tail" + ); + for (i, (_, chunk, data)) in got.iter().enumerate() { + assert_eq!(*chunk, i, "{kind:?}: chunks arrived out of order"); + assert_eq!(*data, flat(&want[i]), "{kind:?} chunk {i} differs"); + } + }; + // Only the tables this fixture actually splits are worth asserting on: a + // table with one chunk compares an empty prefix and proves nothing. + let chunked: Vec<&str> = [ + ("CPU", expected.cpus.len()), + ("MEMW", expected.memws.len()), + ("MEMW_A", expected.memw_aligneds.len()), + ("MEMW_R", expected.memw_registers.len()), + ("LOAD", expected.loads.len()), + ("BRANCH", expected.branches.len()), + ("EQ", expected.eqs.len()), + ("BYTEWISE", expected.bytewises.len()), + ("STORE", expected.stores.len()), + ] + .into_iter() + .filter(|(_, n)| *n > 1) + .map(|(name, _)| name) + .collect(); + assert!( + chunked.len() >= min_split, + "{fixture} must split at least {min_split} tables mid-walk, split: {chunked:?}" + ); + + check(TableKind::Cpu, &expected.cpus); + check(TableKind::Memw, &expected.memws); + check(TableKind::MemwAligned, &expected.memw_aligneds); + check(TableKind::MemwRegister, &expected.memw_registers); + check(TableKind::Load, &expected.loads); + check(TableKind::Cpu32, &expected.cpu32s); + check(TableKind::Branch, &expected.branches); + check(TableKind::Eq, &expected.eqs); + check(TableKind::Bytewise, &expected.bytewises); + check(TableKind::Store, &expected.stores); +} + +#[test] +fn commit_walk_emits_the_same_chunks() { + // Limits small enough that several tables close chunks mid-walk. + assert_walk_matches( + "fib_iterative_160k", + crate::tables::MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + eq: 1 << 12, + bytewise: 1 << 12, + store: 1 << 12, + ..Default::default() + }, + 3, + ); +} + +/// The same, on a program that uses the word instructions. +/// +/// `cpu32_chip_op` appends to SHIFT, MUL and DVRM for every `*W` op, so those +/// tables are not final when a segment ends. A fibonacci fixture has no word +/// instructions and would let a table that is closed too early pass unnoticed; +/// this one would not. +#[test] +fn commit_walk_emits_the_same_chunks_with_word_instructions() { + assert_walk_matches("basic_arith_32", crate::tables::MaxRowsConfig::small(), 1); +} + +/// A chunk committed during the walk must carry the root the normal prover +/// gives that same chunk. +/// +/// This is what makes Approach 1's Commit phase legitimate rather than merely +/// convenient: the phase closes a table before the tables after it exist and +/// puts its commitment in the transcript then and there. If that commitment +/// differed from the one the all-at-once prover would produce, everything +/// downstream — challenges, openings, the verifier — would be reading a +/// different table. +#[test] +fn chunks_committed_during_the_walk_carry_the_normal_roots() { + use crate::tables::register::register_init_from_entry_point; + use crate::tables::trace_builder::{ + DecodeArtifacts, TableKind, Traces as T, build_initial_image, + }; + use executor::elf::Elf; + use stark::prover::IsStarkProver; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let artifacts = DecodeArtifacts::from_elf(&elf).expect("decode artifacts"); + let image = build_initial_image(&elf, &[]); + let register_init = register_init_from_entry_point(elf.entry_point); + let max_rows = crate::tables::MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 15, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let traces = T::from_elf_and_logs( + &elf, + &executor::vm::execution::Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs, + &max_rows, + &[], + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); + let counts = traces.table_counts(); + let airs = crate::VmAirs::new( + &elf, + &proof_options, + false, + &traces.page_configs, + &counts, + None, + true, + None, + None, + None, + ); + + // The AIR each emitted chunk belongs to, by kind and position. + let air_for = |kind: TableKind, chunk: usize| match kind { + TableKind::Cpu => airs.cpus.get(chunk).map(|a| a.as_ref()), + TableKind::Memw => airs.memws.get(chunk).map(|a| a.as_ref()), + TableKind::MemwAligned => airs.memw_aligneds.get(chunk).map(|a| a.as_ref()), + TableKind::MemwRegister => airs.memw_registers.get(chunk).map(|a| a.as_ref()), + TableKind::Load => airs.loads.get(chunk).map(|a| a.as_ref()), + TableKind::Cpu32 => airs.cpu32s.get(chunk).map(|a| a.as_ref()), + TableKind::Branch => airs.branches.get(chunk).map(|a| a.as_ref()), + TableKind::Eq => airs.eqs.get(chunk).map(|a| a.as_ref()), + TableKind::Bytewise => airs.bytewises.get(chunk).map(|a| a.as_ref()), + TableKind::Store => airs.stores.get(chunk).map(|a| a.as_ref()), + _ => None, + }; + + type P = stark::prover::Prover< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + (), + >; + let commit_root = |air: &dyn stark::traits::AIR< + Field = crate::tables::types::GoldilocksField, + FieldExtension = crate::tables::types::GoldilocksExtension, + PublicInputs = (), + >, + t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| {

>::commit_table_root(air, t) }; + + let mut checked = 0usize; + T::walk_and_emit_chunks( + &artifacts, + &elf, + vec![], + &image, + ®ister_init, + &max_rows, + |kind, chunk, table| { + let Some(air) = air_for(kind, chunk) else { + return; + }; + let walked = commit_root(air, &table).expect("the walk's chunk commits"); + let resident = match kind { + TableKind::Cpu => &traces.cpus[chunk], + TableKind::Memw => &traces.memws[chunk], + TableKind::MemwAligned => &traces.memw_aligneds[chunk], + TableKind::MemwRegister => &traces.memw_registers[chunk], + TableKind::Load => &traces.loads[chunk], + TableKind::Cpu32 => &traces.cpu32s[chunk], + TableKind::Branch => &traces.branches[chunk], + TableKind::Eq => &traces.eqs[chunk], + TableKind::Bytewise => &traces.bytewises[chunk], + TableKind::Store => &traces.stores[chunk], + _ => unreachable!(), + }; + let expected = commit_root(air, resident).expect("the resident chunk commits"); + assert_eq!( + walked, expected, + "{kind:?} chunk {chunk}: committed during the walk under a different root" + ); + checked += 1; + }, + ) + .expect("walk"); + + assert!( + checked > 0, + "the fixture must close at least one chunk mid-walk" + ); +} + +/// No table CPU32 feeds may be closed early by the Commit-phase walk. +/// +/// `cpu32_chip_op` appends to SHIFT, MUL and DVRM once per word instruction, so +/// those are not final when a segment ends — closing one early would cut its +/// chunks somewhere the finished run does not. +/// +/// Stated as an invariant rather than left to a fixture: catching it by data +/// needs a program with word instructions AND enough of the affected ops to +/// split a chunk, and a fixture that stops meeting that quietly stops testing +/// it. SHIFT was in fact closed early until this was noticed. +#[test] +fn cpu32_appends_are_excluded_from_early_closing() { + use crate::tables::trace_builder::{CHUNKED_KINDS, CPU32_APPENDS_TO}; + + for kind in CPU32_APPENDS_TO { + assert!( + !CHUNKED_KINDS.contains(&kind), + "{kind:?} takes ops from cpu32_chip_op after a segment ends, so the walk \ + must not close it early" + ); + } +} + +/// The Commit phase must commit every chunk it closes under the root the +/// ordinary prover gives that chunk, and hand back the rest of the run. +/// +/// This is the pass end to end: it walks the execution, commits and drops each +/// table as it fills, and returns what it could not close. Two things have to +/// hold for that to be a prover and not just a producer — the commitments have +/// to be the right ones, and nothing may fall between the chunks it closed and +/// the tail it kept. +#[test] +fn the_commit_phase_commits_what_it_closes_and_keeps_the_rest() { + use crate::tables::trace_builder::{TableKind, Traces as T}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = crate::tables::MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let phase = + crate::commit_phase::run(&elf, &[], &max_rows, &proof_options).expect("commit phase"); + assert!( + !phase.closed.is_empty(), + "the fixture must close at least one chunk mid-walk" + ); + + // Every commitment must be the one the resident chunk carries. + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let resident = T::from_elf_and_logs( + &elf, + &logs, + &max_rows, + &[], + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); + let counts = resident.table_counts(); + let airs = crate::VmAirs::new( + &elf, + &proof_options, + false, + &resident.page_configs, + &counts, + None, + true, + None, + None, + None, + ); + type P = stark::prover::Prover< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + (), + >; + use stark::prover::IsStarkProver; + for (kind, chunk, root) in &phase.closed { + let (air, table) = match kind { + TableKind::Cpu => (airs.cpus[*chunk].as_ref(), &resident.cpus[*chunk]), + TableKind::Memw => (airs.memws[*chunk].as_ref(), &resident.memws[*chunk]), + TableKind::MemwAligned => ( + airs.memw_aligneds[*chunk].as_ref(), + &resident.memw_aligneds[*chunk], + ), + TableKind::MemwRegister => ( + airs.memw_registers[*chunk].as_ref(), + &resident.memw_registers[*chunk], + ), + TableKind::Load => (airs.loads[*chunk].as_ref(), &resident.loads[*chunk]), + TableKind::Cpu32 => (airs.cpu32s[*chunk].as_ref(), &resident.cpu32s[*chunk]), + TableKind::Branch => (airs.branches[*chunk].as_ref(), &resident.branches[*chunk]), + TableKind::Eq => (airs.eqs[*chunk].as_ref(), &resident.eqs[*chunk]), + TableKind::Bytewise => (airs.bytewises[*chunk].as_ref(), &resident.bytewises[*chunk]), + TableKind::Store => (airs.stores[*chunk].as_ref(), &resident.stores[*chunk]), + other => unreachable!("{other:?} is not closed mid-walk"), + }; + let expected = +

>::commit_table_root(air, table).expect("resident commits"); + assert_eq!( + *root, expected, + "{kind:?} chunk {chunk}: the Commit phase used a different root" + ); + } + + // And nothing falls between what it closed and what it kept: the chunks it + // closed plus the tail it kept are the chunks the resident build produced. + // Exact for CPU, which is one op per executed cycle and takes nothing from + // the end-of-run finalization; a bound elsewhere, since that finalization + // appends after the last cycle and can spill the tail into another chunk. + assert_eq!( + phase.walked.leftover.cycles(), + logs.len(), + "the walk executed a different number of cycles than the straight run" + ); + let closed_of = |kind: TableKind| phase.closed.iter().filter(|(k, _, _)| *k == kind).count(); + assert_eq!( + closed_of(TableKind::Cpu) + 1, + resident.cpus.len(), + "CPU: the closed chunks plus the tail are not the run's chunks" + ); + for (kind, produced) in [ + (TableKind::Memw, resident.memws.len()), + (TableKind::MemwAligned, resident.memw_aligneds.len()), + (TableKind::MemwRegister, resident.memw_registers.len()), + (TableKind::Load, resident.loads.len()), + (TableKind::Cpu32, resident.cpu32s.len()), + (TableKind::Branch, resident.branches.len()), + (TableKind::Eq, resident.eqs.len()), + (TableKind::Bytewise, resident.bytewises.len()), + (TableKind::Store, resident.stores.len()), + ] { + assert_eq!( + phase.walked.leftover.emitted(kind), + closed_of(kind), + "{kind:?}: the leftover disagrees with what was committed" + ); + if produced == 0 { + // #977: a kind the run never used has no table at all, so there is + // no tail to leave — and nothing may have been closed for it. + assert_eq!( + closed_of(kind), + 0, + "{kind:?}: the ordinary build has no table, but chunks were closed" + ); + } else { + assert!( + closed_of(kind) < produced, + "{kind:?}: closed {} of the run's {produced} chunks, leaving no tail", + closed_of(kind) + ); + } + } +} + +/// Commit plus Challenge must cover every chunked table, each under the root +/// the ordinary prover gives it. +/// +/// Together the two passes are supposed to account for the chunked side of the +/// proof with nothing missing and nothing committed twice: the chunks closed +/// mid-walk, the tails padded at the end, and the tables the walk could not +/// close at all. Checked against a real proof's roots, in position, so a table +/// committed under the wrong root or in the wrong slot fails here. +#[test] +fn the_two_phases_cover_every_chunked_table() { + use crate::tables::trace_builder::{TableKind, Traces as T}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + use std::collections::HashMap; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let max_rows = crate::tables::MaxRowsConfig { + cpu: 1 << 15, + memw: 1 << 10, + load: 1 << 10, + branch: 1 << 12, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let phase = crate::commit_phase::run(&elf, &[], &max_rows, &proof_options).expect("commit"); + let closed = phase.closed.clone(); + let (rest, _) = + crate::commit_phase::commit_remaining(phase.walked, &[], &max_rows, &proof_options) + .expect("challenge"); + + let mut got: HashMap<(TableKind, usize), _> = HashMap::new(); + for (kind, chunk, root) in closed.into_iter().chain(rest) { + assert!( + got.insert((kind, chunk), root).is_none(), + "{kind:?} chunk {chunk} was committed twice" + ); + } + + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let resident = T::from_elf_and_logs( + &elf, + &logs, + &max_rows, + &[], + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); + let counts = resident.table_counts(); + let airs = crate::VmAirs::new( + &elf, + &proof_options, + false, + &resident.page_configs, + &counts, + None, + true, + None, + None, + None, + ); + type P = stark::prover::Prover< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + (), + >; + use stark::prover::IsStarkProver; + + let groups: [(TableKind, &Vec<_>, &Vec<_>); 4] = [ + (TableKind::Cpu, &airs.cpus, &resident.cpus), + (TableKind::Branch, &airs.branches, &resident.branches), + (TableKind::Lt, &airs.lts, &resident.lts), + (TableKind::Memw, &airs.memws, &resident.memws), + ]; + let mut checked = 0usize; + for (kind, kind_airs, kind_traces) in groups { + assert_eq!( + kind_airs.len(), + kind_traces.len(), + "{kind:?}: AIR and trace counts disagree" + ); + for (chunk, (air, table)) in kind_airs.iter().zip(kind_traces.iter()).enumerate() { + let expected =

>::commit_table_root(air.as_ref(), table) + .expect("resident commits"); + let actual = got + .get(&(kind, chunk)) + .unwrap_or_else(|| panic!("{kind:?} chunk {chunk} was never committed")); + assert_eq!( + *actual, expected, + "{kind:?} chunk {chunk}: committed under a different root" + ); + checked += 1; + } + } + assert!(checked > 4, "the fixture must cover several chunks"); +} + +/// A retired chunk must leave its BITWISE lookups behind. +/// +/// BITWISE counts lookups from tables the Commit phase closes and drops, so the +/// contribution has to be taken while the chunk still exists. If it is not, the +/// table comes out short and the bus stops balancing — a failure that surfaces +/// at verification, far from the chunk that caused it. +/// +/// The limits here are small for the kinds that owe BITWISE, so most of their +/// lookups belong to chunks that were closed and dropped rather than to the +/// tail. Removing the fold that runs before a chunk is drained fails this. +#[test] +fn retiring_a_chunk_keeps_its_bitwise_lookups() { + use crate::tables::register::register_init_from_entry_point; + use crate::tables::trace_builder::{ + DecodeArtifacts, TableKind, Traces as T, WalkLeftover, build_initial_image, + }; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let artifacts = DecodeArtifacts::from_elf(&elf).expect("decode artifacts"); + let image = build_initial_image(&elf, &[]); + let register_init = register_init_from_entry_point(elf.entry_point); + let max_rows = crate::tables::MaxRowsConfig { + memw_aligned: 1 << 10, + memw_register: 1 << 10, + branch: 1 << 10, + eq: 1 << 10, + bytewise: 1 << 10, + store: 1 << 10, + ..Default::default() + }; + + let mut retired = 0usize; + let mut leftover = T::walk_and_emit_chunks( + &artifacts, + &elf, + vec![], + &image, + ®ister_init, + &max_rows, + |kind, _, _| { + if matches!( + kind, + TableKind::MemwAligned + | TableKind::MemwRegister + | TableKind::Branch + | TableKind::Eq + | TableKind::Bytewise + | TableKind::Store + ) { + retired += 1; + } + }, + ) + .expect("walk"); + assert!( + retired > 1, + "the fixture must retire chunks that owe BITWISE, or this proves nothing" + ); + leftover.finalize(&max_rows); + + let mut hist = leftover.bitwise_histogram(); + leftover.build_pages(&image, &[], &mut hist); + let built = WalkLeftover::build_bitwise_from(&hist); + + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let resident = T::from_elf_and_logs( + &elf, + &logs, + &max_rows, + &[], + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); + + let flat = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| { + let (data, _) = t.main_data_row_major(); + data.iter().map(|fe| *fe.value()).collect::>() + }; + assert_eq!( + flat(&built), + flat(&resident.bitwise), + "the lookups of the retired chunks are missing from BITWISE" + ); +} + +/// The tables built from accumulated op lists must match the ordinary build. +/// +/// COMMIT, KECCAK and its round tables, and the accelerator tables are written +/// once at the end from everything the run produced. The Commit phase drops the +/// chunked tables as it goes but has to keep feeding these, so the risk is the +/// opposite one: an op list quietly not accumulated comes out as an empty table +/// that still looks well-formed. +#[test] +fn the_accumulated_tables_match_the_ordinary_build() { + use crate::tables::register::register_init_from_entry_point; + use crate::tables::trace_builder::{DecodeArtifacts, Traces as T, build_initial_image}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + // Uses keccak and the commit ecall, so the tables under test are not empty. + let elf_bytes = crate::test_utils::asm_elf_bytes("test_keccak"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let artifacts = DecodeArtifacts::from_elf(&elf).expect("decode artifacts"); + let image = build_initial_image(&elf, &[]); + let register_init = register_init_from_entry_point(elf.entry_point); + let max_rows = crate::tables::MaxRowsConfig::default(); + + let mut leftover = T::walk_and_emit_chunks( + &artifacts, + &elf, + vec![], + &image, + ®ister_init, + &max_rows, + |_, _, _| {}, + ) + .expect("walk"); + leftover.finalize(&max_rows); + let built = leftover.build_accumulated(); + + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let resident = T::from_elf_and_logs( + &elf, + &logs, + &max_rows, + &[], + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); + + let flat = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| { + let (data, _) = t.main_data_row_major(); + data.iter().map(|fe| *fe.value()).collect::>() + }; + assert_eq!( + flat(&built.keccak_rc), + flat(&resident.keccak_rc), + "KECCAK_RC differs from the ordinary build" + ); + // The accelerators are `Vec` on the ordinary side since #977: a kind the run + // never called has no table there, and `present` is the accumulated side's + // answer to the same question. The table itself cannot be asked — it is + // padded, so a kind the run never called still has rows. + for (slot, (name, a, b)) in [ + ("COMMIT", &built.commit, resident.commits.first()), + ("KECCAK", &built.keccak, resident.keccaks.first()), + ( + "KECCAK_RND", + &built.keccak_rnd, + resident.keccak_rnds.first(), + ), + ("ECSM", &built.ecsm, resident.ecsms.first()), + ("ECDAS", &built.ecdas, resident.ecdases.first()), + ("HINT", &built.hint, resident.hints.first()), + ] + .into_iter() + .enumerate() + { + assert_eq!( + built.present[slot], + b.is_some(), + "{name}: the two builds disagree on whether the run called it" + ); + if let Some(b) = b { + assert_eq!(flat(a), flat(b), "{name} differs from the ordinary build"); + } + } + assert!( + flat(&built.keccak).iter().any(|v| *v != 0), + "the fixture must exercise KECCAK, or this proves nothing" + ); +} + +/// DECODE's multiplicities must survive the chunks being dropped. +/// +/// Every executed cycle looks DECODE up at its pc, and every padding row looks +/// it up at the padding pc. The Commit phase drops the CPU ops that carry those +/// pcs, so the lookups have to be counted while they still exist — by pc, since +/// listing them costs one entry per cycle, which is the thing being avoided. +#[test] +fn decode_multiplicities_survive_retiring_the_cpu_chunks() { + use crate::tables::register::register_init_from_entry_point; + use crate::tables::trace_builder::{DecodeArtifacts, Traces as T, build_initial_image}; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + let artifacts = DecodeArtifacts::from_elf(&elf).expect("decode artifacts"); + let image = build_initial_image(&elf, &[]); + let register_init = register_init_from_entry_point(elf.entry_point); + // Several CPU chunks, so most of the lookups belong to chunks that were + // closed and dropped rather than to the tail. Deliberately NOT a power of + // two: a full chunk then pads, and the padding lookups are a term that a + // power-of-two limit would leave at zero and therefore untested. + let max_rows = crate::tables::MaxRowsConfig { + cpu: 10_000, + ..Default::default() + }; + + let mut closed_cpu = 0usize; + let mut leftover = T::walk_and_emit_chunks( + &artifacts, + &elf, + vec![], + &image, + ®ister_init, + &max_rows, + |kind, _, _| { + if kind == crate::tables::trace_builder::TableKind::Cpu { + closed_cpu += 1; + } + }, + ) + .expect("walk"); + // DECODE is built in the end-of-run phase, after finalization freezes the + // padding count; building it before would read a tail that is still growing. + leftover.finalize(&max_rows); + assert!( + closed_cpu > 1, + "the fixture must drop several CPU chunks, or the counting is untested" + ); + let built = leftover.build_decode( + artifacts.decode_trace.clone(), + &artifacts.decode_pc_to_row, + &max_rows, + ); + + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let resident = T::from_elf_and_logs( + &elf, + &logs, + &max_rows, + &[], + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); + + let flat = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| { + let (data, _) = t.main_data_row_major(); + data.iter().map(|fe| *fe.value()).collect::>() + }; + assert_eq!( + flat(&built), + flat(&resident.decode), + "DECODE's multiplicities differ from the ordinary build" + ); +} + +/// The end-of-run phase must produce every non-chunked table the ordinary build +/// produces, identically. +/// +/// These are the tables that cannot be closed while the run continues: HALT +/// comes from the terminating ECALL, REGISTER's final PC token has to match the +/// last padding write, PAGE reads the memory image at the last cycle, and +/// BITWISE owes lookups that include PAGE's. Each depends on state the walk had +/// to carry rather than on an op list it could keep. +#[test] +fn the_end_of_run_tables_match_the_ordinary_build() { + use crate::tables::trace_builder::Traces as T; + use executor::elf::Elf; + use executor::vm::execution::Executor; + + let elf_bytes = crate::test_utils::asm_elf_bytes("fib_iterative_160k"); + let elf = Elf::load(&elf_bytes).expect("ELF load"); + // Not a power of two, so the CPU chunks pad and REGISTER's final PC token + // depends on a padding count the walk had to accumulate. + let max_rows = crate::tables::MaxRowsConfig { + cpu: 10_000, + ..Default::default() + }; + let proof_options = stark::proof::options::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup 2 is valid"); + + let phase = crate::commit_phase::run(&elf, &[], &max_rows, &proof_options).expect("commit"); + let (_, rest) = + crate::commit_phase::commit_remaining(phase.walked, &[], &max_rows, &proof_options) + .expect("challenge"); + + let logs = Executor::new(&elf, vec![]) + .expect("executor") + .run() + .expect("run") + .logs; + let resident = T::from_elf_and_logs( + &elf, + &logs, + &max_rows, + &[], + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); + + let flat = |t: &stark::trace::TraceTable< + crate::tables::types::GoldilocksField, + crate::tables::types::GoldilocksExtension, + >| { + let (data, _) = t.main_data_row_major(); + data.iter().map(|fe| *fe.value()).collect::>() + }; + assert_eq!(flat(&rest.halt), flat(&resident.halt), "HALT differs"); + { + let a = flat(&rest.register); + let b = flat(&resident.register); + let first = a.iter().zip(b.iter()).position(|(x, y)| x != y); + eprintln!( + "REGISTER: len {} vs {}, first diff at {:?} -> {:?} vs {:?}", + a.len(), + b.len(), + first, + first.map(|i| a[i]), + first.map(|i| b[i]) + ); + } + assert_eq!( + flat(&rest.register), + flat(&resident.register), + "REGISTER differs" + ); + assert_eq!(flat(&rest.decode), flat(&resident.decode), "DECODE differs"); + assert_eq!( + flat(&rest.bitwise), + flat(&resident.bitwise), + "BITWISE differs" + ); + assert_eq!( + rest.pages.len(), + resident.pages.len(), + "a different number of PAGE tables" + ); + assert!( + !rest.pages.is_empty(), + "the fixture must produce PAGE tables" + ); + for (i, (a, b)) in rest.pages.iter().zip(resident.pages.iter()).enumerate() { + assert_eq!(flat(a), flat(b), "PAGE {i} differs"); + } +}