diff --git a/Cargo.lock b/Cargo.lock index 880e57af5..68ea86e2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,6 +817,7 @@ dependencies = [ "rkyv", "serde", "serde_json", + "sha2", "stark", "sysinfo", "tikv-jemalloc-ctl", diff --git a/Makefile b/Makefile index cf794e081..1d5fc1988 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-s clean-recursion-elfs clean test test-asm \ test-rust test-ethrex test-ethrex-offline test-executor test-syscalls test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \ test-profile-recursion-block recursion-profile-block-input \ -test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-cuda-integration test-cuda-d1 test-cuda-fallback \ +test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-rpx-host-kat test-cuda-integration test-cuda-d1 test-cuda-fallback \ test-prover-cuda test-prover-comprehensive-cuda \ bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \ update-ethrex-fixture-checksums check-ethrex-fixture-checksums ethrex-real-block-fixture \ @@ -545,6 +545,20 @@ test-ethrex-crypto: test: compile-programs test-syscalls test-ethrex-crypto cargo test + # The hash counters compile to nothing unless the feature is on, so their + # own tests only execute here. See the `lint` target for why an instrument + # nobody runs is worth a line in the build. + cargo test -p crypto --features hash-metrics + # The transcript counters answer "which sponge ran". Their own integration + # binary, because the counters are process-global and a parallel neighbour's + # reset lands inside another test's measurement window — moving them out of + # the lib binary left four of five failing until they also took a lock. + cargo test -p crypto --features hash-metrics --test transcript_counters + # And the system test that reads them through a real prove: it is the one + # that says the PROVER picked the configuration's sponge, which the + # type-level test next to it cannot observe. + cargo test -p lambda-vm-prover --features hash-metrics --test whir_transcript_configuration + $(MAKE) test-rpx-host-kat # === Quick test shortcuts === @@ -553,6 +567,28 @@ test: compile-programs test-syscalls test-ethrex-crypto test-fast: compile-recursion-elfs cargo test -p lambda-vm-prover -p stark -p executor -F stark/parallel +# ★ The RPX device kernel's arithmetic, checked WITHOUT a GPU. +# +# `kernels/rpx.cu` is compiled as ordinary host C++ through `cuda_host_shim.h`, +# so its field primitives, MDS, S-boxes, cubic extension, seven-round schedule, +# leaf sponge, Merkle parent and every leaf kernel's read pattern are pinned in +# seconds on a laptop. That matters here because GPU CI runs only on +# merge_group, so without this the two WHIR coset kernels — which exist nowhere +# else — would reach a GPU unchecked. +# +# ⚠ Necessary, never sufficient: it cannot tell you whether nvcc accepts the +# file, nor anything about execution rather than arithmetic (grid indexing, +# register pressure, local-memory spills). Those still belong to the GPU tests. +HOST_KAT_DIR := crypto/math-cuda/tests/host_kat +HOST_KAT_CXXFLAGS := -std=c++17 -O2 -Wall -Wno-unknown-pragmas \ + -I$(HOST_KAT_DIR) -Icrypto/math-cuda/kernels + +test-rpx-host-kat: + @mkdir -p target/host_kat + $(CXX) $(HOST_KAT_CXXFLAGS) \ + -o target/host_kat/rpx_host_kat $(HOST_KAT_DIR)/rpx_host_kat.cpp + ./target/host_kat/rpx_host_kat + # Prover tests only test-prover: compile-recursion-elfs cargo test -p lambda-vm-prover @@ -690,6 +726,16 @@ lint: # cubin stubs when nvcc is absent, so this checks on a GPU-less host (CI lint runner, dev laptop) # too — no GPU required. Catches cuda-gated breakage that the non-cuda passes above miss. cargo clippy --workspace --all-targets --features lambda-vm-prover/cuda -- -D warnings -A clippy::op_ref + # `hash-metrics` is host-only and off by default, so no pass above compiles it. + # Without this line the feature can rot untouched — which is how its Merkle + # counters stayed keccak-only after a second hash arrived, reporting ZERO for + # the arm whose whole purpose was to change the hashing. Lints, does not run: + # its tests are in the `test` target. + cargo clippy -p crypto --all-targets --features hash-metrics -- -D warnings -A clippy::op_ref + # The prover's own `hash-metrics` passthrough gates the per-arm transcript + # line and the system test that reads it; without this line neither compiles + # in any pass, which is how an instrument rots. + cargo clippy -p lambda-vm-prover --all-targets --features hash-metrics -- -D warnings -A clippy::op_ref flamegraph-prover: cd crypto/stark && samply record cargo bench --bench profile_prover --features parallel diff --git a/crypto/crypto/src/fiat_shamir/default_transcript.rs b/crypto/crypto/src/fiat_shamir/default_transcript.rs index d64f805a2..e0295cd1f 100644 --- a/crypto/crypto/src/fiat_shamir/default_transcript.rs +++ b/crypto/crypto/src/fiat_shamir/default_transcript.rs @@ -1,6 +1,6 @@ use crate::fiat_shamir::is_transcript::{IsStarkTranscript, IsTranscript}; +use crate::fiat_shamir::transcript_hash::{KeccakTranscriptHash, TranscriptHash}; -use crate::hash::platform_keccak::PlatformKeccak256 as Keccak256; use core::marker::PhantomData; use digest::Digest; use math::{ @@ -16,8 +16,8 @@ use math::{ /// per squeeze). const SQUEEZE_LEN: usize = 32; -/// Keccak-sponge Fiat-Shamir transcript with a Plonky3-style duplex output -/// buffer. +/// Sponge Fiat-Shamir transcript with a Plonky3-style duplex output buffer, +/// over the hash `T` names. /// /// Challenges are derived by squeezing the sponge and rejection-sampling field /// coordinates directly from those bytes — there is **no CSPRNG**. Earlier this @@ -28,8 +28,14 @@ const SQUEEZE_LEN: usize = 32; /// free. The output buffer amortizes one squeeze across up to `SQUEEZE_LEN / 8` /// 64-bit candidates, so a cubic-extension element (3 coordinates) usually costs /// a single squeeze. -pub struct DefaultTranscript { - hasher: Keccak256, +/// +/// `T` defaults to [`KeccakTranscriptHash`], so `DefaultTranscript::::new(..)` +/// still names exactly the transcript this system has always produced: every +/// method body below is hash-agnostic, and the parameter only decides which +/// `digest::Digest` the sponge is. Nothing about the keccak configuration's +/// bytes moves. +pub struct DefaultTranscript { + hasher: T::Digest, /// Duplex output buffer: bytes squeezed from the sponge, consumed 8 at a /// time by field/`u64` sampling. Positions `[out_pos, SQUEEZE_LEN)` are the /// bytes not yet handed out; `out_pos == SQUEEZE_LEN` means "empty, squeeze @@ -37,10 +43,10 @@ pub struct DefaultTranscript { /// squeeze can never reflect input appended after it was produced. out_buf: [u8; SQUEEZE_LEN], out_pos: usize, - phantom: PhantomData, + phantom: PhantomData<(F, T)>, } -impl Clone for DefaultTranscript { +impl Clone for DefaultTranscript { fn clone(&self) -> Self { Self { hasher: self.hasher.clone(), @@ -51,14 +57,15 @@ impl Clone for DefaultTranscript { } } -impl DefaultTranscript +impl DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { pub fn new(data: &[u8]) -> Self { let mut res = Self { - hasher: Keccak256::new(), + hasher: T::Digest::new(), out_buf: [0u8; SQUEEZE_LEN], // Empty: the first sample forces a squeeze. out_pos: SQUEEZE_LEN, @@ -69,12 +76,34 @@ where } /// Raw squeeze: finalize the current sponge state, advance the hash chain by - /// absorbing the (reversed) output, and return it. Also invalidates the - /// duplex output buffer, so interleaving raw `sample()` calls with buffered - /// field/`u64` sampling can never reuse stale squeeze bytes. + /// absorbing the output, and return it. Also invalidates the duplex output + /// buffer, so interleaving raw `sample()` calls with buffered field/`u64` + /// sampling can never reuse stale squeeze bytes. + /// + /// ★ The byte order is the configuration's, via + /// [`TranscriptHash::REVERSES_SQUEEZE`] — `true` for keccak, which is the + /// convention every proof on this branch has been produced under, and + /// `false` for an algebraic sponge, whose squeeze is already four canonical + /// felts and whose consumer is a field-native verifier that would otherwise + /// spend rows undoing the reversal. + /// + /// ⚠ The returned bytes and the chained bytes are the SAME value, and that + /// is deliberate: a replaying verifier reproducing this chain would + /// otherwise have two byte conventions to carry instead of none. Whichever + /// order the constant selects applies to both. + /// + /// The constant is associated, so each configuration monomorphises to + /// straight-line code — the keccak arm keeps the instruction sequence it + /// had before this became a choice. pub fn sample(&mut self) -> [u8; 32] { + // ★ Hash-agnostic, and deliberately here rather than inside a digest: + // a counter that lives in keccak reads ZERO for an algebraic + // transcript, which is indistinguishable from "no transcript ran". + crate::hash_metrics::count_transcript_squeeze::(); let mut result_hash: [u8; 32] = self.hasher.finalize_reset().into(); - result_hash.reverse(); + if T::REVERSES_SQUEEZE { + result_hash.reverse(); + } self.hasher.update(result_hash); self.out_pos = SQUEEZE_LEN; result_hash @@ -83,6 +112,15 @@ where /// Next 64-bit candidate from the duplex output buffer, refilling with one /// squeeze when fewer than 8 bytes remain. Big-endian, matching the byte /// order `sample_u64` used when it read directly from `sample()`. + /// + /// ★ `SQUEEZE_LEN` is 32 and every read is 8, so `out_pos` only ever takes + /// the values `0, 8, 16, 24, 32` and a candidate is always a whole 8-byte + /// group — never two halves of adjacent ones. That is what lets a + /// configuration whose squeeze is four canonical felts promise + /// `CANDIDATES_PER_COORDINATE = Some(1)`: the felt boundaries and the read + /// boundaries are the same boundaries. `append_bytes` invalidates the + /// buffer wholesale rather than partially, so the alignment survives + /// interleaved absorbs. fn next_sample_u64(&mut self) -> u64 { if self.out_pos + 8 > SQUEEZE_LEN { self.out_buf = self.sample(); @@ -95,9 +133,18 @@ where } } -impl Default for DefaultTranscript +impl crate::fiat_shamir::transcript_hash::HasTranscriptHash for DefaultTranscript +where + F: HasDefaultTranscript, + T: TranscriptHash, +{ + type Hash = T; +} + +impl Default for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { fn default() -> Self { @@ -105,9 +152,10 @@ where } } -impl IsTranscript for DefaultTranscript +impl IsTranscript for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, { fn append_bytes(&mut self, new_bytes: &[u8]) { @@ -115,17 +163,35 @@ where // subsequent challenge must depend on this input, so drop the bytes // squeezed before it. self.out_pos = SQUEEZE_LEN; + crate::hash_metrics::count_transcript_absorb::(); self.hasher.update(new_bytes); } fn append_field_element(&mut self, element: &FieldElement) { // Absorb, same invalidation as `append_bytes` (the field element's bytes // are streamed straight into the sponge with no intermediate `Vec`). + // + // ⚠ Counted PER `update` rather than once per call, because that is the + // unit `absorb_calls` has always used and the dimension a block- + // absorption change moves. Today the degree-3 extension writes one + // 24-byte buffer and calls the sink once, so the two happen to agree — + // a field or a serialisation that streams in pieces would not, and the + // counter should follow the sponge rather than the argument list. self.out_pos = SQUEEZE_LEN; - element.stream_bytes(&mut |b| self.hasher.update(b)); + let hasher = &mut self.hasher; + element.stream_bytes(&mut |b| { + crate::hash_metrics::count_transcript_absorb::(); + hasher.update(b); + }); } fn state(&self) -> [u8; 32] { + // ★ Counted, and NOT as a squeeze. This finalizes a CLONE: no reset and + // no re-absorb, so the chain does not advance and a counter hooked to + // `sample` cannot see it. There is one per grind check — 2,996 on a + // block proof against 182,734 squeezes — so a counter that reported + // only their sum could be checked against neither. + crate::hash_metrics::count_transcript_state::(); self.hasher.clone().finalize().into() } @@ -145,9 +211,10 @@ where } } -impl IsStarkTranscript for DefaultTranscript +impl IsStarkTranscript for DefaultTranscript where F: HasDefaultTranscript, + T: TranscriptHash, FieldElement: AsBytes, S: IsField + IsSubFieldOf, { diff --git a/crypto/crypto/src/fiat_shamir/mod.rs b/crypto/crypto/src/fiat_shamir/mod.rs index a16f61b62..27a518d0b 100644 --- a/crypto/crypto/src/fiat_shamir/mod.rs +++ b/crypto/crypto/src/fiat_shamir/mod.rs @@ -6,3 +6,4 @@ pub mod default_transcript; pub mod is_transcript; +pub mod transcript_hash; diff --git a/crypto/crypto/src/fiat_shamir/transcript_hash.rs b/crypto/crypto/src/fiat_shamir/transcript_hash.rs new file mode 100644 index 000000000..67130ae02 --- /dev/null +++ b/crypto/crypto/src/fiat_shamir/transcript_hash.rs @@ -0,0 +1,164 @@ +//! The hash a Fiat-Shamir transcript runs on. +//! +//! [`DefaultTranscript`](super::default_transcript::DefaultTranscript) is a thin +//! `digest::Digest` wrapper, so swapping the hash is a type substitution. What +//! this trait adds beyond naming a digest is that the name travels with the +//! proof: a transcript is part of what a replaying verifier has to reproduce, +//! so the configuration has to be something a call site can state rather than +//! something a default decides. +//! +//! ★ **The two constants, and why they arrived together.** A straight-line +//! machine replaying the transcript needs a draw count that does not depend on +//! the data, which is what [`TranscriptHash::CANDIDATES_PER_COORDINATE`] states. +//! For an algebraic sponge the count is one, because a squeeze is already field +//! elements — but only if the sampler sees them as the sponge produced them. +//! +//! An earlier revision of this file argued the opposite, and the argument was +//! correct at the time: [`DefaultTranscript::sample`] reversed all 32 bytes of +//! every squeeze, so the first eight bytes a sampler read were the LAST felt's +//! canonical bytes backwards — a number with no canonicality property, for +//! which a one-candidate schedule would have been a claim no test could reach. +//! [`TranscriptHash::REVERSES_SQUEEZE`] is what removed that obstacle, so the +//! two constants are one change: the schedule is a consequence of the byte +//! order, not an independent decision. + +use digest::{Digest, FixedOutputReset, OutputSizeUser, typenum::U32}; + +use crate::hash::platform_keccak::PlatformKeccak256; +use crate::hash::rpx::Rpx256Digest; + +/// ★★ Which [`TranscriptHash`] a concrete transcript type is running on. +/// +/// The type-level answer to "what hash is this transcript", so a caller that +/// needs a transcript to match something else can say so in a `where` clause +/// and have the compiler check it. +/// +/// # Why this exists +/// +/// `DefaultTranscript`'s hash parameter has a default, so `DefaultTranscript::` +/// is a keccak transcript and looks like it names no hash at all. Every WHIR +/// call site wrote exactly that, under a dispatch that selects the hash for the +/// Merkle backend and the grind — so the RPX configuration ran an RPX backend, +/// an RPX grind and a KECCAK transcript, for four measured A/Bs, without one +/// instrument disagreeing. Nothing failed: the proofs were valid and the two +/// arms genuinely differed. +/// +/// Naming the hash at every call site would not have prevented it — a site can +/// name the wrong one as easily as it can take a default. What prevents it is +/// an equality the compiler checks, which is what this trait makes sayable: +/// `T: HasTranscriptHash::Transcript>` on the WHIR entry +/// points turns a mismatched transcript into a build error, and leaves the +/// several hundred STARK call sites — for which keccak is not a default but the +/// answer — untouched. +pub trait HasTranscriptHash { + /// The configuration this transcript's sponge runs on. + type Hash: TranscriptHash; +} + +/// One Fiat-Shamir configuration: the digest the sponge runs on. +pub trait TranscriptHash: 'static { + /// The sponge's hash. + /// + /// `Clone` because the transcript is snapshotted (the GPU FRI fallback + /// restores it) and because `state()` finalizes a clone. `FixedOutputReset` + /// because the squeeze is `finalize_reset`. The 32-byte output size is + /// pinned rather than left associated: `state()` returns `[u8; 32]`, and + /// that is what seeds grinding, so a configuration with a different digest + /// width would not be a drop-in anywhere it is consumed. `'static` because + /// the GPU grinding dispatch keys the device search on the concrete digest + /// by `TypeId`, the way the Merkle backends' keccak fast paths do. + type Digest: Digest + FixedOutputReset + OutputSizeUser + Clone + 'static; + + /// Name for KATs, banners and diagnostics. + const NAME: &'static str; + + /// ★ Whether [`DefaultTranscript::sample`] reverses the 32 bytes of a + /// squeeze before handing them out and chaining them back in. + /// + /// A byte convention, not a security parameter: reversing 32 bytes is a + /// bijection, so the distribution a sampler draws from is the digest's + /// either way. What it decides is *which* bijection sits between the digest + /// and the sampler, and for an algebraic sponge that is the whole question + /// — see [`CANDIDATES_PER_COORDINATE`](Self::CANDIDATES_PER_COORDINATE). + /// + /// ⚠ It is `true` for keccak because that is the convention this system's + /// proofs have always been produced under, and moving it would change every + /// keccak proof on this branch for no gain. It is not `true` for any reason + /// a new configuration should copy. + const REVERSES_SQUEEZE: bool; + + /// ★ How many `u64` candidates a base-field coordinate needs, when that is + /// a fixed number. + /// + /// `None` means unbounded: the sampler rejects candidates `>= p` and draws + /// again, however many times that takes. That is the honest description of + /// a byte-oriented hash, whose squeeze is 32 uniform bytes with no relation + /// to the field — there is no bound, only a probability. + /// + /// `Some(n)` is a promise that `n` candidates always suffice, and it exists + /// for a replaying verifier that cannot branch on how many it needed. A + /// configuration may only claim it if the claim is structural. `Some(1)` + /// here means a squeeze IS field elements, canonically encoded, so the + /// rejection test is unreachable rather than merely unlikely. + /// + /// ⚠ Scope: this governs `sample_field_element` alone. Query indices come + /// from `sample_u64`, whose rejection tests `candidate >= 2^64 mod bound` — + /// a different test, about which canonicality says nothing. Those draws are + /// single-candidate on this branch for an unrelated reason (every WHIR + /// query bound is a power of two, making that threshold zero), which is + /// hash-independent and pinned separately. Two facts, two reasons; a + /// verifier that needs both must not take this constant as evidence of the + /// other. + const CANDIDATES_PER_COORDINATE: Option; +} + +/// The keccak-256 configuration — what every `DefaultTranscript` is unless a +/// caller says otherwise, and byte-for-byte the transcript this system has +/// always produced. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct KeccakTranscriptHash; + +impl TranscriptHash for KeccakTranscriptHash { + type Digest = PlatformKeccak256; + + const NAME: &'static str = "keccak256"; + + /// The convention every keccak proof on this branch was produced under. + const REVERSES_SQUEEZE: bool = true; + + /// Unbounded, and it has to be: a keccak squeeze is 32 uniform bytes, so a + /// candidate lands in `[p, 2^64)` with probability about `2^-32` per draw + /// and the number of draws has no ceiling. Small is not fixed. + const CANDIDATES_PER_COORDINATE: Option = None; +} + +/// The RPX256 configuration — the algebraic sponge, for a transcript a +/// field-native verifier has to replay. +/// +/// ★ **Why this one does not reverse, and what that buys.** A squeeze here is +/// `digest_to_commitment(sponge_leaf_bytes(..))` — four canonical felts, each +/// eight big-endian bytes. Handed out in that order, every 8-byte group a +/// sampler reads is a field element by construction, so the rejection test is +/// unreachable and one candidate per coordinate is exact rather than typical. +/// Reversed, the first group is the LAST felt's bytes backwards, a number with +/// no canonicality property at all — which is why an earlier revision of this +/// file argued `Some(1)` could not be claimed. It could not, then. +/// +/// The reversal had no security role to lose: it is a bijection on 32 bytes, so +/// challenges are the digest's distribution before and after. What it cost was +/// the LFM replay — four byte-reversals and four modular reductions per squeeze +/// that a field-native verifier has to pay in rows to undo an encoding the host +/// had no reason to apply. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct RpxTranscriptHash; + +impl TranscriptHash for RpxTranscriptHash { + type Digest = Rpx256Digest; + + const NAME: &'static str = "rpx256"; + + const REVERSES_SQUEEZE: bool = false; + + /// Structural, not probabilistic: see the type's documentation. + const CANDIDATES_PER_COORDINATE: Option = Some(1); +} diff --git a/crypto/crypto/src/grinding.rs b/crypto/crypto/src/grinding.rs index b162ee54c..5ea0d0908 100644 --- a/crypto/crypto/src/grinding.rs +++ b/crypto/crypto/src/grinding.rs @@ -9,9 +9,21 @@ //! prover and the multilinear one grind against the same primitive, and so does //! the device dispatch below: `multilinear` cannot reach `stark`, which depends //! on it. +//! +//! # The hash is a parameter, with NO default +//! +//! The construction is two hashes of one block each — 41 bytes inner, 40 bytes +//! outer — so it costs two compressions whichever hash `D` is, and the seed and +//! the digest are `[u8; 32]` on both sides. Swapping the hash is therefore a +//! type substitution that changes the shape of nothing. +//! +//! `D` is deliberately a parameter rather than a defaulted one: the +//! proof-of-work hash has to be the proof's hash, and a defaulted `D` would +//! silently keep grinding on keccak for a configuration that had moved +//! everything else — self-consistent between prover and verifier, and therefore +//! silent. Every call site states its hash. -use crate::hash::platform_keccak::PlatformKeccak256 as Keccak256; -use digest::Digest; +use digest::{Digest, OutputSizeUser, typenum::U32}; #[cfg(feature = "parallel")] use rayon::prelude::{IntoParallelIterator, ParallelIterator}; @@ -30,14 +42,53 @@ const PREFIX: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xed]; /// # Returns /// /// `true` if the number of leading zeros is at least `grinding_factor`, and `false` otherwise. -pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool { +pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool +where + D: Digest + OutputSizeUser + 'static, +{ debug_assert!( (1..=64).contains(&grinding_factor), "grinding_factor must be in 1..=64, got {grinding_factor}" ); - let inner_hash = get_inner_hash(seed, grinding_factor); + let inner_hash = get_inner_hash::(seed, grinding_factor); let limit = 1 << (64 - grinding_factor); - is_valid_nonce_for_inner_hash(&inner_hash, nonce, limit) + is_valid_nonce_for_inner_hash::(&inner_hash, nonce, limit) +} + +/// ⚠ **A ground proof is NOT byte-reproducible, and the reason is worse than +/// it looks.** +/// +/// [`generate_nonce`] returns *a* valid nonce, not *the* valid nonce: under +/// `parallel` it is rayon's `find_any`, which hands back whichever worker +/// finished first, and on the device arm it is whatever the kernel's scan +/// reached. That much is a known property. What makes it load-bearing for any +/// byte gate is what happens next: **the nonce is absorbed into the +/// transcript** (`multilinear::whir_chain::grind`, and the univariate prover +/// likewise), so every challenge drawn after the first grind depends on which +/// valid nonce the search happened to return. Two honest runs therefore differ +/// in every Merkle root, every out-of-domain value and every opening from the +/// first grind onward — and **no post-hoc normalisation of the nonce fields can +/// recover the agreement**, because the divergence is not in the nonce fields. +/// +/// [`deterministic`] is the escape hatch a byte gate needs: with +/// `LAMBDA_VM_DETERMINISTIC_GRIND` set, the search returns the SMALLEST valid +/// nonce, which is a function of the seed alone, so the whole proof becomes +/// reproducible. It is off by default and changes nothing about validity — the +/// verifier accepts any nonce passing [`is_valid_nonce`] — only about which of +/// them is chosen. Prove time rises: the smallest-first search cannot stop at +/// the first hit any worker finds. +/// +/// Read once, cached, presence-based — the convention `LAMBDA_VM_NO_GPU_GRIND` +/// uses. +#[cfg(feature = "std")] +pub fn deterministic() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| std::env::var_os("LAMBDA_VM_DETERMINISTIC_GRIND").is_some()) +} + +#[cfg(not(feature = "std"))] +pub fn deterministic() -> bool { + false } /// Performs grinding, returning a new nonce for the proof. @@ -46,6 +97,9 @@ pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool /// to the left. /// `prefix` is the bit-string `0x123456789abcded` /// +/// Which valid nonce comes back is NOT a contract — see [`deterministic`] for +/// the one case where it is, and for why that matters to a byte gate. +/// /// # Parameters /// /// * `seed`: the input seed, @@ -54,39 +108,98 @@ pub fn is_valid_nonce(seed: &[u8; 32], nonce: u64, grinding_factor: u8) -> bool /// # Returns /// /// A `nonce` satisfying the required condition. -pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option { +pub fn generate_nonce(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: Digest + OutputSizeUser + 'static, +{ debug_assert!( (1..=64).contains(&grinding_factor), "grinding_factor must be in 1..=64, got {grinding_factor}" ); - let inner_hash = get_inner_hash(seed, grinding_factor); + if deterministic() { + return generate_nonce_smallest::(seed, grinding_factor); + } + let inner_hash = get_inner_hash::(seed, grinding_factor); let limit = 1 << (64 - grinding_factor); #[cfg(not(feature = "parallel"))] return (0..u64::MAX).find(|&candidate_nonce| { - is_valid_nonce_for_inner_hash(&inner_hash, candidate_nonce, limit) + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) }); #[cfg(feature = "parallel")] return (0..u64::MAX).into_par_iter().find_any(|&candidate_nonce| { - is_valid_nonce_for_inner_hash(&inner_hash, candidate_nonce, limit) + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) }); } -/// Successful GPU grind dispatches — one per nonce search that ran on device -/// and produced a nonce the host check accepted (a device miss or an invalid -/// kernel result falls back to the CPU search and is not counted). +/// ★ The SMALLEST valid nonce — a function of the seed and the factor alone, +/// and therefore the thing a byte gate can reproduce. +/// +/// Exposed as its own entry point rather than reachable only through the +/// environment, so the property "this is reproducible, and it is genuinely the +/// smallest" is testable without a process-global switch. `find_first` rather +/// than `find_any`: rayon prunes candidates above the best hit so far, so the +/// cost is bounded by the smallest hit's index rather than by the whole range, +/// but it cannot stop as early as `find_any` and that is the price of the +/// property. +pub fn generate_nonce_smallest(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: Digest + OutputSizeUser + 'static, +{ + debug_assert!( + (1..=64).contains(&grinding_factor), + "grinding_factor must be in 1..=64, got {grinding_factor}" + ); + let inner_hash = get_inner_hash::(seed, grinding_factor); + let limit = 1 << (64 - grinding_factor); + + #[cfg(not(feature = "parallel"))] + return (0..u64::MAX).find(|&candidate_nonce| { + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) + }); + + #[cfg(feature = "parallel")] + return (0..u64::MAX) + .into_par_iter() + .find_first(|&candidate_nonce| { + is_valid_nonce_for_inner_hash::(&inner_hash, candidate_nonce, limit) + }); +} + +/// Successful KECCAK GPU grind dispatches — one per nonce search that ran on +/// device and produced a nonce the host check accepted (a device miss or an +/// invalid kernel result falls back to the CPU search and is not counted). #[cfg(feature = "cuda")] static GPU_GRIND_CALLS: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); +/// ★ The same for RPX, counted SEPARATELY. +/// +/// Two counters rather than one, because the question an assertion needs to +/// answer is not "did a grind reach the device" but "did the RIGHT kernel run". +/// A single counter is satisfied by the keccak arm firing under an RPX +/// configuration — which is the precise failure this dispatch exists to make +/// impossible, so it must not also be the failure the test cannot see. +#[cfg(feature = "cuda")] +static GPU_GRIND_CALLS_RPX: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(0); + #[cfg(feature = "cuda")] pub fn gpu_grind_calls() -> u64 { GPU_GRIND_CALLS.load(core::sync::atomic::Ordering::Relaxed) } +/// Successful RPX device grinds. Zero under a keccak configuration. +#[cfg(feature = "cuda")] +pub fn gpu_grind_calls_rpx() -> u64 { + GPU_GRIND_CALLS_RPX.load(core::sync::atomic::Ordering::Relaxed) +} + +/// Zeroes BOTH counters — a measuring caller resets once and reads both, so an +/// arm cannot inherit the previous arm's count. #[cfg(feature = "cuda")] pub fn reset_gpu_grind_calls() { GPU_GRIND_CALLS.store(0, core::sync::atomic::Ordering::Relaxed); + GPU_GRIND_CALLS_RPX.store(0, core::sync::atomic::Ordering::Relaxed); } /// Grind on the GPU when a CUDA backend is up, falling back to the CPU search @@ -95,8 +208,27 @@ pub fn reset_gpu_grind_calls() { /// while the CPU's `find_any` returns an arbitrary one. Neither is a contract — /// the verifier accepts any nonce passing [`is_valid_nonce`], and nothing /// downstream depends on the choice. +/// +/// ★ The dispatch is keyed on WHICH DEVICE KERNEL `D` HAS, by `TypeId` — the +/// same discipline the Merkle backends' guest fast paths use, and the host twin +/// of the `DeviceHash` key the commit path carries. +/// +/// Two arms, and the endianness differs between them: keccak's kernel takes the +/// inner hash as four LITTLE-endian lanes, RPX's as four BIG-endian felts. A +/// hash with no kernel takes the CPU search and is correct there, rather than +/// being handed a nonce another hash's kernel found. +/// +/// ⚠ **This guard is load-bearing on the measurement, not only on correctness.** +/// While it read "is `D` keccak", an RPX configuration fell to the host search: +/// ~2^20 RPX permutations per grind, thousands of grinds per block proof. A +/// measured WHIR block arm came in at 571 s against keccak's 39 s, and ~510 s of +/// that was this line — the device idle, 31 host threads at 90%, with a +/// correct, KAT-pinned `rpx_grind_search` sitting in the cubin unused. #[cfg(feature = "cuda")] -pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option { +pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: Digest + OutputSizeUser + 'static, +{ debug_assert!( (1..=64).contains(&grinding_factor), "grinding_factor must be in 1..=64, got {grinding_factor}" @@ -106,15 +238,48 @@ pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option< // and fallback-path coverage. Cached; read once. static GPU_DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); if *GPU_DISABLED.get_or_init(|| std::env::var_os("LAMBDA_VM_NO_GPU_GRIND").is_some()) { - return generate_nonce(seed, grinding_factor); + return generate_nonce::(seed, grinding_factor); + } + // The device search returns the smallest nonce in the range IT scanned, + // which is not the same promise as the smallest that exists. Under the + // deterministic knob the host search is the only one that makes the + // promise, so the device steps aside rather than being trusted to keep it. + if deterministic() { + return generate_nonce::(seed, grinding_factor); } - let inner_lanes = inner_hash_lanes(seed, grinding_factor); - if let Some(nonce) = math_cuda::grinding::generate_nonce_gpu(&inner_lanes, grinding_factor) { + // A hash with no device kernel takes the CPU search. + if !has_device_kernel::() { + return generate_nonce::(seed, grinding_factor); + } + let is_keccak = core::any::TypeId::of::() + == core::any::TypeId::of::(); + + // Each arm reads the SAME 32 bytes in its own byte order — see + // `math_cuda::grinding`'s header for what crossing them does. + let found = if is_keccak { + math_cuda::grinding::generate_nonce_gpu( + &inner_hash_lanes::(seed, grinding_factor), + grinding_factor, + ) + } else { + math_cuda::grinding::generate_nonce_rpx_gpu( + &inner_hash_felts::(seed, grinding_factor), + grinding_factor, + ) + }; + + if let Some(nonce) = found { // Validate unconditionally (one host hash against the ~2^grinding_factor // device search): a kernel/driver defect must degrade to the CPU search, - // never append an unverifiable nonce to the transcript. - if is_valid_nonce(seed, nonce, grinding_factor) { - GPU_GRIND_CALLS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + // never append an unverifiable nonce to the transcript. This is also + // what would catch the two byte orders being crossed — the nonce would + // be valid under a message the host never hashed. + if is_valid_nonce::(seed, nonce, grinding_factor) { + if is_keccak { + GPU_GRIND_CALLS.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + } else { + GPU_GRIND_CALLS_RPX.fetch_add(1, core::sync::atomic::Ordering::Relaxed); + } return Some(nonce); } // eprintln, not log::warn: the CLI initialises env_logger with no @@ -126,26 +291,34 @@ pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option< "[gpu] grind returned an invalid nonce ({nonce}); falling back to the CPU search" ); } - generate_nonce(seed, grinding_factor) + generate_nonce::(seed, grinding_factor) } #[cfg(not(feature = "cuda"))] -pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option { - generate_nonce(seed, grinding_factor) +pub fn generate_nonce_maybe_gpu(seed: &[u8; 32], grinding_factor: u8) -> Option +where + D: Digest + OutputSizeUser + 'static, +{ + generate_nonce::(seed, grinding_factor) } /// Checks if the leftmost 8 bytes of `Hash(inner_hash || candidate_nonce)` are less than `limit` /// when interpreted as `u64`. #[inline(always)] -fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool { +fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, limit: u64) -> bool +where + D: Digest + OutputSizeUser + 'static, +{ // Tag this finalize as grinding so a verify-hash metric can report it apart // (see `crate::hash_metrics`); no-op unless the `hash-metrics` feature is on. + // The metric is keccak-only by construction, so a non-keccak `D` leaves it + // at zero rather than reporting another hash's work as keccak's. crate::hash_metrics::count_grinding(); let mut data = [0; 40]; data[..32].copy_from_slice(inner_hash); data[32..].copy_from_slice(&candidate_nonce.to_be_bytes()); - let digest = Keccak256::digest(data); + let digest = D::digest(data); let seed_head = u64::from_be_bytes(digest[..8].try_into().unwrap()); seed_head < limit @@ -154,7 +327,10 @@ fn is_valid_nonce_for_inner_hash(inner_hash: &[u8; 32], candidate_nonce: u64, li /// Returns the bit-string constructed as /// Hash(prefix || seed || grinding_factor) /// `prefix` is the bit-string `0x123456789abcded` -fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] { +fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] +where + D: Digest + OutputSizeUser + 'static, +{ // Grinding finalize (see `crate::hash_metrics`); no-op unless enabled. crate::hash_metrics::count_grinding(); let mut inner_data = [0u8; 41]; @@ -162,7 +338,7 @@ fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] { inner_data[8..40].copy_from_slice(seed); inner_data[40] = grinding_factor; - let digest = Keccak256::digest(inner_data); + let digest = D::digest(inner_data); digest[..32].try_into().unwrap() } @@ -174,7 +350,45 @@ fn get_inner_hash(seed: &[u8; 32], grinding_factor: u8) -> [u8; 32] { /// (`from_le_bytes` → `from_be_bytes` reads identically at a glance) with every /// test still green, while at runtime `is_valid_nonce` rejected every device /// nonce and the search silently sat on the CPU fallback forever. -pub fn inner_hash_lanes(seed: &[u8; 32], grinding_factor: u8) -> [u64; 4] { - let inner_hash = get_inner_hash(seed, grinding_factor); +pub fn inner_hash_lanes(seed: &[u8; 32], grinding_factor: u8) -> [u64; 4] +where + D: Digest + OutputSizeUser + 'static, +{ + let inner_hash = get_inner_hash::(seed, grinding_factor); core::array::from_fn(|i| u64::from_le_bytes(inner_hash[i * 8..i * 8 + 8].try_into().unwrap())) } + +/// ★ The inner hash as the four BIG-endian `u64`s an ALGEBRAIC sponge absorbs — +/// the form the RPX device search takes as input. +/// +/// ⚠ The endianness is the whole difference from [`inner_hash_lanes`], and it is +/// not cosmetic. `felts_from_bytes` reads consecutive eight-byte groups +/// big-endian, so these four `u64`s ARE the felts the host sponge absorbs; +/// keccak reads its lanes little-endian. Crossing the two compiles and runs, and +/// produces a device search for a nonce under a message the host never hashes — +/// every returned nonce rejected, the fallback taken on every grind, and nothing +/// louder than one warning line to say so. That is why there are two named +/// functions and not one with a flag. +/// +/// The four values are already canonical: the inner hash is an algebraic +/// digest's own output, which `digest_to_commitment` writes as four canonical +/// big-endian `u64`s. Nothing here reduces them, and the device does not either. +pub fn inner_hash_felts(seed: &[u8; 32], grinding_factor: u8) -> [u64; 4] +where + D: Digest + OutputSizeUser + 'static, +{ + let inner_hash = get_inner_hash::(seed, grinding_factor); + core::array::from_fn(|i| u64::from_be_bytes(inner_hash[i * 8..i * 8 + 8].try_into().unwrap())) +} + +/// Does `D` have a device grind kernel, and therefore an arm above? +/// +/// The one place the supported set is written down. A hash added to +/// `math_cuda::grinding` without a line here silently keeps grinding on the +/// host, which is the failure that cost a measured block arm 510 seconds. +#[cfg(feature = "cuda")] +fn has_device_kernel() -> bool { + let id = core::any::TypeId::of::(); + id == core::any::TypeId::of::() + || id == core::any::TypeId::of::() +} diff --git a/crypto/crypto/src/hash/mod.rs b/crypto/crypto/src/hash/mod.rs index 78f89fca3..ee3cc352b 100644 --- a/crypto/crypto/src/hash/mod.rs +++ b/crypto/crypto/src/hash/mod.rs @@ -1,3 +1,4 @@ pub mod platform_keccak; pub mod poseidon; +pub mod rpx; pub mod sha3; diff --git a/crypto/crypto/src/hash/rpx/constants.rs b/crypto/crypto/src/hash/rpx/constants.rs new file mode 100644 index 000000000..24a3c89e3 --- /dev/null +++ b/crypto/crypto/src/hash/rpx/constants.rs @@ -0,0 +1,254 @@ +//! The round constants and the MDS row of RPO256/RPX256 at width 12 — +//! **transcribed verbatim, and deliberately carrying no prose of their own.** +//! +//! # Provenance: two independent sources, checked against each other +//! +//! These are not this project's numbers and nothing here derives them: +//! +//! 1. the spec's own generator ([eprint 2022/1577](https://eprint.iacr.org/2022/1577), +//! reference implementation `github.com/ASDiscreteMathematics/rpo`) — +//! `SHAKE256("RPO(18446744069414584321,12,4,128)", 9*2*12*7)` cut into +//! nine-byte little-endian chunks reduced mod `p`; +//! 2. `miden-crypto`'s shipped `ARK1` / `ARK2` tables +//! (`src/hash/algebraic_sponge/rescue/mod.rs`), production code since 2022. +//! +//! The SHAKE256 derivation was re-run outside this repository and reproduces +//! miden's 168 constants exactly. [`MDS_CIRC_ROW`] is likewise the spec's +//! `get_mds(12)` and miden's `MDS` first row, identically. RPO's security +//! argument is MDS-AGNOSTIC (spec §4.1: "Rescue-Prime is secure when +//! instantiated with any MDS matrix"), so the row is a speed choice — it is +//! NTT-friendly — and not a security parameter. +//! +//! ⚠ **RPX shares these tables with RPO, byte for byte.** RPX is a round-SCHEDULE +//! swap on RPO's geometry, not a redesign: same width, same rate 8 / capacity 4, +//! same digest width, same MDS and literally the same `ARK1`/`ARK2`. That is +//! what lets the nineteen external RPO known-answer vectors pin RPX's constants +//! too — see the module header of [`super`]. +//! +//! ⚠ **The same numbers appear in the CUDA kernel** (`math-cuda/kernels/rpx.cu`, +//! `__constant__ ARK1`/`ARK2`/`MDS_CIRC_ROW2`). They are pinned against each +//! other by the host known-answer harness, not by being edited together, so a +//! divergence is caught rather than merely discouraged. + +use super::STATE_FELTS; + +/// The forward S-box exponent. Like Poseidon's, 7 is forced by Goldilocks: +/// `p - 1 = 2^32 * 3 * 5 * 17 * 257 * 65537`, so neither 3 nor 5 is coprime to +/// it and neither `x^3` nor `x^5` is a permutation. +pub const ALPHA: u32 = 7; + +/// The inverse S-box exponent, `ALPHA^-1 mod (p - 1)`. +/// +/// ~2^63, and that is the point: the map is cheap in one direction and +/// astronomically dense in the other. `tests::the_inverse_exponent_inverts_alpha` +/// re-derives it rather than trusting the literal. +pub const INV_ALPHA: u64 = 10540996611094048183; + +/// Rounds. The spec's own formula gives 8; RPO ships 7 and defends the 12.5% +/// shave in §4.2 with a 1.5x margin argument and Gröbner estimates above twice +/// the security level. +pub const NUM_ROUNDS: usize = 7; + +pub const MDS_CIRC_ROW: [u64; STATE_FELTS] = [7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8]; + +pub const ARK1: [[u64; STATE_FELTS]; NUM_ROUNDS] = [ + [ + 5789762306288267392, + 6522564764413701783, + 17809893479458208203, + 107145243989736508, + 6388978042437517382, + 15844067734406016715, + 9975000513555218239, + 3344984123768313364, + 9959189626657347191, + 12960773468763563665, + 9602914297752488475, + 16657542370200465908, + ], + [ + 12987190162843096997, + 653957632802705281, + 4441654670647621225, + 4038207883745915761, + 5613464648874830118, + 13222989726778338773, + 3037761201230264149, + 16683759727265180203, + 8337364536491240715, + 3227397518293416448, + 8110510111539674682, + 2872078294163232137, + ], + [ + 18072785500942327487, + 6200974112677013481, + 17682092219085884187, + 10599526828986756440, + 975003873302957338, + 8264241093196931281, + 10065763900435475170, + 2181131744534710197, + 6317303992309418647, + 1401440938888741532, + 8884468225181997494, + 13066900325715521532, + ], + [ + 5674685213610121970, + 5759084860419474071, + 13943282657648897737, + 1352748651966375394, + 17110913224029905221, + 1003883795902368422, + 4141870621881018291, + 8121410972417424656, + 14300518605864919529, + 13712227150607670181, + 17021852944633065291, + 6252096473787587650, + ], + [ + 4887609836208846458, + 3027115137917284492, + 9595098600469470675, + 10528569829048484079, + 7864689113198939815, + 17533723827845969040, + 5781638039037710951, + 17024078752430719006, + 109659393484013511, + 7158933660534805869, + 2955076958026921730, + 7433723648458773977, + ], + [ + 16308865189192447297, + 11977192855656444890, + 12532242556065780287, + 14594890931430968898, + 7291784239689209784, + 5514718540551361949, + 10025733853830934803, + 7293794580341021693, + 6728552937464861756, + 6332385040983343262, + 13277683694236792804, + 2600778905124452676, + ], + [ + 7123075680859040534, + 1034205548717903090, + 7717824418247931797, + 3019070937878604058, + 11403792746066867460, + 10280580802233112374, + 337153209462421218, + 13333398568519923717, + 3596153696935337464, + 8104208463525993784, + 14345062289456085693, + 17036731477169661256, + ], +]; + +pub const ARK2: [[u64; STATE_FELTS]; NUM_ROUNDS] = [ + [ + 6077062762357204287, + 15277620170502011191, + 5358738125714196705, + 14233283787297595718, + 13792579614346651365, + 11614812331536767105, + 14871063686742261166, + 10148237148793043499, + 4457428952329675767, + 15590786458219172475, + 10063319113072092615, + 14200078843431360086, + ], + [ + 6202948458916099932, + 17690140365333231091, + 3595001575307484651, + 373995945117666487, + 1235734395091296013, + 14172757457833931602, + 707573103686350224, + 15453217512188187135, + 219777875004506018, + 17876696346199469008, + 17731621626449383378, + 2897136237748376248, + ], + [ + 8023374565629191455, + 15013690343205953430, + 4485500052507912973, + 12489737547229155153, + 9500452585969030576, + 2054001340201038870, + 12420704059284934186, + 355990932618543755, + 9071225051243523860, + 12766199826003448536, + 9045979173463556963, + 12934431667190679898, + ], + [ + 18389244934624494276, + 16731736864863925227, + 4440209734760478192, + 17208448209698888938, + 8739495587021565984, + 17000774922218161967, + 13533282547195532087, + 525402848358706231, + 16987541523062161972, + 5466806524462797102, + 14512769585918244983, + 10973956031244051118, + ], + [ + 6982293561042362913, + 14065426295947720331, + 16451845770444974180, + 7139138592091306727, + 9012006439959783127, + 14619614108529063361, + 1394813199588124371, + 4635111139507788575, + 16217473952264203365, + 10782018226466330683, + 6844229992533662050, + 7446486531695178711, + ], + [ + 3736792340494631448, + 577852220195055341, + 6689998335515779805, + 13886063479078013492, + 14358505101923202168, + 7744142531772274164, + 16135070735728404443, + 12290902521256031137, + 12059913662657709804, + 16456018495793751911, + 4571485474751953524, + 17200392109565783176, + ], + [ + 17130398059294018733, + 519782857322261988, + 9625384390925085478, + 1664893052631119222, + 7629576092524553570, + 3485239601103661425, + 9755891797164033838, + 15218148195153269027, + 16460604813734957368, + 9643968136937729763, + 3611348709641382851, + 18256379591337759196, + ], +]; diff --git a/crypto/crypto/src/hash/rpx/mod.rs b/crypto/crypto/src/hash/rpx/mod.rs new file mode 100644 index 000000000..f187e6a18 --- /dev/null +++ b/crypto/crypto/src/hash/rpx/mod.rs @@ -0,0 +1,560 @@ +//! Rescue-Prime eXtended (RPX256 / XHash12) over Goldilocks at width 12. +//! +//! The algebraic hash the WHIR recursion arm commits, transcripts and grinds +//! with. Ported from `prover::lfm::{rpo, rpx, algebraic_commit}` on the +//! per-table branch, where it is the production-candidate tenant of the +//! `LFM_HASH` socket; the permutation, the leaf construction, the parent and +//! every constant are byte-for-byte the same, because the CUDA kernel and its +//! known-answer tables are pinned to exactly those. +//! +//! # Why an algebraic hash at all +//! +//! Only for a proof that is going to be VERIFIED INSIDE a proof. In software a +//! keccak-f is far cheaper than this. In a field-native verifier the ratio +//! inverts by two orders of magnitude: a keccak-f[1600] costs ~73,700 trace +//! cells against RPX's 325, which is the difference between a WHIR wrap that is +//! twice today's and one that is a third of it. Nothing about this hash is an +//! improvement on keccak for a host prover, and the seam that selects it says +//! so. +//! +//! # What it is, and what it shares with RPO +//! +//! RPX is a **round-function swap on RPO's geometry**, not a redesign +//! ([eprint 2023/1045](https://eprint.iacr.org/2023/1045)): the same state +//! width 12, the same rate 8 / capacity 4, the same four-felt digest, the same +//! MDS and literally the same `ARK1`/`ARK2` tables. What changes is the +//! seven-round schedule: +//! +//! | round | kind | content | +//! |---|---|---| +//! | 0, 2, 4 | **FB** | MDS → +ARK1 → `x^7` → MDS → +ARK2 → `x^{1/7}` — RPO's round exactly | +//! | 1, 3, 5 | **E** | +ARK1 → `x^7` in the degree-3 EXTENSION, on four lane-triples. **No MDS.** | +//! | 6 | **M** | MDS → +ARK1. A linear finish, no S-box. | +//! +//! The E round has no linear layer: its only mixing is the extension +//! multiplication inside each triple, and diffusion across triples is the FB +//! rounds' job. That is the design, not an omission (✓ miden's +//! `Rpx256::apply_ext_round_ref`). +//! +//! # ⚠ PROVENANCE — WEAKER THAN RPO'S, AND THAT MUST BE SAID +//! +//! **miden publishes no RPX known-answer table** — ✓ VERIFIED, its `rpx/tests.rs` +//! carries only structural tests (consistency, determinism, padding, no-panic), +//! no oracle. So RPX cannot be anchored end to end the way RPO is, and this +//! module does not pretend otherwise. What it anchors instead: +//! +//! 1. **The shared half is externally anchored through RPO.** Seven `fb_round`s +//! compose to RPO256, and [`tests`] replays that composition over +//! miden-crypto's nineteen `hash_elements` vectors — numbers nothing in this +//! repository produced. They pin `ARK1`/`ARK2`, the MDS row and its +//! orientation, both S-box chains and the lane convention at once. +//! 2. **The new half is pinned to INDEPENDENT algorithms.** The cubic +//! extension's product against naive polynomial multiplication reduced mod +//! `φ³ − φ − 1`; `power7` against generic square-and-multiply in that +//! extension; the inverse S-box chain against `pow(INV_ALPHA)`. Different +//! algorithms for the same functions, not a second transcription. +//! 3. **The schedule** is the one miden's `Rpx256::apply_permutation` runs. +//! +//! ⚖ Net: strong on arithmetic, weaker on end-to-end identity than RPO. A +//! deployment decision should treat "no published KAT" as a real cost. RPX is +//! also a 2023 design and carries a young-design discount BLAKE3 and keccak do +//! not. +//! +//! # ⚠ NOT XHash8 +//! +//! XHash8 is the faster sibling and is deliberately not built here. Its extra +//! speed comes from a PARTIAL S-box layer (8 lanes of 12), and a partial layer +//! is one of the structural footholds the 2026 Poseidon collapse used — +//! eprint 2026/1692's S-box-skipping gadget restricts into the affine +//! complement of the un-S-boxed lanes, independent of round constants and MDS +//! choice. XHash8's S-boxes are not Poseidon's and eprint 2024/605 analyses +//! XHASH8/12 directly, so this is a flag rather than a verdict — but it is not +//! a thing to adopt quietly for the speed. +//! +//! # Lane convention, domains, and the rules that must not drift +//! +//! Lanes follow **miden's**: rate `0..8`, capacity `8..12`, digest `0..4`. +//! Capacity lane 0 carries the sponge's padding flag `len mod 8`; capacity lane +//! 1 carries a DOMAIN tag, which is miden's `merge_in_domain` mechanism. The +//! security argument is the RPX spec's Appendix C: setting a capacity element +//! to a domain tag degrades only pre-image resistance, by at most the log2 of +//! the domain space, and pre-image is not the sponge's binding term until it +//! falls under 2^128. +//! +//! ⚠ **The domain VALUES are pinned by the device kernel and its KAT tables** +//! (`rpx.cu`'s `DOMAIN_COMPRESS = 0`, `DOMAIN_LEAF = 0x4C4D464C`). The `LFM` +//! spelling of [`DOMAIN_LEAF`] is a historical name — it is `"LFML"` read as a +//! little-endian `u32` — and renaming the constant is free while **changing its +//! value forks the hash** from the kernel, from the KAT header and from every +//! root the per-table branch produced. + +pub mod constants; +#[cfg(test)] +mod tests; + +use alloc::vec::Vec; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField; +use math::field::traits::{IsField, IsPrimeField}; +use math::traits::AsBytes; + +use constants::{ARK1, ARK2, MDS_CIRC_ROW, NUM_ROUNDS}; + +/// A Goldilocks field element — the only field this hash is defined over. +pub type Fp = FieldElement; + +/// Lanes in the permutation's state. +pub const STATE_FELTS: usize = 12; +/// Lanes a block of input overwrites — the sponge's rate. +pub const RATE_FELTS: usize = 8; +/// Felts in a digest, hence a 32-byte commitment. +pub const DIGEST_FELTS: usize = 4; +/// Bytes one Goldilocks felt serialises to. +pub const BYTES_PER_FELT: usize = 8; + +/// Capacity lane carrying the sponge padding flag — reserved, never a domain. +pub const CAPACITY_PAD_LANE: usize = 0; +/// Capacity lane carrying the DOMAIN identifier — miden's `merge_in_domain` slot. +pub const CAPACITY_DOMAIN_LANE: usize = 1; + +/// The Merkle-parent domain: ZERO, deliberately. +/// +/// A parent is then bit-identical to `Rpo256::merge`/`Rpx256::merge`, so a +/// parent this code produces is checkable against miden's shipped +/// implementation without knowing anything about this codebase. +pub const DOMAIN_COMPRESS: u64 = 0; + +/// The Merkle LEAF domain — `"LFML"` as a little-endian `u32`, i.e. +/// `0x4C4D464C`. See the module header: the name is historical, the VALUE is +/// pinned by the device kernel and the KAT tables. +pub const DOMAIN_LEAF: u64 = u32::from_le_bytes(*b"LFML") as u64; + +/// Lanes per extension element: the RPX E round reads the state as FOUR triples. +pub const EXT_DEGREE: usize = 3; +/// Extension elements per E round. +pub const EXT_ELEMENTS: usize = STATE_FELTS / EXT_DEGREE; + +/// A four-felt digest. +pub type Digest = [Fp; DIGEST_FELTS]; + +/// The capacity cell for a domain: `[0, domain, 0, 0]`. +/// +/// One rule, stated once, so nothing can disagree about which lane the tag +/// lives in. +pub const fn domain_iv(domain: u64) -> [u64; DIGEST_FELTS] { + let mut iv = [0u64; DIGEST_FELTS]; + iv[CAPACITY_DOMAIN_LANE] = domain; + iv +} + +/// Is round `r` an **FB** round — MDS, forward S-box, MDS, inverse S-box? +pub const fn is_fb_round(r: usize) -> bool { + r.is_multiple_of(2) && r + 1 < NUM_ROUNDS +} + +/// Is round `r` an **E** round — constants then `x^7` in the cubic extension, +/// with NO linear layer? +pub const fn is_ext_round(r: usize) -> bool { + !r.is_multiple_of(2) +} + +/// Is round `r` the **M** round — MDS then constants, and nothing else? +pub const fn is_final_round(r: usize) -> bool { + r + 1 == NUM_ROUNDS +} + +/// Arithmetic in `GF(p³) = GF(p)[φ] / (φ³ − φ − 1)`. +/// +/// ⚠ **Not the VM's own extension**, which is built on `w³ = 2`. Mixing them +/// would be a wrong hash that still type-checks, so this carries its own +/// arithmetic explicitly and never reaches for the VM's. +pub mod cubic_ext { + use super::{EXT_DEGREE, Fp}; + + /// An extension element `a0 + a1·φ + a2·φ²`. + pub type Ext = [Fp; EXT_DEGREE]; + + /// The product, reduced by `φ³ = φ + 1` and `φ⁴ = φ² + φ`. + /// + /// The closed form rather than miden's Karatsuba arrangement, so the three + /// coefficients read as the definition. + /// `tests::the_extension_product_matches_naive_polynomial_arithmetic` pins + /// it against an independent algorithm. + pub fn mul(a: &Ext, b: &Ext) -> Ext { + [ + &(&a[0] * &b[0]) + &(&(&a[1] * &b[2]) + &(&a[2] * &b[1])), + &(&(&a[0] * &b[1]) + &(&a[1] * &b[0])) + + &(&(&(&a[1] * &b[2]) + &(&a[2] * &b[1])) + &(&a[2] * &b[2])), + &(&(&a[0] * &b[2]) + &(&a[1] * &b[1])) + &(&(&a[2] * &b[0]) + &(&a[2] * &b[2])), + ] + } + + /// The square. One function, so a squaring and a product cannot disagree. + pub fn square(a: &Ext) -> Ext { + mul(a, a) + } + + /// `a^7` by the chain `a² → a³ → a⁶ → a⁷`. + pub fn power7(a: &Ext) -> Ext { + let a2 = square(a); + let a3 = mul(&a2, a); + let a6 = square(&a3); + mul(&a6, a) + } +} + +/// `x^7`, in exactly the association the AIR's degree-3 lowering uses +/// (`x²`, `x³ = x²·x`, `x^7 = (x³)²·x`). +pub fn sbox(x: &Fp) -> Fp { + let x2 = x * x; + let x3 = &x2 * x; + let x6 = &x3 * &x3; + &x6 * x +} + +/// `x^{1/7}` over the WHOLE STATE, by miden-crypto's documented addition chain +/// (72 multiplications for a ~2^63 exponent, against ~93 for naive +/// square-and-multiply). +/// +/// ★ **Whole-state rather than per-element, and that is a measurement.** The +/// chain is 72 multiplications each depending on the last, so a single lane is +/// LATENCY-bound and the multiplier pipeline sits idle between them. The twelve +/// lanes are independent, so running them in lockstep interleaves twelve chains +/// and fills it. This layer is the dominant cost of the permutation. +pub fn inv_sbox_layer(state: &mut [Fp; STATE_FELTS]) { + /// `base^(2^m) · tail`, lane-wise — the chain's one building block. + fn exp_acc(base: &[Fp; STATE_FELTS], tail: &[Fp; STATE_FELTS], m: usize) -> [Fp; STATE_FELTS] { + let mut acc = *base; + for _ in 0..m { + for a in acc.iter_mut() { + *a = a.square(); + } + } + core::array::from_fn(|i| &acc[i] * &tail[i]) + } + + let t1: [Fp; STATE_FELTS] = core::array::from_fn(|i| state[i].square()); + let t2: [Fp; STATE_FELTS] = core::array::from_fn(|i| t1[i].square()); + let t3 = exp_acc(&t2, &t2, 3); + let t4 = exp_acc(&t3, &t3, 6); + let t5 = exp_acc(&t4, &t4, 12); + let t6 = exp_acc(&t5, &t3, 6); + let t7 = exp_acc(&t6, &t6, 31); + for (i, s) in state.iter_mut().enumerate() { + let a = (&t7[i].square() * &t6[i]).square().square(); + let b = &(&t1[i] * &t2[i]) * &*s; + *s = &a * &b; + } +} + +/// [`inv_sbox_layer`] for a single element — the same chain, not a second +/// transcription of it. +pub fn inv_sbox(x: &Fp) -> Fp { + let mut state = [*x; STATE_FELTS]; + inv_sbox_layer(&mut state); + state[0] +} + +/// The circulant MDS product, `out_i = Σ_j MDS_CIRC_ROW[(j − i) mod 12]·s_j`. +/// +/// ★ **One `u128` accumulation and one reduction per lane, not twelve field +/// multiplications.** The constants are all ≤ 26, so every term `c·s_j` fits in +/// 70 bits and the twelve-term row sum fits in 73 — comfortably inside a +/// `u128`. The row is accumulated with no reduction and reduced once at the end +/// using `2^64 ≡ EPSILON (mod p)`: `hi·2^64 + lo ≡ lo + hi·EPSILON`, and with +/// `hi < 2^9` the correction `hi·EPSILON < 2^41` needs no reduction of its own. +/// `tests::the_mds_row_sum_cannot_overflow_a_u128` asserts the bound. +pub fn mds(state: &[Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + /// `2^32 − 1`, and `2^64 ≡ EPSILON (mod p)` for the Goldilocks prime. + /// Written here rather than imported because the field crate keeps its own + /// copy private; `tests::the_epsilon_identity_holds` re-derives it. + const EPSILON: u64 = 0xFFFF_FFFF; + + let raw: [u64; STATE_FELTS] = core::array::from_fn(|j| *state[j].value()); + core::array::from_fn(|i| { + let mut acc: u128 = 0; + for (j, s) in raw.iter().enumerate() { + let c = MDS_CIRC_ROW[(j + STATE_FELTS - i) % STATE_FELTS]; + acc += (*s as u128) * (c as u128); + } + let lo = acc as u64; + let hi = (acc >> 64) as u64; + // hi < 2^9, so hi·EPSILON < 2^41 and neither `from` reduces twice. + Fp::from(lo) + Fp::from(hi * EPSILON) + }) +} + +/// ★ The RPX permutation: `FB E FB E FB E M`. +pub fn permute(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let mut s = state; + // Over ARK1 rather than over `0..NUM_ROUNDS`: the round index is still what + // `fb_round` takes, but the E and M rounds read ARK1 and only ARK1, and + // iterating it says so. + for (r, ark1) in ARK1.iter().enumerate() { + if is_fb_round(r) { + s = fb_round(s, r); + } else if is_ext_round(r) { + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ark1[lane]); + } + let mut next = [Fp::zero(); STATE_FELTS]; + for e in 0..EXT_ELEMENTS { + let base = e * EXT_DEGREE; + let x: cubic_ext::Ext = core::array::from_fn(|k| s[base + k]); + let p = cubic_ext::power7(&x); + next[base..base + EXT_DEGREE].copy_from_slice(&p); + } + s = next; + } else { + debug_assert!(is_final_round(r)); + s = mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ark1[lane]); + } + } + } + s +} + +/// One **FB** round: `MDS → +ARK1 → x^7 → MDS → +ARK2 → x^{1/7}`. +/// +/// ★ Exported because it is RPO's round EXACTLY, and seven of them composed are +/// RPO256 — which is how the nineteen external miden vectors reach RPX's +/// constants. [`tests::seven_fb_rounds_are_rpo256`] is that bridge. +pub fn fb_round(state: [Fp; STATE_FELTS], r: usize) -> [Fp; STATE_FELTS] { + let mut s = mds(&state); + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ARK1[r][lane]); + } + for v in s.iter_mut() { + *v = sbox(v); + } + s = mds(&s); + for (lane, v) in s.iter_mut().enumerate() { + *v += Fp::from(ARK2[r][lane]); + } + inv_sbox_layer(&mut s); + s +} + +// ========================================================================= +// The sponge: leaves, parents, and the felt/byte conventions +// ========================================================================= + +/// ★ **THE LEAF CAPACITY RULE, stated once.** +/// +/// Lane 0 is the padding flag `len mod 8` — zero when the length divides the +/// rate, which is why no trailing block is spent on an exact multiple — and +/// lane 1 the LEAF domain. +pub fn leaf_capacity(num_felts: usize) -> Digest { + let iv = domain_iv(DOMAIN_LEAF); + let mut cap: Digest = core::array::from_fn(|k| Fp::from(iv[k])); + cap[CAPACITY_PAD_LANE] = Fp::from((num_felts % RATE_FELTS) as u64); + cap +} + +/// ★ The rate-8 OVERWRITE duplex over a felt stream — the leaf construction. +/// +/// Each block OVERWRITES the eight rate lanes (RPO spec §2.6), so absorption +/// costs no field arithmetic outside the permutation; the tail block is +/// zero-padded. It absorbs eight fresh felts per permutation where a +/// four-felt chain absorbs four. +pub fn sponge_leaf(felts: &[Fp]) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + let cap = leaf_capacity(felts.len()); + state[RATE_FELTS..].copy_from_slice(&cap); + + if felts.is_empty() { + return [state[0], state[1], state[2], state[3]]; + } + for block in felts.chunks(RATE_FELTS) { + for (lane, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + *slot = block.get(lane).copied().unwrap_or_else(Fp::zero); + } + state = permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// `sponge_leaf(&felts_from_bytes(bytes))`, without materialising the felts. +/// +/// ⚠ Equivalent to the two-step form BY TEST +/// (`tests::sponge_leaf_bytes_matches_the_felt_form`), not by construction: the +/// trailing partial group is zero-extended on the LOW side here, which is what +/// [`felts_from_bytes`] does and is easy to get backwards. +pub fn sponge_leaf_bytes(bytes: &[u8]) -> Digest { + let num_felts = bytes.len().div_ceil(BYTES_PER_FELT); + let mut state = [Fp::zero(); STATE_FELTS]; + let cap = leaf_capacity(num_felts); + state[RATE_FELTS..].copy_from_slice(&cap); + + if bytes.is_empty() { + return [state[0], state[1], state[2], state[3]]; + } + // One rate block is eight felts, i.e. 64 bytes. + for block in bytes.chunks(RATE_FELTS * BYTES_PER_FELT) { + for (lane, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + let start = lane * BYTES_PER_FELT; + *slot = if start >= block.len() { + Fp::zero() + } else { + let end = (start + BYTES_PER_FELT).min(block.len()); + let mut b = [0u8; BYTES_PER_FELT]; + b[..end - start].copy_from_slice(&block[start..end]); + Fp::from(u64::from_be_bytes(b)) + }; + } + state = permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// ★ A Merkle parent: ONE permutation of `[left ‖ right ‖ capacity]` with the +/// compress domain, which is zero — so a parent is literally `Rpx256::merge` +/// and externally checkable against miden. +pub fn compress(left: &Digest, right: &Digest) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + state[..DIGEST_FELTS].copy_from_slice(left); + state[DIGEST_FELTS..RATE_FELTS].copy_from_slice(right); + let iv = domain_iv(DOMAIN_COMPRESS); + for (k, slot) in state[RATE_FELTS..].iter_mut().enumerate() { + *slot = Fp::from(iv[k]); + } + let out = permute(state); + [out[0], out[1], out[2], out[3]] +} + +/// Four felts as 32 canonical BIG-endian bytes. +pub fn digest_to_commitment(d: &Digest) -> [u8; 32] { + let mut out = [0u8; 32]; + for (i, f) in d.iter().enumerate() { + let v = GoldilocksField::canonical(f.value()); + out[i * BYTES_PER_FELT..(i + 1) * BYTES_PER_FELT].copy_from_slice(&v.to_be_bytes()); + } + out +} + +/// 32 bytes back to four felts. +pub fn commitment_to_digest(c: &[u8; 32]) -> Digest { + core::array::from_fn(|i| { + let mut b = [0u8; BYTES_PER_FELT]; + b.copy_from_slice(&c[i * BYTES_PER_FELT..(i + 1) * BYTES_PER_FELT]); + Fp::from(u64::from_be_bytes(b)) + }) +} + +/// Every 8-byte big-endian group of `bytes` as a felt. +/// +/// The inverse of the serialisation `ByteConversion::write_bytes_be` performs, +/// which is how leaves reach a Merkle backend. A trailing partial group is +/// zero-extended on the LOW side, matching how a short write would land. +pub fn felts_from_bytes(bytes: &[u8]) -> Vec { + bytes + .chunks(BYTES_PER_FELT) + .map(|c| { + let mut b = [0u8; BYTES_PER_FELT]; + b[..c.len()].copy_from_slice(c); + Fp::from(u64::from_be_bytes(b)) + }) + .collect() +} + +/// Decompose a field element — base or extension — into its base felts, by the +/// same serialisation the STARK uses. +/// +/// ★ Through `AsBytes::stream_bytes` rather than `ByteConversion::write_bytes_be`, +/// and the two are the SAME bytes. The reason for the weaker trait is not +/// style: a Merkle backend generic over `F` has `FieldElement: AsBytes` and +/// nothing more, so a decomposition that required `ByteConversion` could not be +/// used there at all. +pub fn element_felts(e: &FieldElement, out: &mut Vec) +where + F: IsField, + FieldElement: AsBytes, +{ + let mut buf = [0u8; 64]; + let mut len = 0usize; + e.stream_bytes(&mut |bytes| { + debug_assert!( + len + bytes.len() <= buf.len(), + "a field element must fit the scratch" + ); + buf[len..len + bytes.len()].copy_from_slice(bytes); + len += bytes.len(); + }); + out.extend(felts_from_bytes(&buf[..len])); +} + +// ========================================================================= +// The `digest::Digest` adapter — what a transcript and the grind consume +// ========================================================================= + +/// RPX256 as a `digest::Digest`, for the two places that take one: the +/// Fiat-Shamir sponge and the proof-of-work grind. +/// +/// # The construction is the LEAF one, deliberately +/// +/// Grinding hashes a byte string — `state ‖ nonce`, 40 bytes, five felts — +/// which is DATA, exactly what a leaf is. It therefore reuses [`sponge_leaf`] +/// and the LEAF domain rather than inventing a fourth. The reuse is not +/// exploitable: the grinding check tests leading zeros of a hash whose preimage +/// is transcript-bound, so colliding it with some leaf digest buys an adversary +/// nothing. +/// +/// # Why it buffers +/// +/// [`sponge_leaf`]'s padding flag is `len mod 8`, needed in the capacity before +/// the FIRST permutation, so an incremental sponge cannot start until the total +/// length is known. Inventing a length-free padding rule instead would be a +/// cryptographic decision this port does not get to make. +#[derive(Default, Clone)] +pub struct Rpx256Digest { + buf: Vec, +} + +impl Rpx256Digest { + /// The digest of everything absorbed so far. + pub fn finalize_digest(&self) -> [u8; 32] { + digest_to_commitment(&sponge_leaf_bytes(&self.buf)) + } +} + +impl digest::HashMarker for Rpx256Digest {} + +impl digest::OutputSizeUser for Rpx256Digest { + type OutputSize = digest::typenum::U32; +} + +impl digest::Update for Rpx256Digest { + fn update(&mut self, data: &[u8]) { + // ⚠ The GENERIC counters were keccak-only: `count_absorb` is bumped + // from the keccak wrapper's `update` and nothing bumped it here, so + // `absorb_calls` read ZERO for an RPX proof and `total`'s own + // documentation — which says it counts transcript squeezes — was false + // for this sponge. The same trap this module's sibling header describes + // for Merkle, which the transcript and the absorb counters never got. + crate::hash_metrics::count_absorb(data.len()); + self.buf.extend_from_slice(data); + } +} + +impl digest::FixedOutput for Rpx256Digest { + fn finalize_into(self, out: &mut digest::Output) { + crate::hash_metrics::count_total(); + out.copy_from_slice(&self.finalize_digest()); + } +} + +impl digest::Reset for Rpx256Digest { + fn reset(&mut self) { + self.buf.clear(); + } +} + +impl digest::FixedOutputReset for Rpx256Digest { + fn finalize_into_reset(&mut self, out: &mut digest::Output) { + crate::hash_metrics::count_total(); + out.copy_from_slice(&self.finalize_digest()); + self.buf.clear(); + } +} diff --git a/crypto/crypto/src/hash/rpx/tests.rs b/crypto/crypto/src/hash/rpx/tests.rs new file mode 100644 index 000000000..8e86c4561 --- /dev/null +++ b/crypto/crypto/src/hash/rpx/tests.rs @@ -0,0 +1,1084 @@ +//! The oracles RPX256 rests on, and the independent algorithms that hold up +//! the half no oracle covers. +//! +//! # Two anchors, and they are not the same strength +//! +//! **RPO's half is EXTERNAL.** Seven `fb_round`s composed ARE RPO256 — RPX is a +//! schedule swap on RPO's geometry with literally the same constants — so +//! [`MIDEN_HASH_ELEMENTS`], nineteen `hash_elements` vectors published by +//! miden-crypto, reach into this file from outside. They pin `ARK1`, `ARK2`, +//! the MDS row AND its orientation, both S-box chains and the lane convention +//! at once. Nothing in this repository produced those seventy-six numbers. +//! +//! **RPX's own half is NOT externally anchored, and this says so rather than +//! implying otherwise.** miden publishes no RPX known-answer table (✓ VERIFIED: +//! its `rpx/tests.rs` carries only structural tests — consistency, determinism, +//! padding, no-panic). So the RPX tables below are the per-table branch's host +//! implementation speaking: [`RPX_PERMUTATION_VECTORS`], [`RPX_LEAF_VECTORS`] +//! and [`RPX_PARENT_VECTORS`] are transcribed from +//! `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h` on `per-table-gpu` +//! (introduced by `50c633e1`, the tables printed by +//! `prover/tests/rpx_host_kat_vectors.rs` from `prover::lfm::rpx::Rpx256`, +//! `73ee2a64`). That is worth having for a reason beyond "someone else agrees": +//! **the CUDA kernel is pinned to those same tables**, so a port that +//! reproduces them is byte-compatible with both the other branch's host and its +//! device, which is the property H2's device half will need. +//! +//! What still has no oracle at all is the E round and the schedule. Those rest +//! on layer 2: the cubic extension's product against naive polynomial +//! arithmetic mod `φ³ − φ − 1`, `power7` against square-and-multiply, the +//! inverse S-box against `pow(INV_ALPHA)` — different algorithms for the same +//! functions, not second transcriptions of the same one. + +use super::constants::{ALPHA, ARK1, ARK2, INV_ALPHA, MDS_CIRC_ROW, NUM_ROUNDS}; +use super::*; +use alloc::vec::Vec; + +/// The Goldilocks prime. +const P: u64 = 0xFFFF_FFFF_0000_0001; + +fn fe(v: u64) -> Fp { + Fp::from(v) +} + +fn felts(vs: &[u64]) -> Vec { + vs.iter().copied().map(fe).collect() +} + +fn state_of(vs: &[u64; STATE_FELTS]) -> [Fp; STATE_FELTS] { + core::array::from_fn(|i| fe(vs[i])) +} + +fn raw(state: &[Fp; STATE_FELTS]) -> [u64; STATE_FELTS] { + core::array::from_fn(|i| GoldilocksField::canonical(state[i].value())) +} + +fn digest_of(vs: &[u64; DIGEST_FELTS]) -> Digest { + core::array::from_fn(|i| fe(vs[i])) +} + +fn raw_digest(d: &Digest) -> [u64; DIGEST_FELTS] { + core::array::from_fn(|i| GoldilocksField::canonical(d[i].value())) +} + +// ========================================================================= +// LAYER 1 — the EXTERNAL anchor: miden's RPO256 vectors, through `fb_round` +// ========================================================================= + +/// miden-crypto's own `hash_elements` known-answer table — an EXTERNAL oracle. +/// +/// Source: `miden-crypto/src/hash/algebraic_sponge/rescue/rpo/tests.rs`, +/// `EXPECTED` / `hash_test_vectors`. Entry `n` is the digest of the field +/// elements `[0, 1, …, n]`. +/// +/// Entries 1–7 and 9–19 exercise the padding path (`len % 8 ≠ 0`), entries 8 +/// and 16 the exact-block path, and everything above 8 chains two permutations +/// through the capacity — so the table pins the sponge's carry, not only one +/// permutation. +const MIDEN_HASH_ELEMENTS: [[u64; 4]; 19] = [ + [ + 8563248028282119176, + 14757918088501470722, + 14042820149444308297, + 7607140247535155355, + ], + [ + 8762449007102993687, + 4386081033660325954, + 5000814629424193749, + 8171580292230495897, + ], + [ + 16710087681096729759, + 10808706421914121430, + 14661356949236585983, + 5683478730832134441, + ], + [ + 5309818427047650994, + 17172251659920546244, + 8288476618870804357, + 18080473279382182941, + ], + [ + 3647545403045515695, + 3358383208908083302, + 8797161010298072910, + 2412100201132087248, + ], + [ + 8409780526028662686, + 214479528340808320, + 13626616722984122219, + 13991752159726061594, + ], + [ + 4800410126693035096, + 8293686005479024958, + 16849389505608627981, + 12129312715917897796, + ], + [ + 5421234586123900205, + 9738602082989433872, + 7017816005734536787, + 8635896173743411073, + ], + [ + 11707446879505873182, + 7588005580730590001, + 4664404372972250366, + 17613162115550587316, + ], + [ + 6991094187713033844, + 10140064581418506488, + 1235093741254112241, + 16755357411831959519, + ], + [ + 18007834547781860956, + 5262789089508245576, + 4752286606024269423, + 15626544383301396533, + ], + [ + 5419895278045886802, + 10747737918518643252, + 14861255521757514163, + 3291029997369465426, + ], + [ + 16916426112258580265, + 8714377345140065340, + 14207246102129706649, + 6226142825442954311, + ], + [ + 7320977330193495928, + 15630435616748408136, + 10194509925259146809, + 15938750299626487367, + ], + [ + 9872217233988117092, + 5336302253150565952, + 9650742686075483437, + 8725445618118634861, + ], + [ + 12539853708112793207, + 10831674032088582545, + 11090804155187202889, + 105068293543772992, + ], + [ + 7287113073032114129, + 6373434548664566745, + 8097061424355177769, + 14780666619112596652, + ], + [ + 17147873541222871127, + 17350918081193545524, + 5785390176806607444, + 12480094913955467088, + ], + [ + 17273934282489765074, + 8007352780590012415, + 16690624932024962846, + 8137543572359747206, + ], +]; + +/// ★ RPO256's permutation, built from RPX's OWN `fb_round`. +/// +/// This is the bridge. RPX's FB round is RPO's round exactly, so seven of them +/// composed must be RPO256 — and if they are, miden's vectors have pinned +/// RPX's constants, its MDS orientation and both its S-box chains from outside +/// this repository. Composed here rather than imported, because importing an +/// RPO implementation would only pin this file against another copy of the same +/// numbers. +fn rpo256_permute(state: [Fp; STATE_FELTS]) -> [Fp; STATE_FELTS] { + let mut s = state; + for r in 0..NUM_ROUNDS { + s = fb_round(s, r); + } + s +} + +/// miden's `hash_elements` in this module's lane convention: capacity lane 8 +/// takes `len % 8`, the rate is OVERWRITTEN, the tail is zero-padded, the +/// digest is lanes 0–3. +fn rpo_hash_elements(elements: &[u64]) -> Digest { + let mut state = [Fp::zero(); STATE_FELTS]; + state[RATE_FELTS + CAPACITY_PAD_LANE] = fe((elements.len() % RATE_FELTS) as u64); + let mut i = 0; + for e in elements { + state[i] = fe(*e); + i += 1; + if i == RATE_FELTS { + state = rpo256_permute(state); + i = 0; + } + } + if i > 0 { + while i < RATE_FELTS { + state[i] = Fp::zero(); + i += 1; + } + state = rpo256_permute(state); + } + [state[0], state[1], state[2], state[3]] +} + +/// ★★ The differential the whole module rests on: seven FB rounds are RPO256, +/// and RPO256 is what miden published. +#[test] +fn seven_fb_rounds_are_rpo256() { + for (n, want) in MIDEN_HASH_ELEMENTS.iter().enumerate() { + let input: Vec = (0..=n as u64).collect(); + let got = raw_digest(&rpo_hash_elements(&input)); + assert_eq!(got, *want, "hash_elements of 0..={n} must match miden"); + } +} + +/// The compress geometry, pinned to the EXTERNAL table rather than to our own +/// permutation: merging two four-felt cells is the same thing as hashing the +/// eight felts they hold, and the eight-element vector is +/// `MIDEN_HASH_ELEMENTS[7]`. +/// +/// This is what makes the zero compress domain a checkable claim instead of a +/// convention — under RPO, any implementation anywhere computes this digest for +/// this parent. RPX's own parent differs only in the permutation, which is +/// covered by [`RPX_PARENT_VECTORS`]. +#[test] +fn the_compress_geometry_is_a_standard_merge_under_rpo() { + let mut state = [Fp::zero(); STATE_FELTS]; + for (i, slot) in state.iter_mut().take(RATE_FELTS).enumerate() { + *slot = fe(i as u64); + } + let iv = domain_iv(DOMAIN_COMPRESS); + for (k, slot) in state[RATE_FELTS..].iter_mut().enumerate() { + *slot = fe(iv[k]); + } + let out = rpo256_permute(state); + let got: [u64; DIGEST_FELTS] = + core::array::from_fn(|i| GoldilocksField::canonical(out[i].value())); + assert_eq!( + got, MIDEN_HASH_ELEMENTS[7], + "compress(0..4, 4..8) must be the standard merge" + ); +} + +/// ✓ RPX is NOT RPO — a negative control, so "seven FB rounds are RPO256" is +/// not accidentally a statement about `permute` too. +#[test] +fn rpx_is_not_rpo_on_the_same_state() { + let s = state_of(&core::array::from_fn(|i| i as u64 + 1)); + assert_ne!( + raw(&permute(s)), + raw(&rpo256_permute(s)), + "the two schedules must not agree" + ); +} + +// ========================================================================= +// LAYER 2 — the RPX oracle: the per-table host implementation's own tables +// ========================================================================= + +/// The bare permutation on eleven states. See the module header for what this +/// is and is not: the per-table branch's host `Rpx256`, and the numbers the +/// CUDA kernel is checked against — NOT an external publication. +#[allow(clippy::type_complexity)] +const RPX_PERMUTATION_VECTORS: [(&str, [u64; 12], [u64; 12]); 11] = [ + ( + "all-zero", + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ + 8760086638283468260, + 18228666152919569253, + 4041825754230271128, + 16906183286731764961, + 4664375192219530269, + 271590372761485506, + 5612474514543166805, + 8933101171974180471, + 1556877437237031065, + 7026397410864970258, + 15101742939622740655, + 4524429088483979565, + ], + ), + ( + "all-(p-1)", + [ + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + 18446744069414584320, + ], + [ + 7040074528728887770, + 10474261017970959672, + 6160748039461781206, + 9121740959127811013, + 7259505444118573102, + 6771278935515018093, + 18386914479072470354, + 17160039764143535473, + 1815780993504974800, + 17309055307915657636, + 5977169316478634398, + 4250629519753691035, + ], + ), + ( + "lanes 0..12", + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + [ + 3614697924784493998, + 4917065433670799835, + 12893407190838344317, + 16769932886818781879, + 17010299523770013195, + 9826755761378503206, + 1872785960340665977, + 7783788981462778586, + 45778307605882514, + 7437259891664617628, + 17010253034795346176, + 6863075881906649113, + ], + ), + ( + "alternating 0 / p-1", + [ + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + 0, + 18446744069414584320, + ], + [ + 12839024277220712229, + 1805658617972785851, + 11708832562581917975, + 2207339757364837492, + 457975798096500050, + 15656130651128894835, + 3485815494872446363, + 10687968103458402677, + 10384294655078062232, + 1487178939946482695, + 12310600107129561463, + 18388841767871832735, + ], + ), + ( + "one-hot lane 0", + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [ + 8423002511501289529, + 6761734748202534392, + 17987336675889252592, + 14012777376234247391, + 15293807115397414812, + 15290017247514670316, + 10548590320248089637, + 9459855167724924903, + 10549768014422457033, + 13045952392708592140, + 3310663857881768756, + 7584810783597460418, + ], + ), + ( + "one-hot lane 11", + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + [ + 18436166275486246010, + 14000894557392395452, + 10767551609857089912, + 12516698445112165012, + 13131066481882004069, + 9858979976142754244, + 11402636824743634507, + 10600727647028701714, + 11200928220555719329, + 7317761145158236061, + 16857331551667002769, + 16879508045812612150, + ], + ), + ( + "random #1", + [ + 303661977215735624, + 5244312915552057691, + 9817756985327366386, + 15550273871372065883, + 5764353057648779642, + 16198122637140758912, + 7462824619408935181, + 3819703627846067891, + 10378249170554155646, + 11473525795005675318, + 8246620909628934680, + 4793144044164964625, + ], + [ + 15068850129045079395, + 15287067578585128518, + 13369562146120321575, + 10561395445440413441, + 9652992371859647144, + 4276856065313043669, + 5527444075954724606, + 7786060382866009904, + 16451772069079981395, + 198876956612152837, + 15815343923951857286, + 16122126005548441717, + ], + ), + ( + "random #2", + [ + 5204068831683694011, + 601380814908431653, + 258667317409904638, + 8486618912357792900, + 16418043790810515027, + 10319906524521615844, + 8286207029444254408, + 17770698039797916230, + 12310900488678790115, + 11195649432216834664, + 13332813278057623446, + 16898620073423657296, + ], + [ + 9523479656024648568, + 5510889535488554715, + 8599619832581755346, + 3318619196771576895, + 12581966946741818379, + 12200018864226225973, + 4385075405488142149, + 8051813774684357414, + 3019406547981393239, + 7453667634993074437, + 9864259903669275905, + 6156796699962990553, + ], + ), + ( + "random #3", + [ + 13533914130435405040, + 15234815373149021432, + 10183913914233800905, + 9526239132464493568, + 5375977297676405297, + 5765388458641153407, + 4908125521970473579, + 4421030864271922041, + 15641279279696351384, + 16893076439662162884, + 7253714011824234117, + 14616467593891397000, + ], + [ + 15514260962038810700, + 190255547175148079, + 15766300047716671382, + 10145444481310349528, + 6135237967701788176, + 11361125511081474273, + 9927005018743801106, + 17211086950078547559, + 10833199580085782023, + 13634008743082439065, + 6687522208929839355, + 3545879585555314384, + ], + ), + ( + "random #4", + [ + 389113379214421922, + 1947929307647562990, + 667333451960644926, + 3487966933876559811, + 4195385248066926332, + 2153180418459341747, + 2727969323864685845, + 29633526854483411, + 990649808851061115, + 1355410330370587755, + 11605520071788416946, + 4884409355120715354, + ], + [ + 7025469669435110295, + 17270957437800346011, + 13702589935335807876, + 3666927270871270796, + 16666721215101099684, + 531487850530305024, + 15550553335698242665, + 8959489596577675281, + 11020601500923732075, + 16110845767020565054, + 4778394010005480449, + 7715575140819562371, + ], + ), + ( + "canonicalisation witness", + [ + 15055324559807314153, + 10242425218814686878, + 9326602342065331773, + 15451135068213333861, + 17942679252967467289, + 9284164080268346300, + 5090350781253234438, + 9328738269791029498, + 18385380985273671691, + 3238854716908013220, + 5495049682105235955, + 15773368383738726538, + ], + [ + 1, + 9023883145409261355, + 5839950281880325605, + 5697668523532261268, + 13033383890974728246, + 14801658261553133914, + 3025695522291518949, + 12907720598453111556, + 14827640614007773288, + 14642633917625231592, + 3090884930034198616, + 2894057710100710233, + ], + ), +]; + +/// The rate-8 overwrite-duplex leaf at seven lengths: empty, a partial block, +/// one under a block, an exact block (which spends no trailing permutation), +/// one over, two exact blocks, and two plus one. +#[allow(clippy::type_complexity)] +const RPX_LEAF_VECTORS: [(usize, &[u64], [u64; 4]); 7] = [ + (0, &[], [0, 0, 0, 0]), + ( + 1, + &[14681136968691612469], + [ + 16400186102935428425, + 12817983163740802970, + 13449009006350391325, + 2209445548780258712, + ], + ), + ( + 7, + &[ + 2664695409302073823, + 17298518342786888931, + 17367242851809685948, + 13566833943477212382, + 6789339537410032387, + 5202847705797706501, + 6869254230765949416, + ], + [ + 2289345357069865559, + 8509266780934512918, + 13810958145049281723, + 5769431894700133303, + ], + ), + ( + 8, + &[ + 3521541860211663897, + 5585621328801039182, + 3314063895810834828, + 6286715337571703139, + 9272399501810688383, + 17378448552699642502, + 9663403628134293866, + 8225575178453385283, + ], + [ + 14052993739410942603, + 8384701950754250190, + 11473922331550289114, + 16644313465254305812, + ], + ), + ( + 9, + &[ + 15923052634311126246, + 10423360080185943333, + 4604695570423031111, + 15959212651715575539, + 4341333374822801132, + 3169961389438585383, + 7059846953207312362, + 6231597079039193598, + 14413065529971692326, + ], + [ + 15453186885173297365, + 11395279108043639065, + 15954005188014354330, + 2854892578083306874, + ], + ), + ( + 16, + &[ + 9660685076555889599, + 4027567791223379602, + 11432600011703367870, + 6441517771629429252, + 8272264386868866348, + 16565648022353132158, + 16844837242675693755, + 12942506659476152817, + 11839051358503478840, + 1846358602548732379, + 118703897581348635, + 14480592082795401517, + 12015885875590073011, + 7433808365622677077, + 13247077855319202624, + 17837888200692576115, + ], + [ + 18135965004560326100, + 1948492279228612931, + 17772968542724134453, + 12116464713281646840, + ], + ), + ( + 17, + &[ + 14169068543591784110, + 12906798066534908639, + 1898134805181953282, + 3700382130787856361, + 10455317549184205797, + 1564511190292879407, + 5954886065046464361, + 10320234224067579215, + 17095047743397986079, + 8434180870595516882, + 17706992797230203878, + 813257427175065251, + 13312284969041468023, + 15899260221184366980, + 5770785055252949875, + 11176385994046687487, + 8142444693260481147, + ], + [ + 430819886588247494, + 10400188655761849356, + 3003730485848167815, + 13484379440855863704, + ], + ), +]; + +/// The Merkle parent. +#[allow(clippy::type_complexity)] +const RPX_PARENT_VECTORS: [(&str, [u64; 4], [u64; 4], [u64; 4]); 2] = [ + ( + "digits 0..8", + [0, 1, 2, 3], + [4, 5, 6, 7], + [ + 10386438340626196987, + 10820383641790274229, + 5711121060683785078, + 11046870009967209474, + ], + ), + ( + "random", + [ + 10430052842846219471, + 4016318112082366688, + 17186674839268073878, + 16606021345024473049, + ], + [ + 1405896845186672283, + 13799610513837549656, + 17571522367612218822, + 18082329703565322844, + ], + [ + 18019606657308693634, + 10494109104368286361, + 7943124261980338770, + 17971490172695632899, + ], + ), +]; + +/// ★ The permutation reproduces the per-table host's outputs, lane for lane. +/// +/// Includes the "canonicalisation witness" row, whose output lane 0 is `1` — +/// the canonical twin of a raw `p + 1`. A port that forgot to reduce would +/// disagree there and nowhere else, which is exactly why that row exists. +#[test] +fn the_permutation_matches_the_per_table_host_vectors() { + for (name, input, want) in RPX_PERMUTATION_VECTORS { + let got = raw(&permute(state_of(&input))); + assert_eq!(got, want, "permutation vector {name}"); + } +} + +/// Every output lane is canonical (`< p`) — what the device kernel's final +/// reduction loop is pinned on, and a property the raw comparison above assumes. +#[test] +fn the_permutation_leaves_every_lane_canonical() { + for (name, input, _) in RPX_PERMUTATION_VECTORS { + for (lane, v) in raw(&permute(state_of(&input))).iter().enumerate() { + assert!(*v < P, "vector {name}, lane {lane}: {v} is not canonical"); + } + } +} + +/// ✓ Every input lane reaches the output — a diffusion control, so a +/// permutation that ignored half its state could not pass the vectors by luck. +#[test] +fn every_input_lane_changes_the_output() { + let base = state_of(&core::array::from_fn(|i| i as u64 * 7 + 1)); + let want = raw(&permute(base)); + for lane in 0..STATE_FELTS { + let mut moved = base; + moved[lane] += Fp::one(); + assert_ne!( + raw(&permute(moved)), + want, + "lane {lane} does not reach the output" + ); + } +} + +/// ★ The leaf sponge reproduces the per-table host's digests at every length +/// the padding rule distinguishes. +#[test] +fn the_leaf_sponge_matches_the_per_table_host_vectors() { + for (len, input, want) in RPX_LEAF_VECTORS { + assert_eq!(input.len(), len, "vector for length {len} is malformed"); + let got = raw_digest(&sponge_leaf(&felts(input))); + assert_eq!(got, want, "leaf of {len} felts"); + } +} + +/// ★ The parent reproduces the per-table host's digests. +#[test] +fn the_parent_matches_the_per_table_host_vectors() { + for (name, l, r, want) in RPX_PARENT_VECTORS { + let got = raw_digest(&compress(&digest_of(&l), &digest_of(&r))); + assert_eq!(got, want, "parent vector {name}"); + } +} + +/// ✓ A parent is order-sensitive — otherwise a Merkle tree would not bind a +/// sibling's side, and the two vectors above would not distinguish it. +#[test] +fn a_parent_depends_on_the_order_of_its_children() { + let (_, l, r, _) = RPX_PARENT_VECTORS[1]; + let (a, b) = (digest_of(&l), digest_of(&r)); + assert_ne!( + raw_digest(&compress(&a, &b)), + raw_digest(&compress(&b, &a)), + "compress must not be symmetric" + ); +} + +// ========================================================================= +// LAYER 3 — independent algorithms for the half no oracle covers +// ========================================================================= + +/// `ALPHA · INV_ALPHA ≡ 1 (mod p − 1)`, re-derived rather than trusted. +#[test] +fn the_inverse_exponent_inverts_alpha() { + const P_MINUS_ONE: u128 = (P as u128) - 1; + assert_eq!( + (ALPHA as u128 * INV_ALPHA as u128) % P_MINUS_ONE, + 1, + "INV_ALPHA is not alpha's inverse mod p-1" + ); +} + +/// `2^64 ≡ EPSILON (mod p)` — the identity the MDS reduction rests on. +#[test] +fn the_epsilon_identity_holds() { + const EPSILON: u128 = 0xFFFF_FFFF; + assert_eq!((1u128 << 64) % (P as u128), EPSILON % (P as u128)); +} + +/// The MDS row sum bounds the accumulator below `2^73`, so a `u128` cannot +/// overflow and the single-reduction shortcut is sound. +#[test] +fn the_mds_row_sum_cannot_overflow_a_u128() { + let sum: u128 = MDS_CIRC_ROW.iter().map(|c| *c as u128).sum(); + assert_eq!(sum, 160, "the MDS row sums to 160"); + let bound = sum * ((P as u128) - 1); + assert!(bound < (1u128 << 73), "the row sum needs {bound} < 2^73"); + assert!(bound < u128::MAX); +} + +/// The forward and inverse S-boxes invert each other — the property that +/// actually matters, checked on values neither chain was tuned for. +#[test] +fn the_inverse_sbox_inverts_the_forward_sbox() { + for v in [0u64, 1, 2, 7, 12345, P - 1, P - 2, 0x1234_5678_9abc_def0] { + let x = fe(v); + assert_eq!(inv_sbox(&sbox(&x)), x, "x = {v}"); + assert_eq!(sbox(&inv_sbox(&x)), x, "x = {v}"); + } +} + +/// ★ The inverse S-box's addition chain against generic exponentiation — a +/// different algorithm for the same number. +#[test] +fn the_inverse_sbox_chain_agrees_with_the_exponent() { + for v in [0u64, 1, 3, 99, 1 << 40, P - 5] { + let x = fe(v); + assert_eq!(inv_sbox(&x), x.pow(INV_ALPHA), "x = {v}"); + } +} + +/// ★ The cubic extension's product against naive polynomial multiplication +/// reduced by `φ³ = φ + 1`, written out term by term. +#[test] +fn the_extension_product_matches_naive_polynomial_arithmetic() { + fn naive(a: &cubic_ext::Ext, b: &cubic_ext::Ext) -> cubic_ext::Ext { + // The full degree-4 product, then reduce with φ³ = φ + 1, φ⁴ = φ² + φ. + let mut c = [Fp::zero(); 5]; + for (i, ai) in a.iter().enumerate() { + for (j, bj) in b.iter().enumerate() { + c[i + j] += ai * bj; + } + } + // φ³ → φ + 1 + let c3 = c[3]; + c[1] += c3; + c[0] += c3; + // φ⁴ → φ² + φ + let c4 = c[4]; + c[2] += c4; + c[1] += c4; + [c[0], c[1], c[2]] + } + + let mut seed = 0x243f_6a88_85a3_08d3u64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + fe(seed % P) + }; + for _ in 0..64 { + let a: cubic_ext::Ext = [next(), next(), next()]; + let b: cubic_ext::Ext = [next(), next(), next()]; + assert_eq!(cubic_ext::mul(&a, &b), naive(&a, &b)); + } +} + +/// ★ `power7` against generic square-and-multiply in the same extension. +#[test] +fn the_extension_power7_matches_square_and_multiply() { + fn pow(a: &cubic_ext::Ext, mut e: u32) -> cubic_ext::Ext { + let mut acc: cubic_ext::Ext = [Fp::one(), Fp::zero(), Fp::zero()]; + let mut base = *a; + while e > 0 { + if e & 1 == 1 { + acc = cubic_ext::mul(&acc, &base); + } + base = cubic_ext::square(&base); + e >>= 1; + } + acc + } + + let mut seed = 0x1357_9bdf_0246_8aceu64; + let mut next = || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + fe(seed % P) + }; + for _ in 0..32 { + let a: cubic_ext::Ext = [next(), next(), next()]; + assert_eq!(cubic_ext::power7(&a), pow(&a, 7)); + } +} + +/// The round-kind predicates partition `0..NUM_ROUNDS` exactly once each — the +/// schedule `FB E FB E FB E M`, said as a property rather than by reading it. +#[test] +fn the_round_schedule_is_fb_e_fb_e_fb_e_m() { + let kinds: Vec<&str> = (0..NUM_ROUNDS) + .map(|r| { + let k = [is_fb_round(r), is_ext_round(r), is_final_round(r)]; + assert_eq!( + k.iter().filter(|b| **b).count(), + 1, + "round {r} is {k:?}, which is not exactly one kind" + ); + if k[0] { + "FB" + } else if k[1] { + "E" + } else { + "M" + } + }) + .collect(); + assert_eq!(kinds, ["FB", "E", "FB", "E", "FB", "E", "M"]); +} + +/// The constants have the shape the permutation indexes them at. +#[test] +fn the_constant_tables_have_the_shape_the_rounds_index() { + assert_eq!(ARK1.len(), NUM_ROUNDS); + assert_eq!(ARK2.len(), NUM_ROUNDS); + assert!(ARK1.iter().all(|r| r.len() == STATE_FELTS)); + assert!(ARK2.iter().all(|r| r.len() == STATE_FELTS)); + assert_eq!(MDS_CIRC_ROW.len(), STATE_FELTS); + assert!( + ARK1.iter().chain(ARK2.iter()).flatten().all(|c| *c < P), + "every round constant must be canonical" + ); +} + +// ========================================================================= +// The byte / felt conventions +// ========================================================================= + +/// ⚠ The two leaf entry points must agree. Checked, not assumed: the trailing +/// partial group is zero-extended on the LOW side in both, which is easy to get +/// backwards. +#[test] +fn sponge_leaf_bytes_matches_the_felt_form() { + for len in [0usize, 1, 7, 8, 9, 15, 16, 17, 64, 65] { + let bytes: Vec = (0..len as u64).map(|i| (i * 37 + 11) as u8).collect(); + assert_eq!( + sponge_leaf_bytes(&bytes), + sponge_leaf(&felts_from_bytes(&bytes)), + "len {len}" + ); + } +} + +/// A digest survives the round trip through its 32 canonical big-endian bytes. +#[test] +fn a_digest_round_trips_through_its_commitment_bytes() { + let d: Digest = [fe(0), fe(1), fe(P - 1), fe(0x0123_4567_89ab_cdef)]; + assert_eq!(commitment_to_digest(&digest_to_commitment(&d)), d); +} + +/// ✓ The commitment bytes are BIG-endian — stated as a literal, because an +/// endianness flip round-trips perfectly and would pass the test above. +#[test] +fn the_commitment_bytes_are_big_endian() { + let d: Digest = [fe(1), fe(0), fe(0), fe(0)]; + let c = digest_to_commitment(&d); + assert_eq!( + c[7], 1, + "felt 0 = 1 must land in the LAST byte of its group" + ); + assert_eq!(c[0], 0); +} + +/// `element_felts` on a base element is the felt itself; on a cubic extension +/// element it is its three components in order. Pinned because the Merkle leaf +/// layout depends on it and `AsBytes` is a weaker contract than it looks. +#[test] +fn element_felts_decomposes_base_and_extension_the_same_way_the_stark_serialises() { + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + + let mut out = Vec::new(); + element_felts(&fe(12345), &mut out); + assert_eq!(out, alloc::vec![fe(12345)]); + + let mut out = Vec::new(); + let e = FieldElement::::new([fe(7), fe(8), fe(9)]); + element_felts(&e, &mut out); + assert_eq!(out, alloc::vec![fe(7), fe(8), fe(9)]); +} + +/// The empty leaf is the capacity's own rate lanes — zero — which is what the +/// device kernel's `finalize` returns when nothing was absorbed. +#[test] +fn the_empty_leaf_spends_no_permutation() { + assert_eq!(raw_digest(&sponge_leaf(&[])), [0, 0, 0, 0]); + assert_eq!(raw_digest(&sponge_leaf_bytes(&[])), [0, 0, 0, 0]); +} + +/// ⚠ The domain constants are pinned by the device kernel and the KAT header; +/// changing a VALUE forks the hash. Asserted as literals so the fork is a test +/// failure rather than a silent divergence. +#[test] +fn the_domain_values_are_the_ones_the_kernel_carries() { + assert_eq!(DOMAIN_COMPRESS, 0); + assert_eq!(DOMAIN_LEAF, 0x4C4D_464C, "rpx.cu carries 0x4C4D464C"); + assert_eq!(domain_iv(DOMAIN_LEAF), [0, DOMAIN_LEAF, 0, 0]); +} + +/// The `digest::Digest` adapter computes the leaf construction over its +/// buffered bytes, and resets. +#[test] +fn the_digest_adapter_is_the_leaf_construction() { + use digest::{Digest as _, FixedOutputReset, Update}; + + let msg: Vec = (0..40u8).collect(); + let want = digest_to_commitment(&sponge_leaf_bytes(&msg)); + + let mut d = Rpx256Digest::default(); + Update::update(&mut d, &msg[..17]); + Update::update(&mut d, &msg[17..]); + let got: [u8; 32] = d.clone().finalize().into(); + assert_eq!(got, want, "streamed in two pieces must equal one call"); + + let mut out = digest::Output::::default(); + FixedOutputReset::finalize_into_reset(&mut d, &mut out); + assert_eq!(<[u8; 32]>::from(out), want); + let empty: [u8; 32] = d.finalize().into(); + assert_eq!( + empty, + digest_to_commitment(&sponge_leaf_bytes(&[])), + "reset must clear the buffer" + ); +} diff --git a/crypto/crypto/src/hash_metrics.rs b/crypto/crypto/src/hash_metrics.rs index cc16e9348..8eecd2649 100644 --- a/crypto/crypto/src/hash_metrics.rs +++ b/crypto/crypto/src/hash_metrics.rs @@ -1,5 +1,13 @@ -//! Host-only keccak-hash counters for measuring the cost of VERIFYING a proof -//! (a proxy for the recursion guest's dominant work: keccak hashing). +//! Host-only hash counters for measuring the cost of VERIFYING a proof — a +//! proxy for a recursive verifier's dominant work, which is hashing. +//! +//! ⚠ **The counters follow the proof's CONFIGURATION, not one hash.** They were +//! keccak-only when keccak was the only hash on the multilinear path. Under +//! [`crate::hash::rpx`] every Merkle counter would then have read ZERO — a +//! measurement that cannot fail, reporting "no hashing" for the arm whose whole +//! purpose is to change the hashing. The algebraic backend bumps them through +//! [`count_merkle_direct`] / [`count_merkle_node_direct`] instead, which also +//! bump `total`, because unlike the byte backends nothing downstream will. //! //! Behind the `hash-metrics` cargo feature: a normal build keeps //! `PlatformKeccak256 = sha3::Keccak256` and every counter call compiles to @@ -34,6 +42,58 @@ pub struct Counts { pub absorb_calls: u64, /// Bytes fed through absorb (`Sum of data.len()`). pub absorb_bytes: u64, + /// ★★ Fiat-Shamir absorbs, ALL configurations. Counted in + /// `DefaultTranscript`'s own append methods, not in a hash. + pub transcript_absorbs: u64, + /// Of those, the ones whose sponge is keccak. + pub transcript_absorbs_keccak: u64, + /// Of those, the ones whose sponge is RPX256. + pub transcript_absorbs_rpx: u64, + /// ★★ Fiat-Shamir SQUEEZES, ALL configurations — `finalize_reset`, which + /// advances the chain (the output is re-absorbed). + pub transcript_squeezes: u64, + /// Of those, the ones whose sponge is keccak. + pub transcript_squeezes_keccak: u64, + /// Of those, the ones whose sponge is RPX256. + pub transcript_squeezes_rpx: u64, + /// ★★ Fiat-Shamir STATE reads, ALL configurations — `state()`, a finalize + /// on a CLONE. No reset and no re-absorb, so it does NOT advance the chain. + /// + /// ⚠ A DIFFERENT OPERATION from a squeeze, counted separately because + /// conflating them makes a number unfalsifiable. On a block proof the two + /// are 182,734 and 2,996 (lane V1's closed form, pinned to a measured + /// verify with difference 0): a counter hooked only to `finalize_reset` + /// misses every one of the 2,996, and a counter reporting their sum — + /// 185,730 — cannot be checked against either. + /// + /// ★ The state reads are a CONTROL that costs nothing: there is exactly one + /// per grind check, so this must equal the grind count the same run prints. + /// Two independent instruments on one quantity, and a disagreement names + /// which of them is wrong. + pub transcript_states: u64, + /// Of those, the ones whose sponge is keccak. + pub transcript_states_keccak: u64, + /// Of those, the ones whose sponge is RPX256. + pub transcript_states_rpx: u64, +} + +impl Counts { + /// Transcript work this build could not attribute to a known sponge. + /// + /// ★ Zero on every configuration that exists, and it is REPORTED rather + /// than assumed: the failure this whole group of counters exists to catch + /// is a hash nobody instrumented reading as a zero that looks like + /// "nothing ran". A third configuration arriving un-instrumented shows up + /// here instead of being silently folded into one of the two above. + pub fn transcript_unattributed(&self) -> (u64, u64, u64) { + ( + self.transcript_absorbs - self.transcript_absorbs_keccak - self.transcript_absorbs_rpx, + self.transcript_squeezes + - self.transcript_squeezes_keccak + - self.transcript_squeezes_rpx, + self.transcript_states - self.transcript_states_keccak - self.transcript_states_rpx, + ) + } } #[cfg(all(not(target_arch = "riscv64"), feature = "hash-metrics"))] @@ -47,6 +107,28 @@ mod imp { static GRINDING: AtomicU64 = AtomicU64::new(0); static ABSORB_CALLS: AtomicU64 = AtomicU64::new(0); static ABSORB_BYTES: AtomicU64 = AtomicU64::new(0); + static T_ABSORBS: AtomicU64 = AtomicU64::new(0); + static T_ABSORBS_KECCAK: AtomicU64 = AtomicU64::new(0); + static T_ABSORBS_RPX: AtomicU64 = AtomicU64::new(0); + static T_SQUEEZES: AtomicU64 = AtomicU64::new(0); + static T_SQUEEZES_KECCAK: AtomicU64 = AtomicU64::new(0); + static T_SQUEEZES_RPX: AtomicU64 = AtomicU64::new(0); + static T_STATES: AtomicU64 = AtomicU64::new(0); + static T_STATES_KECCAK: AtomicU64 = AtomicU64::new(0); + static T_STATES_RPX: AtomicU64 = AtomicU64::new(0); + + /// Which known sponge `D` is, if any: `Some(true)` keccak, `Some(false)` + /// RPX256, `None` a configuration nobody has instrumented. + fn sponge() -> Option { + let id = core::any::TypeId::of::(); + if id == core::any::TypeId::of::() { + Some(true) + } else if id == core::any::TypeId::of::() { + Some(false) + } else { + None + } + } /// Every keccak-256 finalize, from any site (host `PlatformKeccak256`). #[inline(always)] @@ -59,6 +141,11 @@ mod imp { /// [`count_total`]. This keeps `merkle` a strict subset of `total` for ANY /// `D` (a non-keccak backend, as in the crypto tests, does not go through the /// counted wrapper, so counting it here would let `merkle` exceed `total`). + /// + /// A hash that does not route its Merkle work through a `digest::Digest` at + /// all — the algebraic backend sponges felts directly — uses + /// [`count_merkle_direct`] instead, which keeps the same invariant by + /// bumping both counters itself. #[inline(always)] pub fn count_merkle() { if core::any::TypeId::of::() @@ -95,6 +182,81 @@ mod imp { ABSORB_BYTES.fetch_add(nbytes as u64, Ordering::Relaxed); } + /// ★ A Merkle LEAF finalize by a hash whose Merkle work does not pass + /// through a `digest::Digest` — the algebraic backend, which sponges felts + /// directly and never builds a digest object. + /// + /// Bumps `total` as well as `merkle`, because nothing downstream will: for + /// the byte backends `total` comes from the digest's own `finalize`, and + /// there is no such call here. Doing both in one function is what keeps + /// `merkle ⊆ total` true by construction rather than by two call sites + /// agreeing. + #[inline(always)] + pub fn count_merkle_direct() { + TOTAL.fetch_add(1, Ordering::Relaxed); + MERKLE.fetch_add(1, Ordering::Relaxed); + } + + /// ★ A Merkle PARENT by such a hash. Bumps `total`, `merkle` and + /// `merkle_nodes`, so `merkle - merkle_nodes` is the leaf count on this path + /// exactly as it is on the byte path. + #[inline(always)] + pub fn count_merkle_node_direct() { + TOTAL.fetch_add(1, Ordering::Relaxed); + MERKLE.fetch_add(1, Ordering::Relaxed); + MERKLE_NODES.fetch_add(1, Ordering::Relaxed); + } + + /// ★★ One Fiat-Shamir ABSORB, tagged by the sponge that will consume it. + /// + /// Called from `DefaultTranscript`'s append methods — the transcript, not + /// the hash. Two reasons, and the second is the one that matters: + /// + /// 1. It is hash-agnostic by construction. A counter living inside keccak + /// reads ZERO for an algebraic transcript, which is indistinguishable + /// from "no transcript ran" — the trap this module's header describes + /// for Merkle, which the transcript never got. + /// 2. It counts TRANSCRIPT absorbs only. [`count_absorb`] is bumped from a + /// digest's `update`, so it mixes Merkle leaf bytes with Fiat-Shamir + /// bytes and cannot answer "how much did the transcript absorb" for + /// either hash. + #[inline(always)] + pub fn count_transcript_absorb() { + T_ABSORBS.fetch_add(1, Ordering::Relaxed); + match sponge::() { + Some(true) => T_ABSORBS_KECCAK.fetch_add(1, Ordering::Relaxed), + Some(false) => T_ABSORBS_RPX.fetch_add(1, Ordering::Relaxed), + None => 0, + }; + } + + /// ★★ One Fiat-Shamir SQUEEZE, tagged the same way. + #[inline(always)] + pub fn count_transcript_squeeze() { + T_SQUEEZES.fetch_add(1, Ordering::Relaxed); + match sponge::() { + Some(true) => T_SQUEEZES_KECCAK.fetch_add(1, Ordering::Relaxed), + Some(false) => T_SQUEEZES_RPX.fetch_add(1, Ordering::Relaxed), + None => 0, + }; + } + + /// ★★ One `state()` — a finalize on a CLONE of the sponge. + /// + /// Not a squeeze: no reset, no re-absorb, the chain does not advance. It is + /// its own counter because the two are different operations and a sum + /// cannot be checked against either. One of these per grind check, which is + /// what makes it a free control against the grind count. + #[inline(always)] + pub fn count_transcript_state() { + T_STATES.fetch_add(1, Ordering::Relaxed); + match sponge::() { + Some(true) => T_STATES_KECCAK.fetch_add(1, Ordering::Relaxed), + Some(false) => T_STATES_RPX.fetch_add(1, Ordering::Relaxed), + None => 0, + }; + } + /// Zero all counters. pub fn reset() { TOTAL.store(0, Ordering::Relaxed); @@ -103,6 +265,15 @@ mod imp { GRINDING.store(0, Ordering::Relaxed); ABSORB_CALLS.store(0, Ordering::Relaxed); ABSORB_BYTES.store(0, Ordering::Relaxed); + T_ABSORBS.store(0, Ordering::Relaxed); + T_ABSORBS_KECCAK.store(0, Ordering::Relaxed); + T_ABSORBS_RPX.store(0, Ordering::Relaxed); + T_SQUEEZES.store(0, Ordering::Relaxed); + T_SQUEEZES_KECCAK.store(0, Ordering::Relaxed); + T_SQUEEZES_RPX.store(0, Ordering::Relaxed); + T_STATES.store(0, Ordering::Relaxed); + T_STATES_KECCAK.store(0, Ordering::Relaxed); + T_STATES_RPX.store(0, Ordering::Relaxed); } pub fn snapshot() -> Counts { @@ -113,6 +284,15 @@ mod imp { grinding: GRINDING.load(Ordering::Relaxed), absorb_calls: ABSORB_CALLS.load(Ordering::Relaxed), absorb_bytes: ABSORB_BYTES.load(Ordering::Relaxed), + transcript_absorbs: T_ABSORBS.load(Ordering::Relaxed), + transcript_absorbs_keccak: T_ABSORBS_KECCAK.load(Ordering::Relaxed), + transcript_absorbs_rpx: T_ABSORBS_RPX.load(Ordering::Relaxed), + transcript_squeezes: T_SQUEEZES.load(Ordering::Relaxed), + transcript_squeezes_keccak: T_SQUEEZES_KECCAK.load(Ordering::Relaxed), + transcript_squeezes_rpx: T_SQUEEZES_RPX.load(Ordering::Relaxed), + transcript_states: T_STATES.load(Ordering::Relaxed), + transcript_states_keccak: T_STATES_KECCAK.load(Ordering::Relaxed), + transcript_states_rpx: T_STATES_RPX.load(Ordering::Relaxed), } } } @@ -131,7 +311,17 @@ mod imp { #[inline(always)] pub fn count_grinding() {} #[inline(always)] + pub fn count_merkle_direct() {} + #[inline(always)] + pub fn count_merkle_node_direct() {} + #[inline(always)] pub fn count_absorb(_nbytes: usize) {} + #[inline(always)] + pub fn count_transcript_absorb() {} + #[inline(always)] + pub fn count_transcript_squeeze() {} + #[inline(always)] + pub fn count_transcript_state() {} pub fn reset() {} pub fn snapshot() -> Counts { Counts::default() @@ -139,5 +329,7 @@ mod imp { } pub use imp::{ - count_absorb, count_grinding, count_merkle, count_merkle_node, count_total, reset, snapshot, + count_absorb, count_grinding, count_merkle, count_merkle_direct, count_merkle_node, + count_merkle_node_direct, count_total, count_transcript_absorb, count_transcript_squeeze, + count_transcript_state, reset, snapshot, }; diff --git a/crypto/crypto/src/merkle_tree/backends/mod.rs b/crypto/crypto/src/merkle_tree/backends/mod.rs index 431e6597b..611e95d9e 100644 --- a/crypto/crypto/src/merkle_tree/backends/mod.rs +++ b/crypto/crypto/src/merkle_tree/backends/mod.rs @@ -1,5 +1,6 @@ pub mod field_element; pub mod field_element_vector; +pub mod rpx; /// Configurations for merkle trees /// Setting generics to some value pub mod types; diff --git a/crypto/crypto/src/merkle_tree/backends/rpx.rs b/crypto/crypto/src/merkle_tree/backends/rpx.rs new file mode 100644 index 000000000..c07297cb9 --- /dev/null +++ b/crypto/crypto/src/merkle_tree/backends/rpx.rs @@ -0,0 +1,260 @@ +//! The RPX256 Merkle backend — the algebraic sibling of +//! [`FieldElementVectorBackend`](super::field_element_vector::FieldElementVectorBackend). +//! +//! A sibling type rather than a reparameterisation of the byte backend, for the +//! reason the per-table branch gives: `FieldElementVectorBackend` is built +//! around a `digest::Digest` fed a byte stream, while this hashes FELTS with a +//! rate-8 overwrite duplex. Routing felts through bytes and back would work and +//! would be slower and less obvious; keeping them apart is what leaves the +//! keccak path untouched by this work. +//! +//! # What a leaf and a parent are +//! +//! **A leaf** is [`sponge_leaf`](crate::hash::rpx::sponge_leaf) over the felt +//! sequence the leaf's elements decompose to, in order — eight fresh felts per +//! permutation, capacity lane 0 the padding flag `len mod 8`, lane 1 the LEAF +//! domain. +//! +//! **A parent** is [`compress`](crate::hash::rpx::compress) — ONE permutation of +//! `[left ‖ right ‖ 0⁴]` at the zero domain, which makes it literally +//! `Rpx256::merge` and externally checkable against miden. +//! +//! **A node** is four canonical felts as 32 big-endian bytes, so +//! `IsMerkleTreeBackend::Node` is the same `[u8; 32]` every other backend in +//! this crate uses and no proof type changes width. +//! +//! # ⚠ The one contract a caller has to know +//! +//! `hash_data` on `&[a, b]` must equal what a FRI-style pair backend would +//! compute for the pair `[a, b]`, because the univariate prover commits layers +//! one way and verifies them the other. That invariant is why there is a single +//! implementation here rather than a "batched" and a "pair" one that could be +//! edited apart: both shapes go through [`hash_data_from_slices`], so there are +//! not two encodings to be shown equal. `tests::a_pair_leaf_is_the_two_element_vector` +//! pins it anyway, because "holds by construction" is a claim about today's +//! code. + +use core::marker::PhantomData; + +use alloc::vec::Vec; +use math::{ + field::{element::FieldElement, traits::IsField}, + traits::AsBytes, +}; + +use crate::hash::rpx::{ + Fp, commitment_to_digest, compress, digest_to_commitment, element_felts, felts_from_bytes, + sponge_leaf, +}; +use crate::merkle_tree::traits::IsMerkleTreeBackend; + +/// RPX256 over vectors of field elements. +#[derive(Clone, Debug)] +pub struct RpxVectorBackend { + /// `fn() -> F` rather than `F`, so the marker is unconditionally `Send` and + /// `Sync` without an `unsafe impl`: a real epoch's base layer has millions + /// of leaves, hashed in parallel. + _marker: PhantomData F>, +} + +impl Default for RpxVectorBackend { + fn default() -> Self { + Self { + _marker: PhantomData, + } + } +} + +impl RpxVectorBackend +where + F: IsField, + FieldElement: AsBytes, +{ + /// Leaf-hash the concatenation `a ‖ b` without materialising it. + /// + /// The single source of truth for the leaf's felt sequence: a plain leaf is + /// the concatenation with an empty second slice, so the two shapes cannot + /// disagree. + pub fn hash_data_from_slices(a: &[FieldElement], b: &[FieldElement]) -> [u8; 32] { + // Metric: a Merkle leaf finalize. `_direct` because this path builds no + // `digest::Digest`, so it owes `total` as well — see `crate::hash_metrics`. + crate::hash_metrics::count_merkle_direct(); + let mut felts: Vec = Vec::with_capacity(a.len() + b.len()); + for e in a.iter().chain(b.iter()) { + element_felts(e, &mut felts); + } + digest_to_commitment(&sponge_leaf(&felts)) + } + + /// Leaf-hash a byte buffer, rebuilding the felts it encodes. + /// + /// ⚠ Must equal [`hash_data`](IsMerkleTreeBackend::hash_data) on the + /// elements those bytes encode — the one place an algebraic backend can + /// silently disagree with itself, because the byte route has to rebuild + /// what the felt route was handed. + /// `tests::hash_bytes_agrees_with_hash_data` is the gate. + pub fn hash_bytes(data: &[u8]) -> [u8; 32] { + // Metric: a Merkle leaf finalize, as `hash_data_from_slices` is — the + // two must agree on the digest, so they must agree on the count. + crate::hash_metrics::count_merkle_direct(); + digest_to_commitment(&sponge_leaf(&felts_from_bytes(data))) + } +} + +impl IsMerkleTreeBackend for RpxVectorBackend +where + F: IsField, + FieldElement: AsBytes + Sync + Send, + Vec>: Sync + Send, +{ + type Node = [u8; 32]; + type Data = Vec>; + + fn hash_data(input: &Vec>) -> [u8; 32] { + Self::hash_data_from_slices(input, &[]) + } + + fn hash_new_parent(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] { + // Metric: a Merkle parent. One call, not a leaf count plus a node count: + // this path does not flow through the leaf helper the way the byte + // backend's parent flows through `hash_streamed`. + crate::hash_metrics::count_merkle_node_direct(); + digest_to_commitment(&compress( + &commitment_to_digest(left), + &commitment_to_digest(right), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Base; + + type B = RpxVectorBackend; + type E = RpxVectorBackend; + + fn base(n: usize) -> Vec> { + (0..n as u64) + .map(|i| FieldElement::from(i * 7 + 1)) + .collect() + } + + fn ext(n: usize) -> Vec> { + (0..n as u64) + .map(|i| { + FieldElement::::new([ + FieldElement::from(i + 1), + FieldElement::from(i + 2), + FieldElement::from(i + 3), + ]) + }) + .collect() + } + + /// ★★ The `hash_bytes` / `hash_data` contract, on both fields and at the + /// lengths the padding rule distinguishes. + #[test] + fn hash_bytes_agrees_with_hash_data() { + for n in [0usize, 1, 7, 8, 9, 16, 17] { + let leaf = base(n); + let mut bytes = Vec::new(); + for e in &leaf { + e.stream_bytes(&mut |b| bytes.extend_from_slice(b)); + } + assert_eq!(B::hash_bytes(&bytes), B::hash_data(&leaf), "base, n = {n}"); + + let leaf = ext(n); + let mut bytes = Vec::new(); + for e in &leaf { + e.stream_bytes(&mut |b| bytes.extend_from_slice(b)); + } + assert_eq!(E::hash_bytes(&bytes), E::hash_data(&leaf), "ext3, n = {n}"); + } + } + + /// ⚠ The invariant the univariate FRI path depends on: a two-element leaf + /// is a pair. + #[test] + fn a_pair_leaf_is_the_two_element_vector() { + let a = FieldElement::::from(11u64); + let b = FieldElement::::from(22u64); + assert_eq!( + B::hash_data(&alloc::vec![a, b]), + B::hash_data_from_slices(&[a], &[b]) + ); + } + + /// ✓ A leaf is order-sensitive and length-sensitive — so the equalities + /// above are not equalities between constants. + #[test] + fn a_leaf_depends_on_the_order_and_the_length_of_its_elements() { + let leaf = base(5); + let mut swapped = leaf.clone(); + swapped.swap(0, 1); + assert_ne!(B::hash_data(&leaf), B::hash_data(&swapped)); + + let mut longer = leaf.clone(); + longer.push(FieldElement::from(0u64)); + assert_ne!( + B::hash_data(&leaf), + B::hash_data(&longer), + "a trailing zero must not be invisible" + ); + } + + /// ✓ An exact rate multiple spends no trailing permutation, so the eighth + /// and ninth felts are not interchangeable at the block boundary. + #[test] + fn the_block_boundary_is_not_a_collision() { + assert_ne!(B::hash_data(&base(8)), B::hash_data(&base(9))); + assert_ne!(B::hash_data(&base(16)), B::hash_data(&base(17))); + } + + /// ✓ A parent is order-sensitive. + #[test] + fn a_parent_depends_on_the_order_of_its_children() { + let l = B::hash_data(&base(3)); + let r = B::hash_data(&base(4)); + assert_ne!(B::hash_new_parent(&l, &r), B::hash_new_parent(&r, &l)); + } + + /// ✓ A parent is NOT a leaf of the eight felts its children hold: the + /// domains differ, which is the whole point of the capacity tag. + #[test] + fn a_parent_is_domain_separated_from_a_leaf() { + use crate::hash::rpx::{commitment_to_digest, sponge_leaf}; + + let l = B::hash_data(&base(3)); + let r = B::hash_data(&base(4)); + let parent = B::hash_new_parent(&l, &r); + + let mut felts = Vec::new(); + felts.extend_from_slice(&commitment_to_digest(&l)); + felts.extend_from_slice(&commitment_to_digest(&r)); + let as_leaf = digest_to_commitment(&sponge_leaf(&felts)); + + assert_ne!( + parent, as_leaf, + "the LEAF domain must separate a leaf from a parent over the same felts" + ); + } + + /// ✓ An extension leaf decomposes to three felts per element — checked + /// against the felt sequence rather than against another backend call, so a + /// decomposition that dropped a component would show. + #[test] + fn an_extension_leaf_absorbs_three_felts_per_element() { + let leaf = ext(4); + let mut felts = Vec::new(); + for e in &leaf { + element_felts(e, &mut felts); + } + assert_eq!(felts.len(), 12, "four ext3 elements are twelve felts"); + assert_eq!( + E::hash_data(&leaf), + digest_to_commitment(&sponge_leaf(&felts)) + ); + } +} diff --git a/crypto/crypto/src/merkle_tree/backends/types.rs b/crypto/crypto/src/merkle_tree/backends/types.rs index 2384fda3a..f3c6b6181 100644 --- a/crypto/crypto/src/merkle_tree/backends/types.rs +++ b/crypto/crypto/src/merkle_tree/backends/types.rs @@ -13,3 +13,9 @@ pub type BatchKeccak256Backend = FieldElementVectorBackend; // Fixed-size pair backends (more efficient for FRI layers) pub type PairKeccak256Backend = FieldElementPairBackend; + +/// RPX256 over a vector of field elements — the algebraic backend, for a proof +/// that is going to be verified inside another proof. See +/// [`crate::hash::rpx`] for why an algebraic hash is worth its host cost, and +/// only there. +pub type BatchRpx256Backend = super::rpx::RpxVectorBackend; diff --git a/crypto/crypto/src/tests/grinding_determinism_tests.rs b/crypto/crypto/src/tests/grinding_determinism_tests.rs new file mode 100644 index 000000000..cbf477a30 --- /dev/null +++ b/crypto/crypto/src/tests/grinding_determinism_tests.rs @@ -0,0 +1,129 @@ +//! ★ The reproducible nonce search, and the reason it had to exist. +//! +//! `generate_nonce` returns *a* valid nonce. Which one is not a contract, and +//! under `parallel` it is rayon's `find_any` — whichever worker got there +//! first. That would be harmless if the nonce were merely recorded, but it is +//! **absorbed into the transcript**, so every challenge drawn after the first +//! grind depends on it. Two honest runs of the same prover therefore produce +//! different Merkle roots, different out-of-domain values and different +//! openings, and no normalisation of the nonce fields can undo that, because +//! the divergence is not in those fields. +//! +//! [`generate_nonce_smallest`] is the fix a byte gate needs: the smallest valid +//! nonce is a function of the seed and the factor alone. These tests pin the +//! three things that makes it worth anything — it is reproducible, it really is +//! the smallest, and it is valid — and they are written so each can fail. + +use digest::Digest; + +use crate::grinding::{generate_nonce, generate_nonce_smallest, is_valid_nonce}; +use crate::hash::platform_keccak::PlatformKeccak256 as Keccak; + +/// A factor small enough that the exhaustive scan below is instant and large +/// enough that the answer is not zero on most seeds. +const FACTOR: u8 = 12; + +fn seed(tag: u8) -> [u8; 32] { + let mut out = [0u8; 32]; + out[0] = tag; + for (i, b) in out.iter_mut().enumerate().skip(1) { + *b = (i as u8).wrapping_mul(37).wrapping_add(tag); + } + out +} + +/// It is reproducible: the same inputs give the same nonce. +#[test] +fn the_smallest_nonce_is_reproducible() { + for tag in 0..4u8 { + let s = seed(tag); + let a = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + let b = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + assert_eq!(a, b, "seed {tag}: the smallest nonce must not vary"); + } +} + +/// ★ It really is the smallest — checked against an exhaustive scan, which is a +/// different algorithm from the one under test. +/// +/// This is the assertion that can fail if `find_first` is ever swapped back to +/// `find_any` for speed, which is exactly the regression the deterministic knob +/// exists to prevent. +#[test] +fn it_is_the_smallest_valid_nonce_and_not_merely_a_valid_one() { + // The exhaustive loop below is vacuous when the answer is zero, so at least + // one seed has to land above it for the test to be testing anything. + let mut scanned = 0u64; + for tag in 0..4u8 { + let s = seed(tag); + let n = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + scanned += n; + + assert!( + is_valid_nonce::(&s, n, FACTOR), + "seed {tag}: the chosen nonce {n} does not pass the verifier's own check" + ); + assert!( + (0..n).all(|candidate| !is_valid_nonce::(&s, candidate, FACTOR)), + "seed {tag}: a nonce below {n} is also valid, so {n} is not the smallest" + ); + } + assert!( + scanned > 0, + "every seed's smallest nonce was zero, so the minimality scan never ran" + ); +} + +/// The unpinned search is still correct — it just promises less. +/// +/// Stated as "valid, and never below the smallest" rather than "different": +/// asserting a difference would be a coin flip, and a test that fails at random +/// teaches nothing. +#[test] +fn the_unpinned_search_returns_a_valid_nonce_no_smaller_than_the_smallest() { + for tag in 0..4u8 { + let s = seed(tag); + let smallest = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + let any = generate_nonce::(&s, FACTOR).expect("a nonce exists"); + + assert!( + is_valid_nonce::(&s, any, FACTOR), + "seed {tag}: the unpinned search returned an invalid nonce" + ); + assert!( + any >= smallest, + "seed {tag}: {any} is below the exhaustively-checked smallest {smallest}" + ); + } +} + +/// The construction itself, against the spec in the module doc: the outer hash +/// of `inner ‖ nonce` must have `FACTOR` leading zero bits. +/// +/// An independent reading of the same predicate — `is_valid_nonce` compares a +/// big-endian `u64` against a limit; this counts the bits — so the two cannot +/// be one transcription of the other. +#[test] +fn a_valid_nonce_really_does_have_the_leading_zeros() { + let s = seed(1); + let n = generate_nonce_smallest::(&s, FACTOR).expect("a nonce exists"); + + // Rebuild the inner hash the way the module documents it. + const PREFIX: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xed]; + let mut inner_data = [0u8; 41]; + inner_data[0..8].copy_from_slice(&PREFIX); + inner_data[8..40].copy_from_slice(&s); + inner_data[40] = FACTOR; + let inner = Keccak::digest(inner_data); + + let mut outer_data = [0u8; 40]; + outer_data[..32].copy_from_slice(&inner); + outer_data[32..].copy_from_slice(&n.to_be_bytes()); + let outer = Keccak::digest(outer_data); + + let leading = u64::from_be_bytes(outer[..8].try_into().unwrap()).leading_zeros(); + assert!( + leading >= FACTOR as u32, + "nonce {n} gives only {leading} leading zero bits, needed {FACTOR}" + ); +} diff --git a/crypto/crypto/src/tests/hash_metrics_tests.rs b/crypto/crypto/src/tests/hash_metrics_tests.rs new file mode 100644 index 000000000..2b82dedb1 --- /dev/null +++ b/crypto/crypto/src/tests/hash_metrics_tests.rs @@ -0,0 +1,148 @@ +//! ★ The verify-hash counters count BOTH hash families — the test that would +//! have caught the false zero. +//! +//! `hash_metrics` was keccak-only: `count_merkle` compared `TypeId::of::()` +//! against `PlatformKeccak256` and did nothing otherwise. That was correct while +//! keccak was the only hash on the multilinear path, and it became a check that +//! cannot fail the moment a second one arrived — under RPX every Merkle counter +//! would have read ZERO, reporting "no hashing" for precisely the arm whose +//! purpose is to change the hashing, and a reader comparing the two arms would +//! have concluded the algebraic hash was free. +//! +//! So these tests are written the way that failure would have been caught: +//! **non-zero after the RPX arm, non-zero after the keccak arm, and the subset +//! invariant intact for both.** Every one of them fails on the pre-extension +//! code. +//! +//! ⚠ These run only under `--features hash-metrics`. That is the same reason the +//! module exists at all — the counters compile to nothing otherwise — but it +//! does mean a default `cargo test` does not execute them. `make lint`'s passes +//! do not enable the feature either; the gate is `cargo test -p crypto +//! --features hash-metrics`. + +#![cfg(feature = "hash-metrics")] + +use alloc::vec::Vec; +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as Fp; + +use crate::hash_metrics::{Counts, reset, snapshot}; +use crate::merkle_tree::backends::rpx::RpxVectorBackend; +use crate::merkle_tree::backends::types::BatchKeccak256Backend; +use crate::merkle_tree::traits::IsMerkleTreeBackend; + +type Rpx = RpxVectorBackend; +type Keccak = BatchKeccak256Backend; + +/// The counters are process-global, so the cases take turns rather than +/// running concurrently. A mutex rather than `--test-threads=1`, so the +/// property does not depend on how the suite is invoked. +/// +/// ⚠ Poisoning is IGNORED, and that is not laziness. A failing case panics +/// while holding this lock, and `unwrap()` would then panic every later case on +/// the poisoned mutex — turning one real failure into five, four of them +/// cascades. That was observed while mutation-testing this file: removing the +/// RPX backend's counter calls failed the two cases that assert it AND the +/// keccak case, which is a lie about the keccak path. The counters are reset at +/// the top of every measurement, so a poisoned lock carries no stale state. +static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serialise() -> std::sync::MutexGuard<'static, ()> { + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +fn leaf(n: usize) -> Vec> { + (0..n as u64) + .map(|i| FieldElement::from(i * 7 + 1)) + .collect() +} + +/// Hash one leaf and one parent under `B`, and report what the counters saw. +fn measure() -> Counts +where + B: IsMerkleTreeBackend>>, +{ + reset(); + let a = B::hash_data(&leaf(5)); + let b = B::hash_data(&leaf(6)); + let _ = B::hash_new_parent(&a, &b); + snapshot() +} + +/// ★★ The RPX arm is COUNTED — the proposition the extension exists for. +#[test] +fn the_algebraic_backend_is_counted() { + let _g = serialise(); + let c = measure::(); + + assert_eq!(c.merkle, 3, "two leaves and one parent"); + assert_eq!(c.merkle_nodes, 1, "one parent"); + assert_eq!( + c.merkle - c.merkle_nodes, + 2, + "merkle - nodes must be the leaf count, as it is on the byte path" + ); + assert!( + c.total >= c.merkle, + "total {} must cover merkle {}", + c.total, + c.merkle + ); +} + +/// The keccak arm still is, unchanged — so the extension did not move the +/// number the existing instrument reports. +#[test] +fn the_byte_backend_is_still_counted() { + let _g = serialise(); + let c = measure::(); + + assert_eq!(c.merkle, 3, "two leaves and one parent"); + assert_eq!(c.merkle_nodes, 1, "one parent"); + assert!(c.total >= c.merkle); +} + +/// ★ The two arms agree on the COUNT while differing in the hash — which is +/// what makes a cross-arm comparison of these numbers meaningful at all. +/// +/// If they disagreed, a difference in the counters would not distinguish "this +/// hash does more work" from "this backend is instrumented differently". +#[test] +fn the_two_backends_report_the_same_shape_for_the_same_tree() { + let _g = serialise(); + let rpx = measure::(); + let keccak = measure::(); + + assert_eq!(rpx.merkle, keccak.merkle); + assert_eq!(rpx.merkle_nodes, keccak.merkle_nodes); +} + +/// ⚠ **The subset invariant, on the path that could break it.** `merkle` must +/// never exceed `total`: the byte backends get `total` from the digest's own +/// `finalize`, and the algebraic backend has no digest to finalize, so it owes +/// the bump itself. A `count_merkle_direct` that forgot `TOTAL` would land here. +#[test] +fn merkle_never_exceeds_total_on_the_algebraic_path() { + let _g = serialise(); + reset(); + for n in 0..16 { + let _ = Rpx::hash_data(&leaf(n)); + } + let c = snapshot(); + assert_eq!(c.merkle, 16); + assert!( + c.total >= c.merkle, + "total {} < merkle {} — the algebraic path did not bump total", + c.total, + c.merkle + ); +} + +/// ✓ `reset` really resets, so one case cannot read another's counts. +#[test] +fn reset_clears_every_counter() { + let _g = serialise(); + let _ = Rpx::hash_data(&leaf(4)); + reset(); + assert_eq!(snapshot(), Counts::default()); +} diff --git a/crypto/crypto/src/tests/mod.rs b/crypto/crypto/src/tests/mod.rs index 96bf36e92..3d7be7381 100644 --- a/crypto/crypto/src/tests/mod.rs +++ b/crypto/crypto/src/tests/mod.rs @@ -1,6 +1,10 @@ pub mod default_transcript_tests; pub mod field_element_tests; pub mod field_element_vector_tests; +pub mod grinding_determinism_tests; +pub mod hash_metrics_tests; pub mod merkle_proof_tests; pub mod merkle_tests; pub mod merkle_utils_tests; +pub mod rpx_grind_tests; +pub mod rpx_transcript_tests; diff --git a/crypto/crypto/src/tests/rpx_grind_tests.rs b/crypto/crypto/src/tests/rpx_grind_tests.rs new file mode 100644 index 000000000..be3ca5798 --- /dev/null +++ b/crypto/crypto/src/tests/rpx_grind_tests.rs @@ -0,0 +1,185 @@ +//! ★ The RPX grind's host/device contract, and the endianness that decides it. +//! +//! The device search takes the 32-byte inner hash as four `u64`s, and the two +//! arms read those bytes in OPPOSITE orders — little-endian lanes for keccak, +//! big-endian felts for RPX. Crossing them compiles, runs, and searches for a +//! nonce under a message the host never hashes; the host check then rejects +//! every nonce the device returns and the prover falls back to the CPU forever, +//! which is a performance cliff with no error attached to it. +//! +//! So the mapping is pinned here against the ORACLE TABLE the CUDA kernel is +//! itself checked against — `RPX_GRIND_VECTORS` in +//! `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h`, three rows printed by +//! the per-table branch's host implementation. Each row carries the four +//! big-endian `inner_felts`, the smallest valid nonce, and `le_nonce`: what the +//! same kernel answers on the LITTLE-endian reading, which the header records as +//! `u64::MAX` — nothing found. That last column is the endianness control, and +//! it is why these tests need no GPU: they check the HOST side of an agreement +//! whose device side is pinned to the same numbers. + +use alloc::vec::Vec; +use digest::Digest; + +use crate::grinding::{inner_hash_felts, inner_hash_lanes, is_valid_nonce}; +use crate::hash::rpx::Rpx256Digest; + +/// One row of `RPX_GRIND_VECTORS`: the seed byte (repeated 32 times), the +/// grinding factor, the four BIG-endian inner felts, and the smallest valid +/// nonce. +const RPX_GRIND_VECTORS: [(u8, u8, [u64; 4], u64); 3] = [ + ( + 90, + 12, + [ + 17047917526726690733, + 2027278666509702433, + 4678289907902145381, + 4242003890993108442, + ], + 1342, + ), + ( + 17, + 13, + [ + 3807340077325453675, + 129745844021573959, + 15014385560057355003, + 944573484564438641, + ], + 300, + ), + ( + 32, + 14, + [ + 5597071933014793605, + 8702110216523445336, + 2882478612521280078, + 9429844132731097150, + ], + 705, + ), +]; + +fn seed_of(byte: u8) -> [u8; 32] { + [byte; 32] +} + +/// ★★ `inner_hash_felts` reproduces the oracle's four felts, on every row. +/// +/// This is the mapping itself: the four `u64`s the device search is handed. +/// Nothing in this repository produced these numbers — they are the per-table +/// branch's host implementation, and the CUDA kernel is checked against the +/// same table. +#[test] +fn the_inner_felts_match_the_device_oracle_table() { + for (byte, factor, want, _) in RPX_GRIND_VECTORS { + let got = inner_hash_felts::(&seed_of(byte), factor); + assert_eq!(got, want, "seed 0x{byte:02x}, factor {factor}"); + } +} + +/// ★ And the oracle's nonce passes the HOST predicate — the agreement the +/// device dispatch rests on, checked without a device. +#[test] +fn the_oracle_nonce_passes_the_host_predicate() { + for (byte, factor, _, nonce) in RPX_GRIND_VECTORS { + assert!( + is_valid_nonce::(&seed_of(byte), nonce, factor), + "seed 0x{byte:02x}, factor {factor}: nonce {nonce} rejected" + ); + } +} + +/// ★ …and it really is the SMALLEST, by exhaustive scan — a different algorithm +/// from the one that produced it. +#[test] +fn the_oracle_nonce_is_the_smallest_valid_one() { + for (byte, factor, _, nonce) in RPX_GRIND_VECTORS { + let seed = seed_of(byte); + assert!(nonce > 0, "the scan below is vacuous at nonce 0"); + assert!( + (0..nonce).all(|n| !is_valid_nonce::(&seed, n, factor)), + "seed 0x{byte:02x}, factor {factor}: a nonce below {nonce} is also valid" + ); + } +} + +/// ⚠⚠ **THE ENDIANNESS CONTROL.** The little-endian reading of the same inner +/// hash is a DIFFERENT message. +/// +/// Without this, `inner_hash_felts` could be `inner_hash_lanes` with a new name +/// and every test above would still pass — they would simply all be about the +/// wrong four `u64`s together. The header records `le_nonce = u64::MAX` for all +/// three rows: on the little-endian reading the kernel finds nothing at all in +/// its scanned block. +#[test] +fn the_little_endian_reading_is_a_different_message() { + for (byte, factor, felts, _) in RPX_GRIND_VECTORS { + let seed = seed_of(byte); + let lanes = inner_hash_lanes::(&seed, factor); + assert_ne!( + lanes, felts, + "seed 0x{byte:02x}: the two readings must differ, or there is nothing to get wrong" + ); + } +} + +/// ✓ The two readings are byte-reversals of each other, lane for lane — so the +/// difference above is exactly the endianness and not a hash that moved. +#[test] +fn the_two_readings_are_byte_reversals_of_one_another() { + for (byte, factor, _, _) in RPX_GRIND_VECTORS { + let seed = seed_of(byte); + let lanes = inner_hash_lanes::(&seed, factor); + let felts = inner_hash_felts::(&seed, factor); + for (i, (l, f)) in lanes.iter().zip(&felts).enumerate() { + assert_eq!(l.swap_bytes(), *f, "lane {i} of seed 0x{byte:02x}"); + } + } +} + +/// ★ The preimage is ONE rate-8 block, which is what makes the kernel's +/// `init(5)` right: `inner_hash ‖ nonce` is 40 bytes, five felts, padding flag +/// `5 mod 8 = 5`. +/// +/// Checked by computing the digest the long way — felts in, sponge out — and +/// requiring it to equal what the production predicate hashes from bytes. +#[test] +fn the_grind_preimage_is_five_felts_in_one_block() { + use crate::hash::rpx::{Fp, RATE_FELTS, digest_to_commitment, sponge_leaf}; + + for (byte, factor, felts, nonce) in RPX_GRIND_VECTORS { + // What the kernel absorbs: the four inner felts, then the nonce. + let mut block: Vec = felts.iter().map(|v| Fp::from(*v)).collect(); + block.push(Fp::from(nonce)); + assert_eq!(block.len(), 5, "the grind preimage is five felts"); + assert!(block.len() <= RATE_FELTS, "…and therefore one rate block"); + + let by_felts = digest_to_commitment(&sponge_leaf(&block)); + + // What the host predicate hashes: the 32 inner bytes then the nonce, + // big-endian, through the production digest. + let mut inner = [0u8; 32]; + for (i, v) in felts.iter().enumerate() { + inner[i * 8..(i + 1) * 8].copy_from_slice(&v.to_be_bytes()); + } + let mut data = [0u8; 40]; + data[..32].copy_from_slice(&inner); + data[32..].copy_from_slice(&nonce.to_be_bytes()); + let by_bytes: [u8; 32] = Rpx256Digest::digest(data).into(); + + assert_eq!( + by_felts, by_bytes, + "seed 0x{byte:02x}: the felt form and the byte form must be one hash" + ); + + // And that digest's leading u64 is what `limit` is compared against. + let head = u64::from_be_bytes(by_bytes[..8].try_into().unwrap()); + assert!( + head < 1u64 << (64 - factor), + "seed 0x{byte:02x}: the oracle nonce must clear its own limit" + ); + } +} diff --git a/crypto/crypto/src/tests/rpx_transcript_tests.rs b/crypto/crypto/src/tests/rpx_transcript_tests.rs new file mode 100644 index 000000000..bef46cbca --- /dev/null +++ b/crypto/crypto/src/tests/rpx_transcript_tests.rs @@ -0,0 +1,435 @@ +//! ★★ W1-A — the RPX transcript hands out canonical felts, and the keccak one +//! is untouched. +//! +//! Two facts with two reasons, kept apart on purpose because a verifier that +//! needs both must not take one as evidence of the other: +//! +//! 1. **A field coordinate is one candidate under RPX** — because a squeeze IS +//! four canonical felts once the byte reversal is gone. That is W1-A, and +//! [`TranscriptHash::CANDIDATES_PER_COORDINATE`] states it. +//! 2. **A query index is one draw** — because every WHIR query bound is a power +//! of two, so `sample_u64`'s rejection threshold is zero. That is true today, +//! hash-independent, and has nothing to do with (1). +//! +//! And one finding, which is the strongest reason the reversal had to go and is +//! not a cost argument at all: +//! [`the_reversal_would_have_biased_the_rpx_sampler`] — a reversed candidate is +//! `>= p` for about `2^32` canonical felts, so the old sampler drew uniformly +//! from a subset of the field. +//! +//! # ⛔ Why none of this is tested by sampling, and what that cost +//! +//! The obvious control — "show keccak needing more than one candidate" — is +//! unreachable. A keccak squeeze is 32 uniform bytes, so an 8-byte group lands +//! in `[p, 2^64)` with probability about **2^-32**: observing one takes on the +//! order of a billion squeezes. The same goes for `sample_u64`, whose rejection +//! region is `2^64 mod bound` wide — under `2^-32` of the range for any bound +//! this system uses, power of two or not. +//! +//! So a statistical test cannot tell the two configurations apart, and one that +//! appeared to would be measuring noise. **The difference is structural**: under +//! RPX every group is canonical BY CONSTRUCTION; under keccak every group is +//! canonical WITH HIGH PROBABILITY. The tests below pin the construction — +//! round-trips, the arithmetic of the threshold, and a rejection driven by a +//! candidate constructed to be rejected — rather than waiting for an event that +//! will not arrive. +//! +//! That is also why the rejection path is exercised explicitly: `Some(1)` is a +//! claim about the sampler's INPUTS, and it would be worth nothing if the +//! branch it bypasses had quietly stopped working. +//! +//! # ⚠ WHICH TEST CATCHES WHICH MUTATION — and which do not +//! +//! Run, not assumed. Putting the reversal back (either by flipping +//! `RpxTranscriptHash::REVERSES_SQUEEZE` or by deleting the `if` in `sample`) +//! fails **exactly one** test below: +//! [`an_rpx_squeeze_is_the_unreversed_digest_of_what_was_absorbed`]. +//! +//! [`every_group_of_an_rpx_squeeze_is_a_canonical_felt`] and +//! [`a_cubic_element_costs_exactly_three_draws_under_rpx`] both still PASS with +//! the reversal restored, and that is not a defect in them — it is the same +//! 2^-32 again from the other side. A canonical felt's bytes read backwards are +//! a number below `p` unless the top bytes conspire, so 2048 reversed groups +//! look exactly like 2048 canonical ones. Those two tests guard the +//! canonicalisation inside `digest_to_commitment`, which is a real regression +//! mode; they do **not** guard the byte order, and reading them as if they did +//! would leave the seam covered by nothing. +//! +//! One test guards the byte order. It is the one with the construction in it. + +use digest::Digest; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; +use math::field::goldilocks::{GOLDILOCKS_PRIME, GoldilocksField as Fp}; +use math::field::traits::HasDefaultTranscript; + +use crate::fiat_shamir::default_transcript::DefaultTranscript; +use crate::fiat_shamir::is_transcript::IsTranscript; +use crate::fiat_shamir::transcript_hash::{ + KeccakTranscriptHash, RpxTranscriptHash, TranscriptHash, +}; +use crate::hash::platform_keccak::PlatformKeccak256; +use crate::hash::rpx::{commitment_to_digest, digest_to_commitment, sponge_leaf_bytes}; + +/// The 8-byte big-endian groups a sampler reads out of a squeeze, in the order +/// [`DefaultTranscript::next_sample_u64`] reads them. +fn groups(squeeze: &[u8; 32]) -> [u64; 4] { + core::array::from_fn(|i| { + let mut b = [0u8; 8]; + b.copy_from_slice(&squeeze[i * 8..(i + 1) * 8]); + u64::from_be_bytes(b) + }) +} + +/// ★★ K5 (RPX half). The squeeze is the digest, in the digest's own byte order. +/// +/// Against the CONSTRUCTION — `digest_to_commitment(sponge_leaf_bytes(..))` — +/// not against the other arm. Two transcripts agreeing tells you nothing; they +/// agree on a wrong hash too. +#[test] +fn an_rpx_squeeze_is_the_unreversed_digest_of_what_was_absorbed() { + let absorbed = b"W1-A: the squeeze is the digest"; + let mut transcript = DefaultTranscript::::new(absorbed); + + let squeeze = transcript.sample(); + let expected = digest_to_commitment(&sponge_leaf_bytes(absorbed)); + + assert_eq!( + squeeze, expected, + "the RPX squeeze is not the digest of what was absorbed" + ); + + // …and NOT the reversed one, or the constant is decorative. + let mut reversed = expected; + reversed.reverse(); + assert_ne!( + squeeze, reversed, + "the RPX squeeze is still reversed — REVERSES_SQUEEZE is not being read" + ); + // ⚠ No `assert!(!RpxTranscriptHash::REVERSES_SQUEEZE)` here. Clippy is right + // that it cannot fail at runtime, and it would add nothing: the two + // assertions above test the BEHAVIOUR the constant is supposed to cause, + // which is what a wrong constant would break. Asserting the constant's own + // value would only restate the source line that sets it. +} + +/// ★★ K5 (keccak half). THE CONTROL. The keccak squeeze is still the reversed +/// digest, byte for byte. +/// +/// This is the half that must not move: every proof this system has produced +/// was produced under this convention, and W1-A is only allowed to touch the +/// other arm. +#[test] +fn a_keccak_squeeze_is_still_the_reversed_digest() { + let absorbed = b"W1-A: the squeeze is the digest"; + let mut transcript = DefaultTranscript::::new(absorbed); + + let squeeze = transcript.sample(); + + let mut expected: [u8; 32] = PlatformKeccak256::digest(absorbed).into(); + expected.reverse(); + + assert_eq!( + squeeze, expected, + "the keccak squeeze moved: this commit changed every proof on this branch" + ); +} + +/// ★★ THE PROPERTY BEHIND `Some(1)`. Every group of every RPX squeeze is a +/// canonical felt — by round-trip, not by luck. +/// +/// The chain is exercised, not one squeeze: `sample()` absorbs its own output, +/// so squeeze `n+1` is a function of squeeze `n`, and a canonicality that held +/// only for the first would be an accident of the seed. +#[test] +fn every_group_of_an_rpx_squeeze_is_a_canonical_felt() { + let mut transcript = DefaultTranscript::::new(b"chain"); + + for round in 0..512 { + let squeeze = transcript.sample(); + + for (i, g) in groups(&squeeze).iter().enumerate() { + assert!( + *g < GOLDILOCKS_PRIME, + "round {round}, group {i}: {g} is not a canonical felt, \ + so a one-candidate schedule would miss here" + ); + } + + // The structural statement the bound above is a consequence of: the 32 + // bytes ARE four felts, and reading them back gives the same four. + let digest = commitment_to_digest(&squeeze); + assert_eq!( + digest_to_commitment(&digest), + squeeze, + "round {round}: the squeeze does not round-trip through its felts" + ); + } +} + +/// ★★ ONE CANDIDATE, COMPOSED. The real sampler, fed the real squeeze, +/// consumes exactly three draws for a cubic element. +/// +/// [`Ext::sample_field_element_from`] is the production body; the closure is +/// the production byte source. Only the counter is the test's. +#[test] +fn a_cubic_element_costs_exactly_three_draws_under_rpx() { + let mut transcript = DefaultTranscript::::new(b"three draws"); + let squeeze = transcript.sample(); + let mut supply = groups(&squeeze).into_iter(); + + let mut draws = 0usize; + let element = Ext::sample_field_element_from(|| { + draws += 1; + supply + .next() + .expect("a fourth draw means a rejection occurred") + }); + + assert_eq!( + draws, + 3, + "a cubic element took {draws} draws, so CANDIDATES_PER_COORDINATE = {:?} is wrong", + RpxTranscriptHash::CANDIDATES_PER_COORDINATE + ); + assert_eq!(RpxTranscriptHash::CANDIDATES_PER_COORDINATE, Some(1)); + + // The element is the first three groups, in order — which is what makes the + // draw count meaningful rather than a count of a loop that did nothing. + let expected: Vec> = groups(&squeeze)[..3] + .iter() + .map(|g| FieldElement::from(*g)) + .collect(); + assert_eq!(element.value().to_vec(), expected); +} + +/// ★★ …AND THE REJECTION BRANCH IS STILL ALIVE. +/// +/// `Some(1)` is a claim about the sampler's INPUTS. If the rejection test had +/// been deleted, every test above would still pass and the constant would be +/// true for the wrong reason — so the branch is driven by a candidate +/// constructed to be rejected. This is the only way to reach it: waiting for a +/// keccak squeeze to produce one is a 2^-32 event per group. +#[test] +fn a_non_canonical_candidate_is_rejected_and_redrawn() { + // p itself is the smallest non-canonical u64, and `p + 7` is inside the + // window a uniform draw can land in. + let supply = [GOLDILOCKS_PRIME, GOLDILOCKS_PRIME + 7, 42u64]; + let mut it = supply.into_iter(); + + let mut draws = 0usize; + let element = Fp::sample_field_element_from(|| { + draws += 1; + it.next().expect("the sampler drew more than the supply") + }); + + assert_eq!( + draws, 3, + "the sampler accepted a candidate >= p: the rejection test is gone, and \ + CANDIDATES_PER_COORDINATE = Some(1) would then be true of nothing" + ); + assert_eq!(element, FieldElement::::from(42u64)); +} + +/// ★★★ WHAT THE REVERSAL WAS ACTUALLY DOING: biasing the RPX sampler. +/// +/// This is the strongest reason to remove it, and it is not a cost argument. +/// +/// A candidate under the old code was `byteswap(canonical(felt))`. That is +/// `>= p` exactly when the felt's low four bytes are all `0xFF` — reversing +/// puts them in the top four, and `p`'s top four bytes are `0xFFFFFFFF`. The +/// rejection sampler then drew uniformly from a SUBSET of `[0, p)` missing +/// about `2^32` elements: statistical distance ~`2^-32` per coordinate, and +/// over an epoch verify's ~3e4 coordinate draws a loose hybrid bound of +/// ~`2^-17` of added soundness error. +/// +/// Not a demonstrated attack — the excluded set is fixed and public and no +/// prover steers into it — and never exercised, because the RPX transcript was +/// not wired to any prover (see the commit). But it is exactly the kind of +/// unquoted term an audit names, and it was inherited rather than chosen: +/// harmless under keccak, whose 8-byte groups are uniform on 64 bits and whose +/// sampler is therefore exactly uniform. It exists only in the +/// RPX-under-`DefaultTranscript` combination. +/// +/// The test exhibits the witness rather than describing it, and characterises +/// the whole excluded set so the claim is a statement about all of it. +#[test] +fn the_reversal_would_have_biased_the_rpx_sampler() { + let byteswap = |v: u64| { + u64::from_be_bytes({ + let mut b = v.to_be_bytes(); + b.reverse(); + b + }) + }; + + // V1's witness: a canonical felt whose reversed bytes are NOT canonical, so + // the old code would have rejected this felt every time it appeared. + let witness: u64 = 0x0000_0001_ffff_ffff; + assert!( + witness < GOLDILOCKS_PRIME, + "the witness must be a real felt" + ); + assert!( + byteswap(witness) >= GOLDILOCKS_PRIME, + "the witness's reversed bytes are canonical, so it is not a witness" + ); + + // …and the excluded set is exactly the felts whose low four bytes are all + // `0xFF`, save the one whose high four bytes are zero. Checked over the + // whole set rather than sampled: it is generated, not searched for. + for high in 0..4096u64 { + let v = (high << 32) | 0xFFFF_FFFF; + if v >= GOLDILOCKS_PRIME { + continue; + } + let excluded = byteswap(v) >= GOLDILOCKS_PRIME; + assert_eq!( + excluded, + high != 0, + "felt {v:#018x} is misclassified: the excluded set is not what the \ + bias argument says it is" + ); + } + + // The far larger complement: nothing OUTSIDE that set was excluded, so the + // bias is precisely the one described and not a larger one. + let mut checked = 0u32; + for v in (0..1u64 << 24).map(|i| i.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 1) { + if v >= GOLDILOCKS_PRIME || (v & 0xFFFF_FFFF) == 0xFFFF_FFFF { + continue; + } + assert!( + byteswap(v) < GOLDILOCKS_PRIME, + "felt {v:#018x} was excluded but is not in the described set" + ); + checked += 1; + } + assert!( + checked > 1_000_000, + "only {checked} felts were actually checked" + ); + + // And the fix: a live squeeze's groups are canonical, so no candidate is + // excluded and the distribution is the digest's, entire. + let mut transcript = DefaultTranscript::::new(b"bias"); + for _ in 0..64 { + for g in groups(&transcript.sample()) { + assert!(g < GOLDILOCKS_PRIME); + } + } +} + +/// ★ The keccak configuration makes no such claim, and says so. +#[test] +fn keccak_does_not_claim_a_bounded_candidate_count() { + assert_eq!( + KeccakTranscriptHash::CANDIDATES_PER_COORDINATE, + None, + "keccak's squeeze is uniform bytes: the draw count has a distribution, not a bound" + ); +} + +/// ★★ THE SECOND FACT, WITH ITS OWN REASON. A query index is one draw because +/// the bound is a power of two — not because of anything W1-A did. +/// +/// `sample_u64` rejects `candidate < 2^64 mod bound`. For a power of two that +/// region is EMPTY, so the loop cannot turn, whatever the hash. Every WHIR +/// query bound is `num_leaves = 1 << (log_domain_size - log_folding)` +/// (`multilinear::whir_commit::CodewordCommitment::num_leaves`), so this covers +/// all of them. +/// +/// ⚠ The non-power-of-two half is the point of the test. Without it this would +/// assert that zero equals zero for 64 values and pass on any implementation. +#[test] +fn a_power_of_two_bound_has_no_rejection_region_and_a_ragged_one_does() { + let threshold = |bound: u64| bound.wrapping_neg() % bound; + + for k in 0..64 { + let bound = 1u64 << k; + assert_eq!( + threshold(bound), + 0, + "bound 2^{k} has a rejection region, so a query index is not one draw" + ); + } + + // Constructed counter-examples: a bound that is NOT a power of two must + // have a non-empty rejection region, or the expression above is not + // computing what this test claims it computes. + for bound in [3u64, 5, 6, 100, (1 << 20) + 1, u64::MAX] { + assert_ne!( + threshold(bound), + 0, + "bound {bound} is not a power of two yet shows no rejection region" + ); + } +} + +/// ★ …and the draw count that follows from it, observed through the transcript +/// rather than asserted. +/// +/// Four `sample_u64` calls at a power-of-two bound must consume exactly one +/// squeeze. The witness is the transcript's own state: every squeeze chains its +/// output back in, so a fifth draw would leave `a` somewhere `b` is not. +#[test] +fn four_query_indices_cost_one_squeeze() { + for hash_is_rpx in [false, true] { + let (state_a, state_b) = if hash_is_rpx { + let mut a = DefaultTranscript::::new(b"queries"); + let mut b = a.clone(); + for _ in 0..4 { + a.sample_u64(1 << 20); + } + let _ = b.sample(); + ( + IsTranscript::::state(&a), + IsTranscript::::state(&b), + ) + } else { + let mut a = DefaultTranscript::::new(b"queries"); + let mut b = a.clone(); + for _ in 0..4 { + a.sample_u64(1 << 20); + } + let _ = b.sample(); + ( + IsTranscript::::state(&a), + IsTranscript::::state(&b), + ) + }; + + assert_eq!( + state_a, state_b, + "four query draws consumed more than one squeeze (rpx = {hash_is_rpx})" + ); + } +} + +/// ★ The duplex buffer hands out whole groups, which is what makes the felt +/// boundaries and the read boundaries the same boundaries. +/// +/// Sampling 4 `u64`s consumes exactly one squeeze; the 5th forces the next. If +/// a read ever straddled two groups, `Some(1)` would be false even with a +/// canonical squeeze — the claim depends on this alignment, so it is pinned. +#[test] +fn the_buffer_is_consumed_in_whole_felt_groups() { + let mut transcript = DefaultTranscript::::new(b"alignment"); + let mut reference = transcript.clone(); + + let first = reference.sample(); + let second = reference.sample(); + + let drawn: Vec = (0..8).map(|_| transcript.sample_u64(u64::MAX)).collect(); + + let expected: Vec = groups(&first) + .into_iter() + .chain(groups(&second)) + .map(|g| g % u64::MAX) + .collect(); + + assert_eq!( + drawn, expected, + "the buffer is not being handed out as whole 8-byte groups in order" + ); +} diff --git a/crypto/crypto/tests/transcript_counters.rs b/crypto/crypto/tests/transcript_counters.rs new file mode 100644 index 000000000..934a0a78c --- /dev/null +++ b/crypto/crypto/tests/transcript_counters.rs @@ -0,0 +1,342 @@ +//! ★★ The Fiat-Shamir counters answer "which sponge ran", on BOTH arms. +//! +//! ```text +//! cargo test -p crypto --features hash-metrics --test transcript_counters +//! ``` +//! +//! # The trap these exist to close +//! +//! `hash_metrics`'s header already tells this story for Merkle: a counter that +//! compares `TypeId` against keccak and does nothing otherwise "became a check +//! that cannot fail the moment a second hash arrived — under RPX every Merkle +//! counter would have read ZERO, reporting *no hashing* for precisely the arm +//! whose purpose is to change the hashing". +//! +//! The transcript never got that treatment. `count_absorb` was bumped from one +//! place, the keccak wrapper's `update`; `Rpx256Digest::update` was a bare +//! `Vec::extend`; and `total`'s documentation claimed to count transcript +//! squeezes while nothing on the RPX side bumped it. So an RPX proof read zero +//! absorbs and zero transcript finalizes, and zero is exactly what a +//! *correctly instrumented* keccak-free run would also read. The measurement +//! could not distinguish "the other hash ran" from "nobody instrumented it". +//! +//! That is not hypothetical here: for four measured A/Bs the RPX arm ran a +//! KECCAK transcript, and no instrument disagreed. +//! +//! # So every assertion below is two-sided +//! +//! Each arm asserts both that its own counters MOVED and that the other arm's +//! are ZERO. One half alone is worthless: "rpx > 0, keccak 0" is equally true +//! of a run where the keccak transcript was never instrumented, which is the +//! state this file is about. +//! +//! # ⚠ Its own binary, AND a lock +//! +//! The counters are process-global, so both are needed and neither is enough. +//! +//! The binary, because `crypto`'s lib-test binary runs tests in parallel and +//! several of them hash: an exact assertion there is an assertion about +//! whatever else happened to be running — a keccak arm read 215 squeezes of +//! which 200 were keccak, purely from neighbours. +//! +//! The lock, because an integration test binary ALSO runs its own tests in +//! parallel. Moving the file and stopping there left four of five failing, one +//! reading `left: 2, right: 0` where a sibling had reset the counters between +//! this test's `reset` and its `snapshot`. Every test below takes +//! [`serialise`] for its whole reset-measure-assert window. + +#![cfg(feature = "hash-metrics")] + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::transcript_hash::{KeccakTranscriptHash, RpxTranscriptHash}; +use crypto::hash::rpx::Rpx256Digest; +use crypto::hash_metrics; +use digest::Update; +use math::field::element::FieldElement; +use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext; + +/// ★ Taken by every test here: `reset` and `snapshot` address one global pair +/// of counters, so a measurement is only this test's while it holds this. +/// +/// Poisoning is ignored so one failure does not cascade into unrelated tests. +static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serialise() -> std::sync::MutexGuard<'static, ()> { + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Four absorbs and two squeezes, driven identically on either configuration. +/// +/// `new` absorbs once, so the count is `1 + 3`. +fn drive(seed: &[u8]) -> (u64, u64) +where + T: crypto::fiat_shamir::transcript_hash::TranscriptHash, +{ + let mut t = DefaultTranscript::::new(seed); + t.append_bytes(b"one"); + t.append_bytes(b"two"); + t.append_bytes(b"three"); + let _ = t.sample(); + let _ = t.sample(); + (4, 2) +} + +#[test] +fn a_keccak_transcript_counts_as_keccak_and_nothing_else() { + let _serialised = serialise(); + hash_metrics::reset(); + let (absorbs, squeezes) = drive::(b"seed"); + let c = hash_metrics::snapshot(); + + assert_eq!(c.transcript_absorbs_keccak, absorbs); + assert_eq!(c.transcript_squeezes_keccak, squeezes); + // The other arm must be silent — and it must be silent because nothing RPX + // ran, which the totals below are what establish. + assert_eq!(c.transcript_absorbs_rpx, 0); + assert_eq!(c.transcript_squeezes_rpx, 0); + assert_eq!(c.transcript_absorbs, absorbs); + assert_eq!(c.transcript_squeezes, squeezes); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +#[test] +fn an_rpx_transcript_counts_as_rpx_and_nothing_else() { + let _serialised = serialise(); + hash_metrics::reset(); + let (absorbs, squeezes) = drive::(b"seed"); + let c = hash_metrics::snapshot(); + + assert_eq!( + c.transcript_absorbs_rpx, absorbs, + "the RPX transcript absorbed {} times and the counter saw {} — an \ + un-instrumented sponge reads zero, which is indistinguishable from \ + one that never ran", + absorbs, c.transcript_absorbs_rpx + ); + assert_eq!(c.transcript_squeezes_rpx, squeezes); + assert_eq!( + c.transcript_absorbs_keccak, 0, + "a keccak absorb during an RPX-only run: the transcript is not the \ + configuration's" + ); + assert_eq!(c.transcript_squeezes_keccak, 0); + assert_eq!(c.transcript_absorbs, absorbs); + assert_eq!(c.transcript_squeezes, squeezes); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +/// ★ The two configurations do the SAME amount of transcript work. +/// +/// The counters must differ only in which bucket they land in. If a swap +/// changed the absorb count, the per-arm line would be reporting a protocol +/// difference as a hash difference — and a reader comparing arms would draw +/// the wrong conclusion about what the hash costs. +#[test] +fn the_two_configurations_do_the_same_transcript_work() { + let _serialised = serialise(); + hash_metrics::reset(); + drive::(b"same"); + let k = hash_metrics::snapshot(); + + hash_metrics::reset(); + drive::(b"same"); + let r = hash_metrics::snapshot(); + + assert_eq!(k.transcript_absorbs, r.transcript_absorbs); + assert_eq!(k.transcript_squeezes, r.transcript_squeezes); +} + +/// ★★ The transcript counter counts `Update::update` calls, tied to the +/// counter that already did. +/// +/// On a keccak-only run `absorb_calls` sees every `update` the sponge receives +/// and this file's counter sees every one the TRANSCRIPT issues. They differ by +/// exactly the chaining re-absorb — `sample` feeds its own output back in, once +/// per squeeze, which is part of squeezing and not an absorb anyone asked for. +/// So: +/// +/// ```text +/// absorb_calls == transcript_absorbs + transcript_squeezes +/// ``` +/// +/// This is the assertion that makes the new counters non-vacuous against +/// something that predates them, and it fails if either counter is moved, +/// double-counted, or attached to the wrong call. +/// +/// ⚠ SCOPE: state reads are deliberately NOT a term here, and adding one would +/// be wrong rather than more complete. `state()` finalizes a clone; it issues +/// no `update`, so it cannot move `absorb_calls` and has no business in an +/// identity about absorbs. It is counted separately by +/// [`a_state_read_is_counted_separately_from_a_squeeze`], where its own control +/// is the grind count. +/// +/// Lane V1's closed form for the block, in these terms and satisfying this +/// identity by construction: 582,703 absorbs + 182,734 squeezes == 765,437 +/// `absorb_calls`, with the 2,996 state finalizes outside all three. +/// +/// ⚠ An earlier version of this test asserted a cubic element absorbs in +/// "several chunks". It does not: `stream_bytes` for the degree-3 extension +/// writes one 24-byte buffer and calls the sink ONCE +/// (`extensions_goldilocks.rs:567-571`). The test failed, which is how the +/// claim — and a comment in `default_transcript.rs` repeating it — got fixed. +#[test] +fn the_transcript_absorbs_agree_with_the_generic_absorb_counter() { + let _serialised = serialise(); + hash_metrics::reset(); + let (absorbs, squeezes) = drive::(b"tie"); + let c = hash_metrics::snapshot(); + + assert_eq!( + c.absorb_calls, + c.transcript_absorbs + c.transcript_squeezes, + "the keccak sponge saw {} updates; the transcript issued {} absorbs and \ + {} squeezes, and a squeeze chains exactly one update", + c.absorb_calls, + c.transcript_absorbs, + c.transcript_squeezes + ); + assert_eq!( + (c.transcript_absorbs, c.transcript_squeezes), + (absorbs, squeezes) + ); +} + +/// ★ A field element costs the same absorbs on either arm. +/// +/// The counters must differ only in which bucket they land in: if a hash swap +/// changed the absorb count, the per-arm line would report a protocol +/// difference as a hash difference. +#[test] +fn a_field_element_absorb_costs_the_same_on_both_arms() { + let _serialised = serialise(); + let element = FieldElement::::from(7u64); + + hash_metrics::reset(); + let mut k = DefaultTranscript::::new(&[]); + k.append_field_element(&element); + let k = hash_metrics::snapshot(); + + hash_metrics::reset(); + let mut r = DefaultTranscript::::new(&[]); + r.append_field_element(&element); + let r = hash_metrics::snapshot(); + + assert_eq!( + k.transcript_absorbs_keccak, r.transcript_absorbs_rpx, + "the two arms disagree on how much one element absorbs" + ); + assert_eq!(k.transcript_absorbs_rpx, 0); + assert_eq!(r.transcript_absorbs_keccak, 0); +} + +/// ★★ …and the GENERIC counters are no longer keccak-only either. +/// +/// `count_absorb` had exactly one call site — the keccak wrapper's `update` — +/// so `absorb_calls` and `absorb_bytes` read ZERO for an RPX proof, and +/// `total`'s documentation ("every finalize … transcript squeeze") was false +/// for this sponge. This drives the digest directly, so it fails if either call +/// is removed. +#[test] +fn the_rpx_digest_bumps_the_generic_counters() { + let _serialised = serialise(); + hash_metrics::reset(); + let mut d = Rpx256Digest::default(); + Update::update(&mut d, b"twelve bytes"); + Update::update(&mut d, b"and more"); + let after_absorbs = hash_metrics::snapshot(); + + assert_eq!( + after_absorbs.absorb_calls, 2, + "absorbing into an RPX digest moved `absorb_calls` to {} — it read 0 \ + before this was instrumented, for every RPX proof ever measured", + after_absorbs.absorb_calls + ); + assert_eq!(after_absorbs.absorb_bytes, 12 + 8); + assert_eq!( + after_absorbs.total, 0, + "nothing has been finalized yet, so `total` must not have moved" + ); + + use digest::FixedOutputReset; + let mut out = digest::Output::::default(); + d.finalize_into_reset(&mut out); + assert_eq!( + hash_metrics::snapshot().total, + 1, + "an RPX finalize did not reach `total`, whose own doc says it counts \ + every finalize" + ); +} + +/// ★★★ A `state()` IS COUNTED, AND IT IS NOT A SQUEEZE. +/// +/// The distinction lane V1's closed form turns on. `sample` is a +/// `finalize_reset` whose output is chained back in, so it advances the +/// transcript; `state` finalizes a CLONE and changes nothing. On a block proof +/// they are 182,734 and 2,996 — a counter hooked only to `finalize_reset` +/// misses every one of the 2,996, which is precisely what this file's first +/// version did. +/// +/// Both directions are asserted, because either conflation is a live failure +/// mode: a state must not appear as a squeeze, AND a squeeze must not appear as +/// a state. Counting their sum would satisfy neither of V1's two numbers. +#[test] +fn a_state_read_is_counted_separately_from_a_squeeze() { + let _serialised = serialise(); + + hash_metrics::reset(); + let mut t = DefaultTranscript::::new(b"state"); + let before = t.state(); + let c = hash_metrics::snapshot(); + assert_eq!( + (c.transcript_states, c.transcript_states_rpx), + (1, 1), + "a state() read was not counted" + ); + assert_eq!( + c.transcript_squeezes, 0, + "a state() read was counted as a squeeze — it finalizes a clone and \ + advances nothing, so a squeeze count including it cannot be checked \ + against a closed form" + ); + + // …and it really did not advance the chain, which is why it is a different + // number rather than a different name for the same one. + let again = t.state(); + assert_eq!(before, again, "state() advanced the transcript"); + assert_eq!(hash_metrics::snapshot().transcript_states, 2); + + // The converse: a squeeze is not counted as a state. + hash_metrics::reset(); + let _ = t.sample(); + let c = hash_metrics::snapshot(); + assert_eq!( + (c.transcript_squeezes, c.transcript_squeezes_rpx), + (1, 1), + "a squeeze was not counted" + ); + assert_eq!( + c.transcript_states, 0, + "a squeeze was counted as a state read" + ); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +/// ★ …and the keccak arm tags its state reads too. +/// +/// One side is not evidence: "rpx states > 0, keccak 0" is equally true when +/// the keccak path was never instrumented. +#[test] +fn a_keccak_state_read_is_tagged_as_keccak() { + let _serialised = serialise(); + + hash_metrics::reset(); + let t = DefaultTranscript::::new(b"state"); + let _ = t.state(); + let c = hash_metrics::snapshot(); + + assert_eq!((c.transcript_states, c.transcript_states_keccak), (1, 1)); + assert_eq!(c.transcript_states_rpx, 0); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} diff --git a/crypto/math-cuda/build.rs b/crypto/math-cuda/build.rs index 1b666d7f3..f257b936f 100644 --- a/crypto/math-cuda/build.rs +++ b/crypto/math-cuda/build.rs @@ -181,6 +181,7 @@ fn main() { compile_kernel("arith.cu", "arith.cubin", have_nvcc); compile_kernel("ntt.cu", "ntt.cubin", have_nvcc); compile_kernel("keccak.cu", "keccak.cubin", have_nvcc); + compile_kernel("rpx.cu", "rpx.cubin", have_nvcc); compile_kernel("barycentric.cu", "barycentric.cubin", have_nvcc); compile_kernel("deep.cu", "deep.cubin", have_nvcc); compile_kernel("fri.cu", "fri.cubin", have_nvcc); diff --git a/crypto/math-cuda/kernels/rpx.cu b/crypto/math-cuda/kernels/rpx.cu new file mode 100644 index 000000000..5c6a871af --- /dev/null +++ b/crypto/math-cuda/kernels/rpx.cu @@ -0,0 +1,946 @@ +// RPX256 (Rescue-Prime eXtended / XHash12) over Goldilocks at width 12 on +// device — the permutation, the rate-8 overwrite-duplex leaf sponge and the +// Merkle parent (lane K phase 1, arithmetic), then the leaf/tree kernels that +// stream table rows through `rpx::Sponge` and `rpx::compress` and the +// permutation probe (phase 2, the `extern "C"` surface at the end of the +// file, kernel for kernel the twin of `blake3.cu:338-620`). +// +// THE ORACLE is the Rust host implementation, byte for byte: +// `prover/src/lfm/rpx.rs` `Rpx256::permute` (:280-316) — schedule FB E FB E FB E M, +// `cubic_ext::{mul, power7}` (:118-140); +// `prover/src/lfm/rpo.rs` ARK1/ARK2 (:119-321, RPX imports RPO's tables +// literally), `sbox` (:455-460), `inv_sbox_layer` +// (:481-509, the 72-multiplication chain), `mds` +// (:539-557, the u128 accumulation); +// `prover/src/lfm/algebraic_commit.rs` `leaf_capacity` (:142-147), +// `sponge_leaf` (:169-184), `parent` (:248-252); +// `prover/src/lfm/hash.rs` `permute_two_cells` (:95-108): `[a ‖ b ‖ iv]`, +// digest = lanes 0..4. +// +// PROVENANCE, layered exactly as the Rust module's own (rpx.rs "PROVENANCE"): +// the FB round IS RPO's round with RPO's constants, and those are pinned by +// nineteen EXTERNAL miden-crypto vectors, which `tests/host_kat/rpx_host_kat.cpp` +// replays through `fb_round(s, r)` composed seven times. The E round (the cubic +// extension) and the schedule have no published vector anywhere; they are +// pinned to the Rust oracle's output (`prover/tests/rpx_host_kat_vectors.rs`) +// and, independently, to naive polynomial arithmetic in the harness. +// +// REPRESENTATION. Inputs may be raw `[0, 2^64)` Goldilocks storage exactly as +// `goldilocks.cuh` allows everywhere else; every step here (`add`, `mul`, +// `dot3`, the MDS bound) accepts that. `permute` CANONICALISES its output, so +// digests are canonical `< p` and their big-endian bytes are what +// `digest_to_commitment` (algebraic_commit.rs:112-118) writes — the device +// Merkle tree can be compared to the host's byte for byte. +// +// ⚠ TWO CUBIC EXTENSIONS EXIST AND THIS FILE USES THE OTHER ONE. `ext3.cuh` is +// the VM's `w³ = 2`; RPX's is `φ³ = φ + 1` (rpx.rs:98-103). Only the GENERIC +// three-term dot product `ext3::dot3` is borrowed from that header — never +// `ext3::mul`. The reduction polynomial lives in `rpx::ext_mul` alone. +// +// COST MODEL (one permutation; counted by the harness's op counters, static +// for the MDS): +// FB round ×3 : 12·(4 + 72) = 912 Goldilocks multiplications (48 forward +// S-box, 864 inverse), 2 MDS, 24 constant adds; +// E round ×3 : 4 triples × 4 extension products = 16 `ext_mul` = 144 wide +// 64×64 products folded into 48 reductions (`dot3`), 12 +// constant adds, 32 operand pre-adds; +// M round ×1 : 1 MDS, 12 constant adds; +// MDS ×7 : 288 32×32→64 multiply-adds + 12 reductions each — the ported +// u128 property (see `mds`), ~6× under twelve field +// multiplications per lane. +// Total: 2736 field multiplications + 144 dot3 (432 wide products) + 300 adds +// + 2016 narrow MACs. The inverse S-box is 2592/2736 = 95% of the field +// multiplications; RPO spends 7 such layers, RPX 3 — that is the whole +// reason RPX exists (rpx.rs:22-28). +// +// PHASE-2 TUNING NOTES (not done here, do not guess at them): `inv_sbox` is a +// serial 72-deep chain per lane — one thread per permutation interleaves twelve +// of them; register pressure is what to measure (`-Xptxas -v`). `Sponge::absorb` +// indexes the state dynamically, which nvcc lowers to local memory unless the +// caller's loop is unrolled — the same trade `Blake3Chain::push_word` makes. +// ARK reads are warp-uniform constant-bank operands and cost nothing. + +#include +#include "goldilocks.cuh" +#include "ext3.cuh" + +// `permute` is a REAL device function, never inlined (see its CODE SHAPE +// note). The host shim has no `__noinline__`; on the host the attribute only +// matters to the code-size probe, which asks for it explicitly. +#if defined(__CUDACC__) +#define RPX_NOINLINE __noinline__ +#elif defined(RPX_HOST_NOINLINE) +#define RPX_NOINLINE __attribute__((noinline)) +#else +#define RPX_NOINLINE +#endif + +namespace rpx { + +enum : int { + STATE_FELTS = 12, + RATE_FELTS = 8, + CAPACITY_FELTS = 4, + DIGEST_FELTS = 4, + NUM_ROUNDS = 7, + EXT_DEGREE = 3, + EXT_ELEMENTS = 4, + // Absolute lanes of the two capacity cells the socket names: the padding + // flag (`rpo.rs:339` CAPACITY_PAD_LANE = 0 within the capacity) and the + // domain tag (`rpo.rs:343` CAPACITY_DOMAIN_LANE = 1). Capacity = lanes 8..12. + CAPACITY_PAD_LANE = RATE_FELTS + 0, + CAPACITY_DOMAIN_LANE = RATE_FELTS + 1, +}; + +// The Merkle-parent domain is ZERO on purpose (rpo.rs:350): a parent is a +// standard `Rpx256::merge`, checkable against miden without this codebase. +__device__ constexpr uint64_t DOMAIN_COMPRESS = 0; +// The leaf domain: `u32::from_le_bytes(b"LFML")` (rpo.rs:358) = 1280132684. +__device__ constexpr uint64_t DOMAIN_LEAF = 0x4C4D464CULL; + +// --------------------------------------------------------------------------- +// Constants. Transcribed MECHANICALLY (a script over rpo.rs, not by hand) from +// `rpo.rs` ARK1 (:119-218), ARK2 (:222-321) and MDS_CIRC_ROW (:114). RPX +// imports exactly these (rpx.rs:69); `rpx_uses_rpos_constant_tables` asserts +// the import on the host, and the miden vectors in the harness pin them here. +// Only the FB rounds (0, 2, 4) consume ARK2; E and M rounds add ARK1 alone. +// --------------------------------------------------------------------------- +__device__ __constant__ uint64_t ARK1[NUM_ROUNDS][STATE_FELTS] = { + {5789762306288267392ull, 6522564764413701783ull, 17809893479458208203ull, 107145243989736508ull, + 6388978042437517382ull, 15844067734406016715ull, 9975000513555218239ull, 3344984123768313364ull, + 9959189626657347191ull, 12960773468763563665ull, 9602914297752488475ull, 16657542370200465908ull}, + {12987190162843096997ull, 653957632802705281ull, 4441654670647621225ull, 4038207883745915761ull, + 5613464648874830118ull, 13222989726778338773ull, 3037761201230264149ull, 16683759727265180203ull, + 8337364536491240715ull, 3227397518293416448ull, 8110510111539674682ull, 2872078294163232137ull}, + {18072785500942327487ull, 6200974112677013481ull, 17682092219085884187ull, 10599526828986756440ull, + 975003873302957338ull, 8264241093196931281ull, 10065763900435475170ull, 2181131744534710197ull, + 6317303992309418647ull, 1401440938888741532ull, 8884468225181997494ull, 13066900325715521532ull}, + {5674685213610121970ull, 5759084860419474071ull, 13943282657648897737ull, 1352748651966375394ull, + 17110913224029905221ull, 1003883795902368422ull, 4141870621881018291ull, 8121410972417424656ull, + 14300518605864919529ull, 13712227150607670181ull, 17021852944633065291ull, 6252096473787587650ull}, + {4887609836208846458ull, 3027115137917284492ull, 9595098600469470675ull, 10528569829048484079ull, + 7864689113198939815ull, 17533723827845969040ull, 5781638039037710951ull, 17024078752430719006ull, + 109659393484013511ull, 7158933660534805869ull, 2955076958026921730ull, 7433723648458773977ull}, + {16308865189192447297ull, 11977192855656444890ull, 12532242556065780287ull, 14594890931430968898ull, + 7291784239689209784ull, 5514718540551361949ull, 10025733853830934803ull, 7293794580341021693ull, + 6728552937464861756ull, 6332385040983343262ull, 13277683694236792804ull, 2600778905124452676ull}, + {7123075680859040534ull, 1034205548717903090ull, 7717824418247931797ull, 3019070937878604058ull, + 11403792746066867460ull, 10280580802233112374ull, 337153209462421218ull, 13333398568519923717ull, + 3596153696935337464ull, 8104208463525993784ull, 14345062289456085693ull, 17036731477169661256ull}, +}; + +__device__ __constant__ uint64_t ARK2[NUM_ROUNDS][STATE_FELTS] = { + {6077062762357204287ull, 15277620170502011191ull, 5358738125714196705ull, 14233283787297595718ull, + 13792579614346651365ull, 11614812331536767105ull, 14871063686742261166ull, 10148237148793043499ull, + 4457428952329675767ull, 15590786458219172475ull, 10063319113072092615ull, 14200078843431360086ull}, + {6202948458916099932ull, 17690140365333231091ull, 3595001575307484651ull, 373995945117666487ull, + 1235734395091296013ull, 14172757457833931602ull, 707573103686350224ull, 15453217512188187135ull, + 219777875004506018ull, 17876696346199469008ull, 17731621626449383378ull, 2897136237748376248ull}, + {8023374565629191455ull, 15013690343205953430ull, 4485500052507912973ull, 12489737547229155153ull, + 9500452585969030576ull, 2054001340201038870ull, 12420704059284934186ull, 355990932618543755ull, + 9071225051243523860ull, 12766199826003448536ull, 9045979173463556963ull, 12934431667190679898ull}, + {18389244934624494276ull, 16731736864863925227ull, 4440209734760478192ull, 17208448209698888938ull, + 8739495587021565984ull, 17000774922218161967ull, 13533282547195532087ull, 525402848358706231ull, + 16987541523062161972ull, 5466806524462797102ull, 14512769585918244983ull, 10973956031244051118ull}, + {6982293561042362913ull, 14065426295947720331ull, 16451845770444974180ull, 7139138592091306727ull, + 9012006439959783127ull, 14619614108529063361ull, 1394813199588124371ull, 4635111139507788575ull, + 16217473952264203365ull, 10782018226466330683ull, 6844229992533662050ull, 7446486531695178711ull}, + {3736792340494631448ull, 577852220195055341ull, 6689998335515779805ull, 13886063479078013492ull, + 14358505101923202168ull, 7744142531772274164ull, 16135070735728404443ull, 12290902521256031137ull, + 12059913662657709804ull, 16456018495793751911ull, 4571485474751953524ull, 17200392109565783176ull}, + {17130398059294018733ull, 519782857322261988ull, 9625384390925085478ull, 1664893052631119222ull, + 7629576092524553570ull, 3485239601103661425ull, 9755891797164033838ull, 15218148195153269027ull, + 16460604813734957368ull, 9643968136937729763ull, 3611348709641382851ull, 18256379591337759196ull}, +}; + +// First ROW of the circulant MDS, `M[i][j] = ROW[(j − i) mod 12]` +// (rpo.rs:107-114), stored TWICE so that `MDS_CIRC_ROW2[j + 12 − i]` is the +// entry with no modulo: the output-lane loop in `mds` is rolled, so `i` is a +// runtime value there. 32-bit so each MDS term is one 32×32→64 MAC. The row +// sums to 160, which is the bound `mds` rests on. +__device__ __constant__ uint32_t MDS_CIRC_ROW2[2 * STATE_FELTS] = { + 7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8, 7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8}; + +// --------------------------------------------------------------------------- +// Field-op forwarders. Under nvcc they are the `goldilocks.cuh` / `ext3.cuh` +// primitives, nothing more. The host-KAT harness defines RPX_HOST_OP_COUNT +// before including this file so it can COUNT them per round kind and print the +// cost model above as a measurement rather than a claim. +// --------------------------------------------------------------------------- +#ifdef RPX_HOST_OP_COUNT +struct OpCount { + unsigned long long mul, dot3, add; +}; +static OpCount g_ops = {0, 0, 0}; +#define RPX_COUNT(field) (++g_ops.field) +#else +#define RPX_COUNT(field) ((void)0) +#endif + +__device__ __forceinline__ uint64_t fmul(uint64_t a, uint64_t b) { + RPX_COUNT(mul); + return goldilocks::mul(a, b); +} + +__device__ __forceinline__ uint64_t fadd(uint64_t a, uint64_t b) { + RPX_COUNT(add); + return goldilocks::add(a, b); +} + +// `a0·b0 + a1·b1 + a2·b2` with ONE reduction — the generic part of `ext3.cuh`, +// independent of that header's reduction polynomial. +__device__ __forceinline__ uint64_t fdot3(uint64_t a0, uint64_t b0, uint64_t a1, uint64_t b1, + uint64_t a2, uint64_t b2) { + RPX_COUNT(dot3); + return ext3::dot3(a0, b0, a1, b1, a2, b2); +} + +// --------------------------------------------------------------------------- +// The circulant MDS, `out_i = Σ_j MDS_CIRC_ROW[(j − i) mod 12] · s_j`, in +// `rpo.rs:539-557`'s orientation (the one the miden vectors pin). +// +// ★ THE PORTED PROPERTY (rpo.rs:527-536): one accumulation and ONE reduction +// per output lane, no per-term field multiplication. Every coefficient is ≤ 26 +// and every stored lane is < 2^64, so the twelve-term row sum is < 12·26·2^64 +// < 2^73 and needs no reduction before the end. The host accumulates it in a +// u128; the device has no u128, so the SAME integer is assembled from 32-bit +// halves. With `s_j = h_j·2^32 + l_j`, +// +// acc = 2^32 · Σ_j c_j·h_j + Σ_j c_j·l_j , +// +// and each half-sum is ≤ 160·(2^32 − 1) < 2^40 — the row sums to 160 — so both +// fit a u64 with 24 bits to spare and every term is a single 32×32→64 +// multiply-add (no 64-bit multiplier anywhere in the MDS). The halves are then +// recombined into the u128's `(lo, hi)` exactly as the host holds them and +// reduced the host's way: `acc = hi·2^64 + lo ≡ lo + hi·EPSILON (mod p)`, with +// `hi < 2^9` so `hi·EPSILON < 2^41` needs no reduction of its own +// (`the_mds_row_sum_cannot_overflow_a_u128` asserts the same bound on the host). +// --------------------------------------------------------------------------- +__device__ __forceinline__ void mds(uint64_t s[STATE_FELTS]) { + uint32_t lo32[STATE_FELTS], hi32[STATE_FELTS]; +#pragma unroll + for (int j = 0; j < STATE_FELTS; ++j) { + lo32[j] = (uint32_t)s[j]; + hi32[j] = (uint32_t)(s[j] >> 32); + } + uint64_t out[STATE_FELTS]; + // Rolled over output lanes: twelve iterations of twenty-four MACs, one + // twelfth of the unrolled body's code for the same instruction count. +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) { + uint64_t acc_lo = 0, acc_hi = 0; // Σ c·l_j and Σ c·h_j, each < 2^40 + const int rot = STATE_FELTS - i; // MDS_CIRC_ROW2[j + rot] = ROW[(j − i) mod 12] +#pragma unroll + for (int j = 0; j < STATE_FELTS; ++j) { + const uint32_t c = MDS_CIRC_ROW2[j + rot]; + acc_lo += (uint64_t)c * (uint64_t)lo32[j]; + acc_hi += (uint64_t)c * (uint64_t)hi32[j]; + } + // acc = acc_hi·2^32 + acc_lo, exactly. Split it at bit 64. + const uint64_t lo = (acc_hi << 32) + acc_lo; + const uint64_t carry = (lo < acc_lo) ? 1ull : 0ull; + const uint64_t hi = (acc_hi >> 32) + carry; // < 2^9 + out[i] = fadd(lo, hi * goldilocks::EPSILON); + } +#pragma unroll + for (int i = 0; i < STATE_FELTS; ++i) s[i] = out[i]; +} + +// --------------------------------------------------------------------------- +// S-boxes. +// --------------------------------------------------------------------------- + +// `x^7` in the association the AIR's degree-3 lowering uses (rpo.rs:455-460): +// `x², x³ = x²·x, x^7 = (x³)²·x`. Two squarings, two products. +__device__ __forceinline__ uint64_t sbox(uint64_t x) { + const uint64_t x2 = fmul(x, x); + const uint64_t x3 = fmul(x2, x); + const uint64_t x6 = fmul(x3, x3); + return fmul(x6, x); +} + +template +__device__ __forceinline__ uint64_t square_n(uint64_t x) { + // Rolled: the chain is serial anyway, and unrolled it is what made one + // permutation ~49k lines of PTX. The unroll factor here is a tuning knob. +#pragma unroll 1 + for (int i = 0; i < N; ++i) x = fmul(x, x); + return x; +} + +// `base^(2^M) · tail` — the inverse chain's one building block (rpo.rs:483-495). +template +__device__ __forceinline__ uint64_t exp_acc(uint64_t base, uint64_t tail) { + return fmul(square_n(base), tail); +} + +// `x^{1/7} = x^10540996611094048183` by miden-crypto's addition chain, as +// `rpo.rs:481-509` runs it lane-wise: 63 squarings + 9 products = 72 +// multiplications against ~93 for square-and-multiply. Per lane rather than +// whole-state: on a GPU the twelve lanes' independence is the compiler's to +// interleave, and a lane-wise body keeps only six values live. +__device__ __forceinline__ uint64_t inv_sbox(uint64_t x) { + const uint64_t t1 = fmul(x, x); // x^2 + const uint64_t t2 = fmul(t1, t1); // x^4 + const uint64_t t3 = exp_acc<3>(t2, t2); // x^36 + const uint64_t t4 = exp_acc<6>(t3, t3); // x^(36·65) + const uint64_t t5 = exp_acc<12>(t4, t4); // x^(36·65·4097) + const uint64_t t6 = exp_acc<6>(t5, t3); // x^0x24924924 + const uint64_t t7 = exp_acc<31>(t6, t6); // x^0x1249249224924924 + // ((t7² · t6)²)² · ((t1 · t2) · x) — rpo.rs:504-508. + const uint64_t a = square_n<2>(fmul(fmul(t7, t7), t6)); + const uint64_t b = fmul(fmul(t1, t2), x); + return fmul(a, b); +} + +// --------------------------------------------------------------------------- +// The cubic extension `GF(p)[φ] / (φ³ − φ − 1)` — rpx.rs:98-140. NOT `ext3.cuh`'s. +// --------------------------------------------------------------------------- +struct CubicExt { + uint64_t c0, c1, c2; // c0 + c1·φ + c2·φ² +}; + +// The product reduced by `φ³ = φ + 1`, `φ⁴ = φ² + φ`. `rpx.rs:118-125`'s +// closed form, regrouped so each coefficient is ONE three-term dot product +// with a single reduction (the same fold `dot_product_3` gives the VM's own +// extension): +// c0 = a0·b0 + a1·b2 + a2·b1 +// c1 = a0·b1 + a1·(b0 + b2) + a2·(b1 + b2) [= a0b1 + a1b0 + a1b2 + a2b1 + a2b2] +// c2 = a0·b2 + a1·b1 + a2·(b0 + b2) [= a0b2 + a1b1 + a2b0 + a2b2] +// Nine wide products, three reductions, two operand pre-adds. +__device__ __forceinline__ CubicExt ext_mul(const CubicExt &a, const CubicExt &b) { + const uint64_t b02 = fadd(b.c0, b.c2); + const uint64_t b12 = fadd(b.c1, b.c2); + CubicExt r; + r.c0 = fdot3(a.c0, b.c0, a.c1, b.c2, a.c2, b.c1); + r.c1 = fdot3(a.c0, b.c1, a.c1, b02, a.c2, b12); + r.c2 = fdot3(a.c0, b.c2, a.c1, b.c1, a.c2, b02); + return r; +} + +// One function for squaring and product, as on the host (rpx.rs:128-130). +__device__ __forceinline__ CubicExt ext_square(const CubicExt &a) { return ext_mul(a, a); } + +// `a^7` by `a² → a³ → a⁶ → a⁷` (rpx.rs:135-140): two squarings, two products. +__device__ __forceinline__ CubicExt ext_power7(const CubicExt &a) { + const CubicExt a2 = ext_square(a); + const CubicExt a3 = ext_mul(a2, a); + const CubicExt a6 = ext_square(a3); + return ext_mul(a6, a); +} + +// --------------------------------------------------------------------------- +// Rounds. `r` is the round index into ARK1/ARK2 — a runtime value, so one copy +// of each round body serves every round; the constant-bank address is +// computed, which costs nothing next to the round's arithmetic. Every lane +// loop is rolled for the same reason (see `permute`'s CODE SHAPE note). +// --------------------------------------------------------------------------- + +// FB: `MDS → +ARK1 → x^7 → MDS → +ARK2 → x^{1/7}` — RPO's round exactly +// (rpo.rs:561-582, rpx.rs:283-295). RPX runs it at R = 0, 2, 4; RPO at 0..7. +__device__ __forceinline__ void fb_round(uint64_t s[STATE_FELTS], int r) { + mds(s); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK1[r][i]); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = sbox(s[i]); + mds(s); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK2[r][i]); + // The twelve chains are independent; a GPU hides their latency with other + // warps, not by unrolling one thread's twelve chains into straight line. +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = inv_sbox(s[i]); +} + +// E: `+ARK1 → x^7` in the cubic extension on four lane-triples, NO linear +// layer (rpx.rs:296-307; the design, not an omission — rpx.rs:275-279). +__device__ __forceinline__ void ext_round(uint64_t s[STATE_FELTS], int r) { +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK1[r][i]); +#pragma unroll 1 + for (int e = 0; e < EXT_ELEMENTS; ++e) { + const int base = e * EXT_DEGREE; + CubicExt x; + x.c0 = s[base]; + x.c1 = s[base + 1]; + x.c2 = s[base + 2]; + const CubicExt y = ext_power7(x); + s[base] = y.c0; + s[base + 1] = y.c1; + s[base + 2] = y.c2; + } +} + +// M: `MDS → +ARK1`, a linear finish with no S-box (rpx.rs:308-313). +__device__ __forceinline__ void final_round(uint64_t s[STATE_FELTS], int r) { + mds(s); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = fadd(s[i], ARK1[r][i]); +} + +// The permutation: `FB E FB E FB E M` (rpx.rs:280-316), output CANONICAL. +// +// ★ CODE SHAPE. A real (`RPX_NOINLINE`) function with rolled loops, on +// purpose. The first cubin build of the fully inlined, fully unrolled form ran +// 41 minutes and emitted 56 MB of PTX: one permutation was ~49k straight-line +// lines (the inverse S-box chain unrolled over twelve lanes, three times) and +// every leaf kernel carried one copy per `permute` call site — seven in the +// comp-poly kernel. Rolled and called, the whole file is a few thousand lines +// and every kernel shares one body. The price is loop overhead of order 10% of +// the permutation's instructions and the state living in local memory across +// the call; the `-Xptxas -v` report and the unroll factors of `square_n` and +// the lane loops are the tuning knobs, in that order. +RPX_NOINLINE __device__ void permute(uint64_t s[STATE_FELTS]) { +#pragma unroll 1 + for (int r = 0; r + 1 < NUM_ROUNDS; r += 2) { + fb_round(s, r); + ext_round(s, r + 1); + } + final_round(s, NUM_ROUNDS - 1); +#pragma unroll 1 + for (int i = 0; i < STATE_FELTS; ++i) s[i] = goldilocks::canonical(s[i]); +} + +// --------------------------------------------------------------------------- +// The socket's two constructions over the permutation. +// --------------------------------------------------------------------------- + +// The rate-8 OVERWRITE duplex — `algebraic_commit::sponge_leaf` (:169-184) +// with `leaf_capacity` (:142-147), streamed. Capacity lane 8 carries the +// padding flag `len mod 8`, lane 9 the LEAF domain, lanes 10-11 zero. Each +// block OVERWRITES the eight rate lanes (spec §2.6): absorption is a store, no +// field arithmetic. The total length is needed BEFORE the first permutation +// (algebraic_commit.rs "A1"), hence `init(num_felts)`; callers absorb exactly +// that many felts. +struct Sponge { + uint64_t s[STATE_FELTS]; + int pos; + + __device__ __forceinline__ void init(uint64_t num_felts) { +#pragma unroll + for (int i = 0; i < RATE_FELTS; ++i) s[i] = 0; + s[CAPACITY_PAD_LANE] = num_felts % RATE_FELTS; + s[CAPACITY_DOMAIN_LANE] = DOMAIN_LEAF; + s[CAPACITY_DOMAIN_LANE + 1] = 0; + s[CAPACITY_DOMAIN_LANE + 2] = 0; + pos = 0; + } + + __device__ __forceinline__ void absorb(uint64_t felt) { + s[pos++] = felt; + if (pos == RATE_FELTS) { + permute(s); + pos = 0; + } + } + + // A pending partial block is zero-padded and permuted. An exact multiple of + // the rate spends no trailing permutation — including the EMPTY leaf, whose + // digest is therefore the untouched zero rate lanes, exactly what + // `sponge_leaf` returns for `felts.is_empty()` (:174-176). + __device__ __forceinline__ void finalize(uint64_t digest[DIGEST_FELTS]) { + if (pos != 0) { + for (int k = pos; k < RATE_FELTS; ++k) s[k] = 0; + permute(s); + pos = 0; + } +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) digest[i] = s[i]; + } +}; + +// `sponge_leaf` over a contiguous array — the one-call form for the KAT and +// for any phase-2 kernel that has its felts in hand. +__device__ __forceinline__ void sponge_leaf(const uint64_t *felts, uint64_t num_felts, + uint64_t digest[DIGEST_FELTS]) { + Sponge sp; + sp.init(num_felts); + for (uint64_t i = 0; i < num_felts; ++i) sp.absorb(felts[i]); + sp.finalize(digest); +} + +// The Merkle parent: ONE permutation of `[left ‖ right ‖ capacity]` with the +// compress domain, which is zero (algebraic_commit.rs:248-252 → +// hash.rs:95-108). Capacity = `domain_iv(0)` = all zeros. +__device__ __forceinline__ void compress(const uint64_t left[DIGEST_FELTS], + const uint64_t right[DIGEST_FELTS], + uint64_t out[DIGEST_FELTS]) { + uint64_t s[STATE_FELTS]; +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) { + s[i] = left[i]; + s[DIGEST_FELTS + i] = right[i]; + s[RATE_FELTS + i] = 0; + } + s[CAPACITY_DOMAIN_LANE] = DOMAIN_COMPRESS; + permute(s); +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) out[i] = s[i]; +} + +} // namespace rpx + +// =========================================================================== +// PHASE 2 — the device-facing surface: node bytes, leaf kernels, Merkle +// compressors and the permutation probe. Kernel for kernel the twin of +// `blake3.cu:338-620`, with the chain replaced by `rpx::Sponge` and the parent +// by `rpx::compress`. +// +// NODE BYTES. A node is four canonical felts, each stored as eight BIG-ENDIAN +// bytes — `digest_to_commitment` (algebraic_commit.rs:112-118) — so 32 bytes, +// the same slot width as a BLAKE3 or keccak node, and the device tree's bytes +// equal the host's. Digests leave `permute` canonical; a parent reads its +// children back with `commitment_to_digest`'s big-endian decoding. The device +// is little-endian, so both directions byte-swap (`bswap64`); the 32-byte node +// offsets inside a 256-byte-aligned `cuMemAlloc` buffer make the u64 accesses +// aligned, the same precondition the BLAKE3 u32 accesses rest on. +// +// A LEAF absorbs exactly the felt sequence the host leaf hashes: the same +// read pattern as the BLAKE3 kernel it twins (`leaves_bit_reversed_grouped`, +// commitment.rs:67 — bit-reversed rows, each column by column, an ext3 element +// as its three components), which is the sequence `felts_from_bytes` rebuilds +// from the leaf bytes, so `hash_bytes == hash_data` holds on device by +// construction. The felt count is known before the loop, as the overwrite +// duplex's padding flag needs it (A1). Raw `[0, 2^64)` storage is absorbed as +// is: the permutation is representation-independent, and the host +// canonicalises before serialising — same field value, same digest. +// =========================================================================== + +namespace rpx { + +// Byte-swap a u64: the device reads a host big-endian felt from a node and +// writes one back. Plain shifts so the host shim compiles it; nvcc lowers it +// to two PRMTs. +__device__ __forceinline__ uint64_t bswap64(uint64_t x) { + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x >> 8) & 0x00FF00FF00FF00FFull); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x >> 16) & 0x0000FFFF0000FFFFull); + return (x << 32) | (x >> 32); +} + +// Four felts → one 32-byte node, `digest_to_commitment`'s layout. +__device__ __forceinline__ void store_digest_be(const uint64_t digest[DIGEST_FELTS], uint8_t *node) { + uint64_t *dst = reinterpret_cast(node); +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) dst[i] = bswap64(digest[i]); +} + +// One 32-byte node → four felts, `commitment_to_digest`'s decoding. +__device__ __forceinline__ void load_digest_be(const uint8_t *node, uint64_t digest[DIGEST_FELTS]) { + const uint64_t *src = reinterpret_cast(node); +#pragma unroll + for (int i = 0; i < DIGEST_FELTS; ++i) digest[i] = bswap64(src[i]); +} + +// A Merkle parent in place in the node buffer — `parent` (algebraic_commit.rs +// :248-252): decode both children, `compress`, encode. Node buffer layout as +// `blake3.cu` / `keccak.cu` / the CPU `merkle.rs`: children at +// `nodes[parent_begin + n_pairs .. parent_begin + 3*n_pairs]`, parents at +// `nodes[parent_begin .. parent_begin + n_pairs]`, 32 bytes per node. +__device__ __forceinline__ void hash_merkle_parent(uint8_t *nodes, uint64_t parent_begin, + uint64_t n_pairs, uint64_t tid) { + uint64_t left[DIGEST_FELTS], right[DIGEST_FELTS], out[DIGEST_FELTS]; + load_digest_be(nodes + (parent_begin + n_pairs + 2 * tid) * 32, left); + load_digest_be(nodes + (parent_begin + n_pairs + 2 * tid + 1) * 32, right); + compress(left, right, out); + store_digest_be(out, nodes + (parent_begin + tid) * 32); +} + +} // namespace rpx + +// --------------------------------------------------------------------------- +// Leaf kernels. Twins of `blake3_leaves_*` / `blake3_comp_poly_leaves_ext3` / +// `blake3_fri_leaves_ext3`, argument for argument; one thread hashes one leaf. +// --------------------------------------------------------------------------- + +// Goldilocks BASE-FIELD leaf hashing, one leaf per bit-reversed row: column +// `c` of row `br` at `columns_base_ptr[c * col_stride + br]`. +// Twin of `blake3_leaves_base_batched` (`blake3.cu:346`). +extern "C" __global__ void rpx_leaves_base_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(num_cols); + for (uint64_t c = 0; c < num_cols; ++c) sp.absorb(columns_base_ptr[c * col_stride + br]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// BASE-FIELD row-pair leaf hashing: leaf `tid` hashes bit-reversed rows +// `2*tid` and `2*tid+1`, each column by column, first row then second. +// `num_leaves = num_rows / 2`. Twin of `blake3_leaves_base_row_pair_batched`. +extern "C" __global__ void rpx_leaves_base_row_pair_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(2 * num_cols); + for (uint64_t c = 0; c < num_cols; ++c) sp.absorb(columns_base_ptr[c * col_stride + br_0]); + for (uint64_t c = 0; c < num_cols; ++c) sp.absorb(columns_base_ptr[c * col_stride + br_1]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// EXT3 leaf hashing, one leaf per bit-reversed row, components in three +// separate base slabs: column `c` component `k` at +// `columns_base_ptr[(c*3 + k) * col_stride + br]`; an element is absorbed as +// `[comp0, comp1, comp2]`, matching `write_bytes_be`. +// Twin of `blake3_leaves_ext3_batched`. +extern "C" __global__ void rpx_leaves_ext3_batched( + const uint64_t *columns_base_ptr, + uint64_t col_stride, + uint64_t num_cols, // number of ext3 columns (NOT slabs) + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_rows) return; + uint64_t br = __brevll(tid) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(3 * num_cols); + for (uint64_t c = 0; c < num_cols; ++c) { +#pragma unroll + for (int k = 0; k < 3; ++k) { + sp.absorb(columns_base_ptr[(c * 3 + (uint64_t)k) * col_stride + br]); + } + } + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// Composition-polynomial leaf hashing: each leaf absorbs `2 * num_parts` ext3 +// values from bit-reversed rows `2*tid` and `2*tid+1`, (row 0: parts) then +// (row 1: parts), three base components per value. +// Twin of `blake3_comp_poly_leaves_ext3`. +extern "C" __global__ void rpx_comp_poly_leaves_ext3( + const uint64_t *parts_base_ptr, + uint64_t col_stride, + uint64_t num_parts, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + + rpx::Sponge sp; + sp.init(2 * 3 * num_parts); + for (uint64_t p = 0; p < num_parts; ++p) { +#pragma unroll + for (int k = 0; k < 3; ++k) { + sp.absorb(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_0]); + } + } + for (uint64_t p = 0; p < num_parts; ++p) { +#pragma unroll + for (int k = 0; k < 3; ++k) { + sp.absorb(parts_base_ptr[(p * 3 + (uint64_t)k) * col_stride + br_1]); + } + } + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, leaves_out + tid * 32); +} + +// FRI layer leaf hashing: each leaf absorbs two consecutive ext3 values from an +// interleaved eval vector `[a0,a1,a2,b0,b1,b2,...]` — six felts, so a single +// block, no padding flag (`6 mod 8 = 6` in capacity lane 8). No bit reversal. +// The host is `AlgebraicPairBackend::hash_data` (algebraic_commit.rs:318-329). +// Twin of `blake3_fri_leaves_ext3`. +extern "C" __global__ void rpx_fri_leaves_ext3( + const uint64_t *evals_interleaved, // 3 * num_evals u64s + uint64_t num_leaves, // = num_evals / 2 + uint8_t *leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + const uint64_t *pair = evals_interleaved + 2 * tid * 3; + + rpx::Sponge sp; + sp.init(6); +#pragma unroll + for (int i = 0; i < 6; ++i) sp.absorb(pair[i]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, leaves_out + tid * 32); +} + +// Row-major ROW-PAIR leaf hashing: leaf `tid` absorbs row `reverse_index(2*tid)` +// then row `reverse_index(2*tid+1)`, each `m` lanes read contiguously from +// `data + br * m`. `m` is the row stride in u64s: base trace = column count, +// ext3 trace = 3 * column count (an ext3 element's components are consecutive). +// Twin of `blake3_leaves_base_row_major_row_pair`; the fused LDE+commit +// pipeline's leaf kernel (`lde.rs` `coset_lde_row_major_inner`). +extern "C" __global__ void rpx_leaves_base_row_major_row_pair( + const uint64_t *data, + uint64_t m, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + rpx::Sponge sp; + sp.init(2 * m); + for (uint64_t c = 0; c < m; ++c) sp.absorb(row_0[c]); + for (uint64_t c = 0; c < m; ++c) sp.absorb(row_1[c]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// Column-range variant: each leaf absorbs only columns `[col_start, col_end)` +// of the two bit-reversed rows while `m` stays the full row stride — the CPU +// `commit_rows_bit_reversed_subset`, how preprocessed tables commit their +// precomputed and multiplicity column ranges to separate trees over one LDE. +// Twin of `blake3_leaves_base_row_major_row_pair_range`. +extern "C" __global__ void rpx_leaves_base_row_major_row_pair_range( + const uint64_t *data, + uint64_t m, + uint64_t col_start, + uint64_t col_end, + uint64_t num_rows, + uint64_t log_num_rows, + uint8_t *hashed_leaves_out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t num_leaves = num_rows >> 1; + if (tid >= num_leaves) return; + uint64_t br_0 = __brevll(2 * tid) >> (64 - log_num_rows); + uint64_t br_1 = __brevll(2 * tid + 1) >> (64 - log_num_rows); + const uint64_t *row_0 = data + br_0 * m; + const uint64_t *row_1 = data + br_1 * m; + + rpx::Sponge sp; + sp.init(2 * (col_end - col_start)); + for (uint64_t c = col_start; c < col_end; ++c) sp.absorb(row_0[c]); + for (uint64_t c = col_start; c < col_end; ++c) sp.absorb(row_1[c]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, hashed_leaves_out + tid * 32); +} + +// --------------------------------------------------------------------------- +// COSET leaf hashing — the WHIR shape, and the two kernels the per-table branch +// has no twin for. +// +// Every other leaf kernel in this file hashes a ROW GROUP: a leaf is a row (or +// a row pair) read across the columns. WHIR's leaf is a fold COSET: leaf `j` +// holds the `2^log_folding` codeword positions that fold onto `j`, which are +// strided by `num_leaves`. Twins of `keccak256_leaves_base_coset` / +// `keccak256_leaves_ext3_coset` (`keccak.cu`), argument for argument, with the +// sponge swapped. +// +// The felt count is known before the loop, which the overwrite duplex needs for +// its padding flag: a base leaf is `block` felts, an ext3 leaf `3 * block`. Raw +// `[0, 2^64)` storage is absorbed as is — the permutation is +// representation-independent and the host canonicalises before serialising, so +// the same field value gives the same digest either way. +// --------------------------------------------------------------------------- + +// Goldilocks BASE-FIELD coset leaves: leaf `tid` hashes +// `codeword[tid + t * num_leaves]` for `t` in `[0, block)`. +extern "C" __global__ void rpx_leaves_base_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + rpx::Sponge sp; + sp.init(block); + for (uint64_t t = 0; t < block; ++t) sp.absorb(codeword[tid + t * num_leaves]); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, out + tid * 32); +} + +// EXT3 coset leaves: the same stride, each element as its three components in +// order — what `element_felts` produces for a cubic-extension element, and what +// the base kernel above does one component at a time. +extern "C" __global__ void rpx_leaves_ext3_coset(const uint64_t *__restrict__ codeword, + uint64_t num_leaves, uint64_t block, + uint8_t *__restrict__ out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_leaves) return; + + rpx::Sponge sp; + sp.init(block * 3); + for (uint64_t t = 0; t < block; ++t) { + const uint64_t *at = codeword + (tid + t * num_leaves) * 3; +#pragma unroll + for (int k = 0; k < 3; ++k) sp.absorb(at[k]); + } + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + rpx::store_digest_be(digest, out + tid * 32); +} + +// --------------------------------------------------------------------------- +// Merkle level / tail. Same launch split as BLAKE3's: one thread per pair per +// level while a level is wide, then ONE single-block launch that grid-strides +// every remaining level with a barrier between them. +// --------------------------------------------------------------------------- + +// One level of the inner tree: each thread compresses one child pair. +extern "C" __global__ void rpx_merkle_level(uint8_t *nodes, + uint64_t parent_begin, // in 32-byte nodes + uint64_t n_pairs) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n_pairs) return; + rpx::hash_merkle_parent(nodes, parent_begin, n_pairs, tid); +} + +// Every remaining level from `level_begin` up to the root, in one block. +// Twin of `blake3_merkle_tail`. +extern "C" __global__ void rpx_merkle_tail(uint8_t *nodes, uint64_t level_begin) { + uint64_t lb = level_begin; + while (lb != 0) { + uint64_t nb = lb / 2; + uint64_t n_pairs = lb - nb; + for (uint64_t tid = threadIdx.x; tid < n_pairs; tid += blockDim.x) { + rpx::hash_merkle_parent(nodes, nb, n_pairs, tid); + } + __syncthreads(); + lb = nb; + } +} + +// --------------------------------------------------------------------------- +// Parity-harness entry point: `n` independent permutations, one thread each. +// The bare device permutation is otherwise unreachable from host code; this is +// what lets the GPU be checked against the host `Rpx256` (and the host-KAT's +// oracle tables) before any tree is built. Not on any production path. +// --------------------------------------------------------------------------- +extern "C" __global__ void rpx_permute_probe(const uint64_t *states, uint64_t n, uint64_t *out) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= n) return; + uint64_t s[rpx::STATE_FELTS]; +#pragma unroll + for (int i = 0; i < rpx::STATE_FELTS; ++i) s[i] = states[tid * rpx::STATE_FELTS + i]; + rpx::permute(s); +#pragma unroll + for (int i = 0; i < rpx::STATE_FELTS; ++i) out[tid * rpx::STATE_FELTS + i] = s[i]; +} + +// --------------------------------------------------------------------------- +// Proof-of-work grinding search, RPX arm. +// +// Twin of `keccak.cu`'s `grind_search`, same signature shape and the same +// first-hit reduction; only the outer hash differs. The host path it replaces +// is `stark::grinding::generate_nonce`, a per-table ~2^grinding_factor search +// that is the prover's dominant CPU cost once the transcript is algebraic. +// +// THE MAPPING this reproduces, derived from the host (✓ VERIFIED against the +// sources named) and stated here so it is not re-derived at each reading: +// +// host predicate stark/src/grinding.rs::is_valid_nonce_for_inner_hash: +// valid ⇔ u64::from_be_bytes(D::digest(inner ‖ nonce.to_be_bytes())[..8]) < limit, +// limit = 1 << (64 − grinding_factor); inner = D::digest(PREFIX ‖ seed ‖ factor), +// 41 bytes, computed ONCE per table on the host and never on device. +// D for RPX prover/src/lfm/algebraic_commit.rs AlgebraicDigest: +// D::digest(bytes) = digest_to_commitment(sponge_leaf(Rpx, felts_from_bytes(bytes))) +// — the LEAF construction, on purpose. +// bytes → felts felts_from_bytes: consecutive 8-byte groups, each +// FE::from(u64::from_be_bytes(group)) — BIG-endian, and `FE::from` is +// `from_u64`, which maps a raw value ≥ p to raw − p (ONE subtraction, +// goldilocks.rs:172-178 — exactly `goldilocks::canonical`). +// The 40-byte outer block is therefore EXACTLY five felts: +// f0..f3 = the inner hash's four big-endian u64s (canonical already — +// they are digest_to_commitment output, so each < p), +// f4 = the nonce. +// ⚠ Big-endian, unlike keccak's `inner_hash_lanes` (LITTLE-endian lanes). +// A separate host helper, `stark::grinding::inner_hash_felts` (BE), feeds +// this kernel; feeding it the keccak lanes is a silent wrong hash. +// sponge mode sponge_leaf over five felts: ONE permutation of +// [f0, f1, f2, f3, nonce, 0, 0, 0 | 5, 0x4C4D464C, 0, 0] +// — rate lanes 5..8 zero-padded, capacity lane 8 = padding flag +// `5 mod 8 = 5`, lane 9 = DOMAIN_LEAF ("LFML"), lanes 10, 11 = 0. +// ★ Built here through `rpx::Sponge` — `init(5)`, five `absorb`s, +// `finalize` — rather than by writing those twelve lanes out, so the +// capacity rule has ONE statement on device and a change to it cannot +// leave the grind behind. +// the head digest_to_commitment writes lane 0 CANONICAL as 8 big-endian bytes and +// the host reads those 8 bytes back big-endian, so `seed_head` IS the +// canonical value of state lane 0 after the permutation — no byte +// reinterpretation. `permute` canonicalises its output, so on device the +// predicate is just `digest[0] < limit`. +// +// THE NONCE LANE is `goldilocks::canonical(nonce)`, matching `FE::from(nonce)` +// exactly. It is a no-op for every nonce this search can reach (the first +// nonce ≥ p is 2^64 − 2^32 + 1, and the launcher's range walk bails long +// before), and the device representation is lazy anyway — but absorbing the +// canonical value is what makes "the device absorbs what `FE::from` produces" +// true by inspection rather than by an argument about reachability. +// +// Each thread strides over `[base, base+count)` and `atomicMin`s the smallest +// valid nonce it finds into `*result` (initialised to U64_MAX by the caller), +// so the launch returns the globally smallest valid nonce in the searched +// block — deterministic despite the parallel grid, and any valid nonce +// satisfies the verifier. +// --------------------------------------------------------------------------- + +// The outer block is `inner_hash ‖ nonce`: 40 bytes, five felts. Named because +// the capacity's padding flag is `5 mod 8` and the count is what `init` needs. +__device__ constexpr uint64_t GRIND_FELTS = 5; + +extern "C" __global__ void rpx_grind_search(const uint64_t *inner_felts, + uint64_t limit, + uint64_t base, + uint64_t count, + volatile unsigned long long *result) { + uint64_t tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; + uint64_t stride = (uint64_t)gridDim.x * blockDim.x; + const uint64_t f0 = inner_felts[0], f1 = inner_felts[1], f2 = inner_felts[2], + f3 = inner_felts[3]; + for (uint64_t i = tid; i < count; i += stride) { + uint64_t nonce = base + i; + // Guard the u64 wrap on the final block (the launcher bails before it, + // so this is unreachable in practice): a wrapped nonce is < base, so + // stop rather than re-scan from 0. + if (nonce < base) break; + // A thread's nonces only increase, so once a smaller valid one is known + // this thread can never beat it — stop scanning. `result` is volatile + // so this load re-reads L2 (where the atomicMin writes land) instead of + // being hoisted into a register or served stale from L1; the early exit + // depends on that, though correctness does not. + if (nonce >= (uint64_t)*result) break; + rpx::Sponge sp; + sp.init(GRIND_FELTS); + sp.absorb(f0); + sp.absorb(f1); + sp.absorb(f2); + sp.absorb(f3); + sp.absorb(goldilocks::canonical(nonce)); + uint64_t digest[rpx::DIGEST_FELTS]; + sp.finalize(digest); + if (digest[0] < limit) { + atomicMin((unsigned long long *)result, (unsigned long long)nonce); + } + } +} diff --git a/crypto/math-cuda/src/device.rs b/crypto/math-cuda/src/device.rs index 76782879e..b2a5d81f8 100644 --- a/crypto/math-cuda/src/device.rs +++ b/crypto/math-cuda/src/device.rs @@ -128,6 +128,7 @@ impl Drop for PinnedStaging { const ARITH_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/arith.cubin")); const NTT_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ntt.cubin")); const KECCAK_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/keccak.cubin")); +const RPX_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rpx.cubin")); const BARY_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/barycentric.cubin")); const DEEP_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/deep.cubin")); const FRI_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/fri.cubin")); @@ -215,6 +216,18 @@ pub struct Backend { pub keccak_merkle_tail: CudaFunction, pub merkle_gather_paths: CudaFunction, + // rpx.cubin — the algebraic hash's twins of the keccak entries above. + // Only the ones the WHIR path reaches are bound: the coset leaves, the two + // tree compressors and the grind. The row-group leaf kernels the per-table + // branch uses are in the cubin but are not loaded here, because nothing on + // this path launches them and an unused handle is a claim that something + // does. + pub rpx_leaves_base_coset: CudaFunction, + pub rpx_leaves_ext3_coset: CudaFunction, + pub rpx_merkle_level: CudaFunction, + pub rpx_merkle_tail: CudaFunction, + pub rpx_grind_search: CudaFunction, + // barycentric.cubin pub barycentric_base_batched: CudaFunction, pub barycentric_ext3_batched: CudaFunction, @@ -321,15 +334,71 @@ fn retain_default_mempool(ctx: &CudaContext) { } /// Device bytes held for as long as this lives. See [`Backend::reserve`]. +/// +/// ★ The count is atomic and the reservation is GROWABLE, because a chain +/// promises its room once and then discovers more of it: the tree a commitment +/// caches is not known when the codeword reserves, and it is shared through an +/// `Arc` by the time it is. Growing this rather than taking a second +/// reservation is what keeps ONE number answering "what does this chain hold" — +/// two accountings for one working set is a shape this codebase has paid for +/// before. #[derive(Debug)] pub struct DeviceReservation { - bytes: u64, + bytes: AtomicU64, +} + +impl DeviceReservation { + /// Bytes this reservation currently accounts for. + pub fn bytes(&self) -> u64 { + self.bytes.load(Ordering::Relaxed) + } + + /// Promise `extra` more against the same budget, under this reservation. + /// + /// Returns false and changes nothing if the budget will not take it — the + /// caller then does without whatever it wanted the bytes for, rather than + /// holding memory the accounting cannot see. + pub fn grow(&self, extra: u64) -> bool { + let Ok(be) = backend() else { return false }; + let mut held = be.reserved.load(Ordering::Relaxed); + loop { + if held.saturating_add(extra) > be.vram_budget_bytes { + return false; + } + match be.reserved.compare_exchange_weak( + held, + held + extra, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => { + self.bytes.fetch_add(extra, Ordering::Relaxed); + return true; + } + Err(seen) => held = seen, + } + } + } + + /// Give `given` of them back, when what they were promised for is dropped + /// before the reservation is. + pub fn shrink(&self, given: u64) { + let given = given.min(self.bytes.load(Ordering::Relaxed)); + if given == 0 { + return; + } + self.bytes.fetch_sub(given, Ordering::Relaxed); + if let Ok(be) = backend() { + be.reserved.fetch_sub(given, Ordering::Relaxed); + } + } } impl Drop for DeviceReservation { fn drop(&mut self) { if let Ok(be) = backend() { - be.reserved.fetch_sub(self.bytes, Ordering::Relaxed); + be.reserved + .fetch_sub(self.bytes.load(Ordering::Relaxed), Ordering::Relaxed); } } } @@ -365,7 +434,11 @@ fn trim_default_mempool() { /// free until that stream reaches the drop — and the pool cannot return what /// it has not been given yet. Draining the whole context first is what makes /// the trim worth doing. -fn drain_and_trim() -> Result<()> { +/// +/// Public because a test that asks the driver how much memory this process has +/// taken needs the pool empty first, or it is measuring the pool's retention +/// instead of the caller's — see `a_group_holds_only_its_codewords_before_any_open`. +pub fn drain_and_trim() -> Result<()> { let be = backend()?; be.ctx.synchronize()?; trim_default_mempool(); @@ -494,6 +567,7 @@ impl Backend { let arith = ctx.load_module(Ptx::from_binary(ARITH_CUBIN.to_vec()))?; let ntt = ctx.load_module(Ptx::from_binary(NTT_CUBIN.to_vec()))?; let keccak = ctx.load_module(Ptx::from_binary(KECCAK_CUBIN.to_vec()))?; + let rpx = ctx.load_module(Ptx::from_binary(RPX_CUBIN.to_vec()))?; let bary = ctx.load_module(Ptx::from_binary(BARY_CUBIN.to_vec()))?; let deep = ctx.load_module(Ptx::from_binary(DEEP_CUBIN.to_vec()))?; let fri = ctx.load_module(Ptx::from_binary(FRI_CUBIN.to_vec()))?; @@ -596,6 +670,11 @@ impl Backend { keccak_merkle_level: keccak.load_function("keccak_merkle_level")?, keccak_merkle_tail: keccak.load_function("keccak_merkle_tail")?, merkle_gather_paths: keccak.load_function("merkle_gather_paths")?, + rpx_leaves_base_coset: rpx.load_function("rpx_leaves_base_coset")?, + rpx_leaves_ext3_coset: rpx.load_function("rpx_leaves_ext3_coset")?, + rpx_merkle_level: rpx.load_function("rpx_merkle_level")?, + rpx_merkle_tail: rpx.load_function("rpx_merkle_tail")?, + rpx_grind_search: rpx.load_function("rpx_grind_search")?, barycentric_base_batched: bary.load_function("barycentric_base_batched")?, barycentric_ext3_batched: bary.load_function("barycentric_ext3_batched")?, barycentric_base_batched_strided: bary @@ -672,6 +751,31 @@ impl Backend { self.vram_budget_bytes } + /// Bytes the device reports free, right now. + /// + /// The driver's own accounting rather than this module's: [`reserve`] + /// counts what callers PROMISED, which is silent about anything allocated + /// without a reservation. A test that wants to know whether a structure is + /// holding device memory it never declared has to ask the device, and this + /// is how — see `a_group_holds_only_its_codewords_before_any_open`. + /// + /// ⚠ The stream-ordered pool retains freed blocks, so this falls as memory + /// is used and does not always rise as it is released. It answers "how + /// much has this process taken from the device", not "how much is live", + /// which is the question a retention test is asking. + pub fn free_vram_bytes(&self) -> Result { + use cudarc::driver::sys; + self.ctx.bind_to_thread()?; + // SAFETY: a raw driver query writing into two stack slots, with the + // context bound to this thread on the line above. + unsafe { + let mut free: usize = 0; + let mut total: usize = 0; + sys::cuMemGetInfo_v2(&mut free as *mut usize, &mut total as *mut usize).result()?; + Ok(free as u64) + } + } + /// Promises `bytes` of the device to something about to be built there, or /// refuses. /// @@ -696,12 +800,23 @@ impl Backend { Ordering::Relaxed, Ordering::Relaxed, ) { - Ok(_) => return Some(DeviceReservation { bytes }), + Ok(_) => { + return Some(DeviceReservation { + bytes: AtomicU64::new(bytes), + }); + } Err(seen) => held = seen, } } } + /// Bytes promised across every live reservation — what `reserve` checks + /// the budget against. Exposed so a test can assert the number rather than + /// assert that nothing crashed. + pub fn reserved_bytes(&self) -> u64 { + self.reserved.load(Ordering::Relaxed) + } + /// Round-robin over the stream pool. Concurrent callers get different /// streams so their kernel launches overlap on the GPU. pub fn next_stream(&self) -> Arc { diff --git a/crypto/math-cuda/src/fri.rs b/crypto/math-cuda/src/fri.rs index 533ff6e32..e0a523334 100644 --- a/crypto/math-cuda/src/fri.rs +++ b/crypto/math-cuda/src/fri.rs @@ -182,7 +182,13 @@ impl FriCommitState { .launch(kcfg)?; } } - build_inner_tree_levels(self.stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels( + self.stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; // Update inv_twiddles for the next layer: `new[j] = old[2j]^2` for // j in 0..n_out/2. (If n_out == 1, skip; no next fold.) Writes into diff --git a/crypto/math-cuda/src/grinding.rs b/crypto/math-cuda/src/grinding.rs index fe7803eb9..1a83a859a 100644 --- a/crypto/math-cuda/src/grinding.rs +++ b/crypto/math-cuda/src/grinding.rs @@ -1,7 +1,25 @@ -//! GPU proof-of-work grinding: a parallel Keccak nonce search that mirrors the -//! host `stark::grinding::generate_nonce`, offloading the ~2^grinding_factor -//! hashes it does per table per epoch from the CPU (where they dominate the -//! prove) to the otherwise-idle GPU. +//! GPU proof-of-work grinding: a parallel nonce search that mirrors the host +//! `crypto::grinding::generate_nonce`, offloading the ~2^grinding_factor hashes +//! it does per grind from the CPU — where they dominate the prove — to the +//! otherwise-idle GPU. +//! +//! Two arms, one per outer hash: [`generate_nonce_gpu`] for keccak-256 and +//! [`generate_nonce_rpx_gpu`] for RPX256. They differ in the kernel and in **how +//! the 32-byte inner hash is read into four `u64`s** — LITTLE-endian lanes for +//! keccak, BIG-endian felts for RPX. Everything else (the min-factor gate, the +//! block sizing, the sentinel loop, the first-hit reduction) is one policy, +//! written once in [`search`]. +//! +//! # ⚠ Why the two entry points are named rather than flagged +//! +//! The endianness is the whole difference and it is not cosmetic. An algebraic +//! digest reads consecutive eight-byte groups big-endian, so those four `u64`s +//! ARE the felts the host sponge absorbs; keccak reads its lanes little-endian. +//! Crossing them compiles, runs, and searches for a nonce under a message the +//! host never hashes: every returned nonce fails the host check, the prover +//! falls back to the CPU on every grind, and nothing says so louder than one +//! warning line. Two functions with two doc comments is the cheapest way to +//! make that mistake hard to type. use cudarc::driver::{LaunchConfig, PushKernelArg}; @@ -10,31 +28,80 @@ use crate::device::backend; const BLOCK_DIM: u32 = 256; const GRID_DIM: u32 = 1024; +/// Threads per block for the RPX arm. +/// +/// Half keccak's, for the reason every RPX kernel in this crate launches +/// narrow: a thread carries a twelve-lane `u64` state plus the inverse S-box's +/// live temporaries across a non-inlined `permute` call, so occupancy is bought +/// with registers rather than threads. +const RPX_BLOCK_DIM: u32 = 128; + /// Below this grinding factor the CPU search finds a valid nonce in well under /// a microsecond, so a device launch + shared-stream `synchronize` (which also /// stalls whatever a rayon peer queued on that stream) is pure loss. Bounce /// those to the CPU. The production factor is 20; only tests use tiny factors. -const GRIND_MIN_FACTOR: u8 = 12; +pub const GRIND_MIN_FACTOR: u8 = 12; + +/// Which outer hash the search runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Arm { + Keccak256, + Rpx256, +} -/// Smallest nonce whose grind head is `< limit`, or `None` when the CUDA path -/// is unavailable/errors (the caller then runs the CPU search). +/// Smallest nonce whose keccak grind head is `< limit`, or `None` when the CUDA +/// path is unavailable/errors (the caller then runs the CPU search). /// -/// `inner_lanes` are the four little-endian-read u64 lanes of the 32-byte -/// inner hash — build them with `stark::grinding::inner_hash_lanes`, which is -/// what the prover and the tests here both call. `grinding_factor` (1..=64) -/// fixes `limit = 1 << (64 - grinding_factor)` and sizes the search: the -/// expected first valid nonce is ~`2^grinding_factor`, so each launch scans a -/// contiguous block several times that, from 0 upward, and the first block that -/// hits yields the globally smallest valid nonce (the kernel `atomicMin`s it). +/// `inner_lanes` are the four **little-endian**-read `u64` lanes of the 32-byte +/// inner hash — build them with `crypto::grinding::inner_hash_lanes`. pub fn generate_nonce_gpu(inner_lanes: &[u64; 4], grinding_factor: u8) -> Option { + search(Arm::Keccak256, inner_lanes, grinding_factor) +} + +/// Smallest nonce whose RPX grind head is `< limit`, or `None` when the CUDA +/// path is unavailable/errors (the caller then runs the CPU search). +/// +/// ⚠ `inner_felts` are the four **big-endian**-read `u64`s of the 32-byte inner +/// hash — build them with `crypto::grinding::inner_hash_felts`, never with +/// `inner_hash_lanes`. See the module header for what crossing them does. +/// +/// # The preimage, stated where the kernel is called +/// +/// The host predicate hashes `inner_hash ‖ nonce.to_be_bytes()` — **40 bytes, +/// which is five felts, which is one rate-8 block and therefore exactly one +/// permutation**. Its capacity is `leaf_capacity(5)`: lane 0 the padding flag +/// `5 mod 8 = 5`, lane 1 the LEAF domain. `rpx_grind_search` builds the same +/// block — `init(5)`, absorb `f0..f3` then `canonical(nonce)` — and compares +/// `digest[0] < limit`, which is the same number the host compares because +/// `u64::from_be_bytes(digest[..8])` IS felt 0's canonical value. +/// +/// The four `inner_felts` need no reduction on either side: they are an +/// algebraic digest's own output, which `digest_to_commitment` writes as four +/// canonical big-endian `u64`s. +pub fn generate_nonce_rpx_gpu(inner_felts: &[u64; 4], grinding_factor: u8) -> Option { + search(Arm::Rpx256, inner_felts, grinding_factor) +} + +/// The range walk both arms share. +/// +/// `grinding_factor` (1..=64) fixes `limit = 1 << (64 - grinding_factor)` and +/// sizes the search: the expected first valid nonce is ~`2^grinding_factor`, so +/// each launch scans a contiguous block several times that, from 0 upward, and +/// the first block that hits yields the globally smallest valid nonce (the +/// kernels `atomicMin` it). +fn search(arm: Arm, inner: &[u64; 4], grinding_factor: u8) -> Option { if !(GRIND_MIN_FACTOR..=64).contains(&grinding_factor) { return None; } let limit: u64 = 1u64 << (64 - grinding_factor); let be = backend().ok()?; + let (kernel, block_dim) = match arm { + Arm::Keccak256 => (&be.grind_search, BLOCK_DIM), + Arm::Rpx256 => (&be.rpx_grind_search, RPX_BLOCK_DIM), + }; let stream = be.next_stream(); - let inner_dev = stream.clone_htod(inner_lanes.as_slice()).ok()?; + let inner_dev = stream.clone_htod(inner.as_slice()).ok()?; // Per-launch block size: ~8× the expected hit distance, clamped so tiny // factors still launch a full grid and huge factors don't ask for an @@ -45,7 +112,7 @@ pub fn generate_nonce_gpu(inner_lanes: &[u64; 4], grinding_factor: u8) -> Option let cfg = LaunchConfig { grid_dim: (GRID_DIM, 1, 1), - block_dim: (BLOCK_DIM, 1, 1), + block_dim: (block_dim, 1, 1), shared_mem_bytes: 0, }; @@ -60,7 +127,7 @@ pub fn generate_nonce_gpu(inner_lanes: &[u64; 4], grinding_factor: u8) -> Option stream.memcpy_htod(&sentinel, &mut result_dev).ok()?; unsafe { stream - .launch_builder(&be.grind_search) + .launch_builder(kernel) .arg(&inner_dev) .arg(&limit) .arg(&base) diff --git a/crypto/math-cuda/src/lde.rs b/crypto/math-cuda/src/lde.rs index 9bbd9958d..927f22cd0 100644 --- a/crypto/math-cuda/src/lde.rs +++ b/crypto/math-cuda/src/lde.rs @@ -603,7 +603,13 @@ fn coset_lde_row_major_inner( &mut leaves_view, )?; } - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + crate::merkle::build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; // Copy the 32-byte root BEFORE queueing the big drain/transpose: this // pageable copy host-blocks until everything queued so far lands, so @@ -779,7 +785,13 @@ pub fn coset_lde_row_major_split_trees( &mut leaves_view, )?; } - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + crate::merkle::build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; Ok(nodes_dev) }; @@ -1570,7 +1582,13 @@ fn coset_lde_batch_base_into_with_merkle_tree_inner( } if commit == KeccakCommit::FullTree { - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + crate::merkle::build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; } // Release the staging slot before the drain: the uploads have landed once @@ -1788,7 +1806,13 @@ fn evaluate_poly_coset_batch_ext3_into_inner( .launch(cfg)?; } } - crate::merkle::build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + crate::merkle::build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; Some((nodes_dev, nodes_out)) } else { None diff --git a/crypto/math-cuda/src/lib.rs b/crypto/math-cuda/src/lib.rs index d57f6461c..cc2148ac2 100644 --- a/crypto/math-cuda/src/lib.rs +++ b/crypto/math-cuda/src/lib.rs @@ -5,6 +5,35 @@ //! Everything else (`ntt`, element-wise arith) is either internal to those //! pipelines or used by the parity test suite. +/// ★ Which hash family a device tree entry point must run. +/// +/// The dispatch key the callers hand down, and the reason it exists: on the +/// host a Merkle backend both NAMES a hash and computes it, so a tree cannot +/// wear a name its own code did not produce. On the device the backend only +/// names it — the kernels do the hashing — so without a key travelling with the +/// request, a tree labelled RPX could be built by keccak's kernels and nothing +/// would notice. Every entry point that hashes takes one of these, and every +/// launch site matches on it exhaustively, so a hash added here is a compile +/// error at each site rather than a silent fallthrough to a default. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DeviceHash { + /// Keccak-256 at both the leaf and the parent layer. + Keccak256, + /// RPX256 (XHash12) at both layers — the rate-8 overwrite duplex for + /// leaves, one permutation of `[l || r || 0^4]` for parents. + Rpx256, +} + +impl DeviceHash { + /// The name a tree built under this key may be called by. + pub const fn name(self) -> &'static str { + match self { + Self::Keccak256 => "keccak256", + Self::Rpx256 => "rpx256", + } + } +} + pub mod barycentric; pub mod columns; pub mod constraint_interp; diff --git a/crypto/math-cuda/src/merkle.rs b/crypto/math-cuda/src/merkle.rs index 02532f6de..7741b082b 100644 --- a/crypto/math-cuda/src/merkle.rs +++ b/crypto/math-cuda/src/merkle.rs @@ -152,12 +152,23 @@ pub(crate) fn keccak_launch_cfg(num_rows: u64) -> LaunchConfig { /// `log2(leaves_len)` times invoking `keccak_merkle_level` to fill in the /// inner nodes from the bottom up. Mirrors the CPU `build(nodes, leaves_len)` /// scan in `crypto/crypto/src/merkle_tree/merkle.rs`. +/// Build every inner level of a Merkle tree, under the hash `hash` names. +/// +/// The two kernel families have identical signatures and identical node layout, +/// so the dispatch is a choice of handle and nothing else — which is the whole +/// reason an algebraic hash costs two kernels here rather than a second tree +/// builder. pub(crate) fn build_inner_tree_levels( stream: &CudaStream, be: &Backend, nodes_dev: &mut CudaSlice, leaves_len: usize, + hash: crate::DeviceHash, ) -> Result<()> { + let (level_fn, tail_fn) = match hash { + crate::DeviceHash::Keccak256 => (&be.keccak_merkle_level, &be.keccak_merkle_tail), + crate::DeviceHash::Rpx256 => (&be.rpx_merkle_level, &be.rpx_merkle_tail), + }; // Once a level fits this many pairs, one single-block launch // (`keccak_merkle_tail`) builds all remaining levels with barriers // between them: the top levels of a big tree are each smaller than the @@ -186,7 +197,7 @@ pub(crate) fn build_inner_tree_levels( }; unsafe { stream - .launch_builder(&be.keccak_merkle_tail) + .launch_builder(tail_fn) .arg(&mut *nodes_dev) .arg(&level_begin) .launch(cfg)?; @@ -196,7 +207,7 @@ pub(crate) fn build_inner_tree_levels( let cfg = keccak_launch_cfg(n_pairs); unsafe { stream - .launch_builder(&be.keccak_merkle_level) + .launch_builder(level_fn) .arg(&mut *nodes_dev) .arg(&new_begin) .arg(&n_pairs) @@ -341,7 +352,13 @@ pub fn build_merkle_tree_on_device(hashed_leaves: &[u8]) -> Result> { stream.memcpy_htod(hashed_leaves, &mut slice)?; } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, leaves_len)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + leaves_len, + crate::DeviceHash::Keccak256, + )?; let out = stream.clone_dtoh(&nodes_dev)?; stream.synchronize()?; @@ -483,7 +500,13 @@ fn build_comp_poly_tree_nodes_dev( } } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; Ok((nodes_dev, num_leaves, stream)) } @@ -528,7 +551,13 @@ pub fn build_comp_poly_tree_from_slabs_dev( .launch(cfg)?; } } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; let mut root = [0u8; 32]; stream.memcpy_dtoh(&nodes_dev.slice(0..32), &mut root)?; stream.synchronize()?; @@ -598,7 +627,13 @@ pub fn build_fri_layer_tree_from_evals_ext3(evals: &[u64]) -> Result> { } } - build_inner_tree_levels(stream.as_ref(), be, &mut nodes_dev, num_leaves)?; + build_inner_tree_levels( + stream.as_ref(), + be, + &mut nodes_dev, + num_leaves, + crate::DeviceHash::Keccak256, + )?; let out = stream.clone_dtoh(&nodes_dev)?; stream.synchronize()?; diff --git a/crypto/math-cuda/src/whir.rs b/crypto/math-cuda/src/whir.rs index 242b4ae86..cc8f282f0 100644 --- a/crypto/math-cuda/src/whir.rs +++ b/crypto/math-cuda/src/whir.rs @@ -9,8 +9,38 @@ use std::sync::Arc; use cudarc::driver::{CudaSlice, CudaStream, LaunchConfig, PushKernelArg}; +use core::sync::atomic::{AtomicU64, Ordering}; + use crate::Result; use crate::device::{alloc_or_trim, backend}; + +/// Leaf-hash passes over a codeword — one per tree actually built. +/// +/// The quantity H4 is about: a commitment that is opened used to cost TWO of +/// these, one for the root and one for the paths. It counts launches, not +/// leaves, because "how many times was this codeword's leaf layer hashed" is +/// the question, and a test can assert an integer. +static LEAF_HASH_CALLS: AtomicU64 = AtomicU64::new(0); + +pub fn leaf_hash_calls() -> u64 { + LEAF_HASH_CALLS.load(Ordering::Relaxed) +} + +pub fn reset_leaf_hash_calls() { + LEAF_HASH_CALLS.store(0, Ordering::Relaxed); +} + +/// Leaf-hash passes over ONE codeword. +/// +/// ⚠ The global [`LEAF_HASH_CALLS`] is a diagnostic: it is process-wide, so a +/// test asserting on it is asserting about every other test sharing the binary +/// too. That is not a hypothetical — the first gate run of this file had a +/// counting test read 5 instead of 1 purely because its neighbours were +/// committing at the same time. A per-codeword count is what an assertion can +/// actually be about, and it localises a failure to the codeword that caused +/// it rather than to whoever ran alongside. +type BuildCount = Arc; + use crate::merkle::{build_inner_tree_levels, keccak_launch_cfg}; /// A codeword the device holds, base-field or ext3. @@ -24,10 +54,15 @@ pub struct DeviceCodeword { stream: Arc, elements: usize, base: bool, - /// The room the chain promised itself: this codeword, the folds that halve - /// it, and the tree each of them is committed and opened through. Shared - /// with the folds, which live inside it. - _room: Arc, + /// Leaf-hash passes this codeword has paid for: one per tree built, so + /// two for a commitment that is opened — the root's and the paths'. + builds: BuildCount, + /// The room the chain promised itself: this codeword and the folds that + /// halve it, shared with those folds because they live inside it. + /// + /// A tree is NOT in this number, because a tree is never held past the + /// call that builds it — see [`with_tree`](Self::with_tree). + room: Arc, } impl DeviceCodeword { @@ -56,7 +91,11 @@ impl DeviceCodeword { /// A leaf is the `2^log_folding` coset that folds onto one position, and /// the layout is the host's: `2*num_leaves - 1` nodes of 32 bytes, root /// first. - fn build_tree(&self, log_folding: usize) -> Result<(CudaSlice, usize)> { + fn build_tree( + &self, + log_folding: usize, + hash: crate::DeviceHash, + ) -> Result<(CudaSlice, usize)> { let num_leaves = self.elements >> log_folding; assert!(num_leaves >= 2, "tree needs at least two leaves"); let be = backend()?; @@ -70,10 +109,15 @@ impl DeviceCodeword { let mut leaves = nodes.slice_mut(leaves_offset..leaves_offset + num_leaves * 32); let num_leaves_u64 = num_leaves as u64; let block = 1u64 << log_folding; - let kernel = if self.base { - &be.keccak256_leaves_base_coset - } else { - &be.keccak256_leaves_ext3_coset + // ★ The hash is chosen HERE, not by the host backend that will + // label the result. `hash` is the key the caller's `WhirHash` + // supplied, so a tree labelled RPX was hashed by RPX's kernels or + // was not built here at all. + let kernel = match (hash, self.base) { + (crate::DeviceHash::Keccak256, true) => &be.keccak256_leaves_base_coset, + (crate::DeviceHash::Keccak256, false) => &be.keccak256_leaves_ext3_coset, + (crate::DeviceHash::Rpx256, true) => &be.rpx_leaves_base_coset, + (crate::DeviceHash::Rpx256, false) => &be.rpx_leaves_ext3_coset, }; unsafe { self.stream @@ -85,43 +129,109 @@ impl DeviceCodeword { .launch(keccak_launch_cfg(num_leaves_u64))?; } } - build_inner_tree_levels(self.stream.as_ref(), be, &mut nodes, num_leaves)?; + build_inner_tree_levels(self.stream.as_ref(), be, &mut nodes, num_leaves, hash)?; + LEAF_HASH_CALLS.fetch_add(1, Ordering::Relaxed); + self.builds.fetch_add(1, Ordering::Relaxed); Ok((nodes, num_leaves)) } + /// Run `f` against this codeword's tree, built here and freed on return. + /// + /// # Why the tree is not kept + /// + /// A commitment that is opened pays for its leaf layer twice — once for + /// the root, once for the paths — and keeping the first tree would remove + /// the second pass. H4 built that cache and measured it: it returned the + /// hashing it promised and cost more than it returned, ~+15 s in both + /// hashes, because the retention is not one tree but one per commitment + /// in the group. + /// + /// The window is forced by the protocol, not by this file. + /// `StackedCommitment::commit` builds EVERY chain's commitment before it + /// returns, because all the roots go into the transcript before any query + /// index is drawn; the openings come afterwards, one chain at a time. So + /// the last chain's tree would live from its commit to its opening — the + /// whole proof — and no placement of an eviction call bounds that peak, + /// since all N trees exist before the first opening. Ten chains at half a + /// gigabyte put the card at 96%, after which device allocations fail, + /// commits silently fall back to the host, and the host grows by ~1.5 GiB + /// per fallen-back chain. + /// + /// `crypto/multilinear/src/whir_commit.rs`'s `paths` said this in its doc + /// comment before any of it was built, and `StackedCommitment::commit`'s + /// reservation — "nine codewords of room instead of sixteen" — budgets a + /// retained codeword per commitment and no tree. Both were right. + /// + /// What is left of H4 is the counters: [`tree_builds`](Self::tree_builds) + /// and [`leaf_hash_calls`] make the two passes visible, and the group-scale + /// test in `tests/whir_tree_cache.rs` fails if a tree is ever held past + /// this call again. + fn with_tree( + &self, + log_folding: usize, + hash: crate::DeviceHash, + f: impl FnOnce(&CudaSlice, usize) -> Result, + ) -> Result { + let (nodes, num_leaves) = self.build_tree(log_folding, hash)?; + f(&nodes, num_leaves) + } + + /// Bytes this codeword's chain has promised the device budget. For tests + /// and diagnostics. + pub fn reserved_bytes(&self) -> u64 { + self.room.bytes() + } + + /// ★ How many times THIS codeword's leaf layer has been hashed. + /// + /// One after a commit, and one more for each round that opens it. Unlike + /// the process-wide counter this number is unaffected by whatever else + /// shares the test binary, so an assertion on it is about this codeword. + pub fn tree_builds(&self) -> u64 { + self.builds.load(Ordering::Relaxed) + } + /// The root of that tree, which is the commitment. /// - /// The tree itself is dropped: the only other thing anyone wants from it - /// is a path per query, and by then the queries are known — see - /// [`paths`](Self::paths). - pub fn commit(&self, log_folding: usize) -> Result<[u8; 32]> { - let (nodes, _) = self.build_tree(log_folding)?; - let head = self.stream.clone_dtoh(&nodes.slice(0..32))?; - self.stream.synchronize()?; - let mut root = [0u8; 32]; - root.copy_from_slice(&head); - Ok(root) + /// ★ The tree is KEPT (H4). The other thing anyone wants from it is a path + /// per query, and rebuilding it then cost a second leaf-hash pass over the + /// whole codeword — half of this path's device hashing, for a buffer that + /// was already in hand. + pub fn commit(&self, log_folding: usize, hash: crate::DeviceHash) -> Result<[u8; 32]> { + self.with_tree(log_folding, hash, |nodes, _| { + let head = self.stream.clone_dtoh(&nodes.slice(0..32))?; + self.stream.synchronize()?; + let mut root = [0u8; 32]; + root.copy_from_slice(&head); + Ok(root) + }) } /// The whole tree in the host node layout — what a caller that walks it /// here needs, and what the parity test compares against. - pub fn nodes_to_host(&self, log_folding: usize) -> Result> { - let (nodes, _) = self.build_tree(log_folding)?; - let out = self.stream.clone_dtoh(&nodes)?; - self.stream.synchronize()?; - Ok(out) + pub fn nodes_to_host(&self, log_folding: usize, hash: crate::DeviceHash) -> Result> { + self.with_tree(log_folding, hash, |nodes, _| { + let out = self.stream.clone_dtoh(nodes)?; + self.stream.synchronize()?; + Ok(out) + }) } /// The authentication paths of `positions`, against the same tree. /// - /// Rebuilt rather than kept or carried home. Keeping it costs half a - /// gigabyte of device memory per commitment for the whole proof; bringing - /// it back costs ten times the rehash, because a pageable copy of half a - /// gigabyte is the slowest thing in the commit. What the host needs of a - /// tree is a kilobyte per query. - pub fn paths(&self, log_folding: usize, positions: &[u32]) -> Result> { - let (nodes, num_leaves) = self.build_tree(log_folding)?; - crate::merkle::gather_merkle_paths_dev(&nodes, num_leaves, positions, &self.stream) + /// ★ Read from the tree the commit kept, not rebuilt (H4). Bringing the + /// tree home is still not done — a pageable copy of half a gigabyte is the + /// slowest thing in the commit, and what the host needs of a tree is a + /// kilobyte per query. + pub fn paths( + &self, + log_folding: usize, + positions: &[u32], + hash: crate::DeviceHash, + ) -> Result> { + self.with_tree(log_folding, hash, |nodes, num_leaves| { + crate::merkle::gather_merkle_paths_dev(nodes, num_leaves, positions, &self.stream) + }) } /// The fold blocks `indices` open — `block` values at stride `num_leaves` @@ -184,12 +294,14 @@ pub fn commit_codeword_parts( log_blowup: usize, log_folding: usize, transient: bool, + hash: crate::DeviceHash, ) -> Result<(DeviceCodeword, [u8; 32])> { commit_from( Source::Parts { parts, log_evals }, log_blowup, log_folding, transient, + hash, ) } @@ -203,6 +315,7 @@ pub fn commit_codeword_resident( log_blowup: usize, log_folding: usize, transient: bool, + hash: crate::DeviceHash, ) -> Result<(DeviceCodeword, [u8; 32])> { commit_from( Source::Resident { @@ -213,6 +326,7 @@ pub fn commit_codeword_resident( log_blowup, log_folding, transient, + hash, ) } @@ -269,8 +383,15 @@ pub fn commit_codeword( log_blowup: usize, log_folding: usize, transient: bool, + hash: crate::DeviceHash, ) -> Result<(DeviceCodeword, [u8; 32])> { - commit_from(Source::Whole(evals), log_blowup, log_folding, transient) + commit_from( + Source::Whole(evals), + log_blowup, + log_folding, + transient, + hash, + ) } fn commit_from( @@ -278,6 +399,7 @@ fn commit_from( log_blowup: usize, log_folding: usize, transient: bool, + hash: crate::DeviceHash, ) -> Result<(DeviceCodeword, [u8; 32])> { let log_evals = source.log_evals(); let log_n = log_evals + log_blowup as u64; @@ -349,9 +471,10 @@ fn commit_from( stream, elements: n, base: true, - _room: Arc::new(room), + builds: BuildCount::default(), + room: Arc::new(room), }; - let root = codeword.commit(log_folding)?; + let root = codeword.commit(log_folding, hash)?; Ok((codeword, root)) } @@ -444,11 +567,12 @@ pub fn commit_codeword_to_host( evals: &[u64], log_blowup: usize, log_folding: usize, + hash: crate::DeviceHash, ) -> Result<(Vec, Vec)> { - let (codeword, _root) = commit_codeword(evals, log_blowup, log_folding, true)?; + let (codeword, _root) = commit_codeword(evals, log_blowup, log_folding, true, hash)?; let values = codeword.stream.clone_dtoh(codeword.buffer.as_ref())?; codeword.stream.synchronize()?; - let nodes = codeword.nodes_to_host(log_folding)?; + let nodes = codeword.nodes_to_host(log_folding, hash)?; Ok((values, nodes)) } @@ -591,9 +715,13 @@ pub fn fold_resident( stream, elements: half, base: false, + // A fold is its OWN codeword: its own cache slot and its own count. It + // is committed and opened in its own right, and sharing the parent's + // slot would make one of them evict the other every round. + builds: BuildCount::default(), // The fold lives inside the room the codeword it came from promised: // it is half of it, and that one is still alive. - _room: codeword._room.clone(), + room: codeword.room.clone(), }) } @@ -650,7 +778,11 @@ pub fn fold_codeword_ext3( /// /// The codeword itself stays where the caller has it: a folded codeword is the /// next round's input on the host side, so only the tree comes back. -pub fn commit_codeword_ext3(codeword: &[u64], log_folding: usize) -> Result> { +pub fn commit_codeword_ext3( + codeword: &[u64], + log_folding: usize, + hash: crate::DeviceHash, +) -> Result> { assert!( codeword.len().is_multiple_of(3), "three u64 per ext3 element" @@ -679,7 +811,10 @@ pub fn commit_codeword_ext3(codeword: &[u64], log_folding: usize) -> Result &be.keccak256_leaves_ext3_coset, + crate::DeviceHash::Rpx256 => &be.rpx_leaves_ext3_coset, + }) .arg(&values) .arg(&num_leaves_u64) .arg(&block) @@ -687,7 +822,7 @@ pub fn commit_codeword_ext3(codeword: &[u64], log_folding: usize) -> Result(&seed, factor), + factor, + ) + .expect("GPU grind (needs a GPU)"); assert!( - is_valid_nonce(&seed, nonce, factor), + is_valid_nonce::(&seed, nonce, factor), "GPU nonce {nonce} fails is_valid_nonce (factor {factor})" ); assert!( - (0..nonce).all(|n| !is_valid_nonce(&seed, n, factor)), + (0..nonce).all(|n| !is_valid_nonce::(&seed, n, factor)), "GPU nonce {nonce} is not the smallest valid nonce (factor {factor})" ); } @@ -51,10 +54,13 @@ fn gpu_grind_returns_smallest_valid_nonce() { fn gpu_grind_valid_at_production_factor() { let seed = [20u8; 32]; let factor = 20u8; - let nonce = math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes(&seed, factor), factor) - .expect("GPU grind (needs a GPU)"); + let nonce = math_cuda::grinding::generate_nonce_gpu( + &inner_hash_lanes::(&seed, factor), + factor, + ) + .expect("GPU grind (needs a GPU)"); assert!( - is_valid_nonce(&seed, nonce, factor), + is_valid_nonce::(&seed, nonce, factor), "GPU nonce {nonce} fails is_valid_nonce (factor {factor})" ); } @@ -65,7 +71,8 @@ fn gpu_grind_valid_at_production_factor() { fn gpu_grind_declines_below_min_factor() { let seed = [1u8; 32]; assert!( - math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes(&seed, 1), 1).is_none(), + math_cuda::grinding::generate_nonce_gpu(&inner_hash_lanes::(&seed, 1), 1) + .is_none(), "GPU grind should decline factor 1" ); } diff --git a/crypto/math-cuda/tests/host_kat/cuda_host_shim.h b/crypto/math-cuda/tests/host_kat/cuda_host_shim.h new file mode 100644 index 000000000..29ccc505f --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/cuda_host_shim.h @@ -0,0 +1,98 @@ +// Enough of the CUDA language to compile a `.cu` kernel file as ordinary host +// C++, so its arithmetic can be checked without a GPU. +// +// This exists because the GPU parity suite (`crypto/math-cuda/tests/blake3_*.rs`) +// runs only where a GPU does, and per-PR CI has none. Including a kernel through +// this shim turns its device functions into plain functions a host program can +// call, which is all a known-answer test needs. +// +// ⚠ What it CANNOT check, and what therefore still belongs to the GPU tests: +// anything about execution rather than arithmetic — thread/block indexing, +// `__syncthreads` ordering, memory alignment on device, register pressure, and +// whether nvcc accepts the file at all. A kernel that passes here can still be +// wrong on a GPU. Treat this as a lower bound on correctness, never a substitute. +#pragma once + +#include + +// The execution-space and inlining qualifiers carry no meaning on host. +#define __device__ +#define __constant__ +#define __forceinline__ inline +#define __global__ + +// Single-threaded host execution: one thread, block 0, and a barrier that has +// nothing to wait for. Kernel thread coordinates are ordinary mutable globals, +// so a caller can drive them (see `CUDA_HOST_FOR_EACH_THREAD`) and replay a +// whole launch's worth of thread slices one at a time. +#define __syncthreads() ((void)0) +struct CudaHostDim3 { + unsigned x = 0, y = 0, z = 0; +}; +static CudaHostDim3 blockIdx; +static CudaHostDim3 threadIdx; +static CudaHostDim3 cuda_host_block_dim; +static CudaHostDim3 cuda_host_grid_dim; +#define blockDim cuda_host_block_dim +#define gridDim cuda_host_grid_dim + +// The one atomic the grid-stride search kernels use. Single-threaded on host, +// so the read-modify-write needs no protection; it returns the OLD value, as +// CUDA's does, and takes a non-volatile pointer because the kernels cast the +// volatility away at the call (the `volatile` there is for the *reads* that +// drive the early exit, which the shim's single thread makes moot). +static inline unsigned long long atomicMin(unsigned long long *address, + unsigned long long val) { + unsigned long long old = *address; + if (val < old) *address = val; + return old; +} + +// `goldilocks.cuh`'s field multiply needs this intrinsic. `blake3.cu` only uses +// `goldilocks::canonical`, but the header compiles as a whole, so supply it. +static inline uint64_t __umul64hi(uint64_t a, uint64_t b) { + return (uint64_t)(((unsigned __int128)a * (unsigned __int128)b) >> 64); +} + +// Bit-reverse a 64-bit word. Every leaf kernel derives its row index as +// `__brevll(tid) >> (64 - log_num_rows)`, so replaying one on host needs it. +// Written out rather than deferring to a compiler builtin so the shim stays +// toolchain-neutral. +static inline uint64_t __brevll(uint64_t x) { + x = ((x & 0x5555555555555555ull) << 1) | ((x >> 1) & 0x5555555555555555ull); + x = ((x & 0x3333333333333333ull) << 2) | ((x >> 2) & 0x3333333333333333ull); + x = ((x & 0x0F0F0F0F0F0F0F0Full) << 4) | ((x >> 4) & 0x0F0F0F0F0F0F0F0Full); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x >> 8) & 0x00FF00FF00FF00FFull); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x >> 16) & 0x0000FFFF0000FFFFull); + return (x << 32) | (x >> 32); +} + +// Replay a `__global__` kernel once per thread index, sequentially, by driving +// the shim's thread coordinates. A kernel computing +// `tid = blockIdx.x * blockDim.x + threadIdx.x` sees `tid = i` on iteration `i`, +// so a whole launch can be reproduced on host: +// +// CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) some_leaf_kernel(args...); +// +// ⚠ Only valid for kernels whose threads are independent — which the leaf +// kernels are (one thread, one leaf, disjoint output) and the Merkle *tail* is +// not. It says nothing about `__syncthreads` ordering, races or occupancy. +#define CUDA_HOST_FOR_EACH_THREAD(i, n) \ + for (unsigned i = 0; \ + i < (unsigned)(n) && \ + (blockIdx.x = 0, blockDim.x = 0, threadIdx.x = i, true); \ + ++i) + +// Replay a GRID-STRIDE kernel as ONE thread that covers the whole range: +// `tid = 0`, `stride = gridDim.x * blockDim.x = 1`, so a kernel written as +// `for (i = tid; i < count; i += stride)` scans `[0, count)` in order. +// +// `CUDA_HOST_FOR_EACH_THREAD` cannot do this — it leaves `blockDim.x = 0`, +// which is a zero stride and an unterminated loop. +// +// ⚠ Says nothing about the parallel reduction a real launch performs. What it +// checks is the per-candidate arithmetic and the loop's bounds; that +// `atomicMin` over many threads yields the same answer is a property of the +// reduction, pinned on a GPU. +#define CUDA_HOST_SINGLE_THREAD() \ + (gridDim.x = 1, blockDim.x = 1, blockIdx.x = 0, threadIdx.x = 0, (void)0) diff --git a/crypto/math-cuda/tests/host_kat/rpx_canon_witness.py b/crypto/math-cuda/tests/host_kat/rpx_canon_witness.py new file mode 100644 index 000000000..7c8dfd17d --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/rpx_canon_witness.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Derives the "canonicalisation witness" row of `rpx_kat_vectors.h`. + +WHY. `rpx::permute` (kernels/rpx.cu) ends in a loop that canonicalises the +state, which is what makes device digests byte-comparable to the host's. A +known-answer check cannot see that loop unless some output lane is a raw twin +(`value + p`, in `[p, 2^64)`) before it — a 2^-32 event per lane on random +inputs. This script builds an input for which it is certain. + +HOW. The permutation's last operation is `out_i = add(m_i, ARK1[6][i])`, where +`m_i` is the M-round MDS output. With `m_i` canonical and +`m_i + ARK1[6][i] < 2^64`, the device `add` returns `m_i + ARK1[6][i]` as is; if +that sum lies in `[p, 2^64)` it is the raw twin of `sum − p`. So choose the +canonical MDS output `u` with `u_0 = p − ARK1[6][0] + 1` (raw `out_0 = p + 1`, +field value 1), fill the other eleven lanes at random, invert the MDS to get the +M-round input, and invert rounds 5..0 — `x^{1/7}` in `GF(p³)` for the E rounds, +`x^7` / `MDS⁻¹` / `x^{1/7}` / `MDS⁻¹` for the FB rounds — to get the +permutation input. `m_0` cannot itself be a twin (`u_0 + p > 2^64`), so the raw +lane is deterministic whatever representation the earlier rounds happen to +carry. + +TRUST. This is a THIRD transcription of the permutation, so it trusts nothing +about itself: before printing, it reproduces every row of the header's Table 2 +forward and inverts each one back to its input. Run from anywhere: + + python3 crypto/math-cuda/tests/host_kat/rpx_canon_witness.py + +The printed input goes into `prover/tests/rpx_host_kat_vectors.rs` +(`permutation_inputs`, the row named "canonicalisation witness"); its output +row comes from that generator, never from here. +""" +import pathlib +import random +import re + +REPO = pathlib.Path(__file__).resolve().parents[4] +P = (1 << 64) - (1 << 32) + 1 +INV_ALPHA = 10540996611094048183 # rpo.rs:96 +assert (7 * INV_ALPHA) % (P - 1) == 1 +ROW = [7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8] # rpo.rs:114 + +RPO_RS = (REPO / "prover/src/lfm/rpo.rs").read_text() + + +def constant_table(name): + m = re.search( + r"pub const %s: \[\[u64; HASH_STATE_FELTS\]; NUM_ROUNDS\] = \[(.*?)\n\];" % name, + RPO_RS, + re.S, + ) + rows = re.findall(r"\[\s*((?:\d+,\s*)+)\]", m.group(1)) + vals = [[int(x) for x in re.findall(r"\d+", r)] for r in rows] + assert len(vals) == 7 and all(len(r) == 12 for r in vals), name + return vals + + +ARK1, ARK2 = constant_table("ARK1"), constant_table("ARK2") + + +# --- the field, the MDS and its inverse, the cubic extension ------------------------- + +def mds(s): + return [sum(ROW[(j - i) % 12] * s[j] for j in range(12)) % P for i in range(12)] + + +def matrix_inverse_mod_p(m): + n = len(m) + a = [row[:] + [1 if i == j else 0 for j in range(n)] for i, row in enumerate(m)] + for col in range(n): + piv = next(r for r in range(col, n) if a[r][col] % P) + a[col], a[piv] = a[piv], a[col] + inv = pow(a[col][col], P - 2, P) + a[col] = [(v * inv) % P for v in a[col]] + for r in range(n): + if r != col and a[r][col]: + f = a[r][col] + a[r] = [(vr - f * vc) % P for vr, vc in zip(a[r], a[col])] + return [row[n:] for row in a] + + +MDS_INV = matrix_inverse_mod_p([[ROW[(j - i) % 12] for j in range(12)] for i in range(12)]) + + +def mds_inv(s): + return [sum(MDS_INV[i][j] * s[j] for j in range(12)) % P for i in range(12)] + + +def ext_mul(a, b): # rpx.rs:118-125, φ³ = φ + 1 + return [ + (a[0] * b[0] + a[1] * b[2] + a[2] * b[1]) % P, + (a[0] * b[1] + a[1] * b[0] + a[1] * b[2] + a[2] * b[1] + a[2] * b[2]) % P, + (a[0] * b[2] + a[1] * b[1] + a[2] * b[0] + a[2] * b[2]) % P, + ] + + +def ext_pow(a, e): + r, b = [1, 0, 0], a[:] + while e: + if e & 1: + r = ext_mul(r, b) + b = ext_mul(b, b) + e >>= 1 + return r + + +EXT_INV7 = pow(7, -1, P**3 - 1) # x ↦ x^7 permutes GF(p³) (rpx.rs tests), so this exists + + +# --- the permutation, forward (rpx.rs:280-316) and inverse ---------------------------- + +def add_constants(s, table, r, sign=1): + return [(v + sign * table[r][i]) % P for i, v in enumerate(s)] + + +def fb_round(s, r): + s = add_constants(mds(s), ARK1, r) + s = mds([pow(v, 7, P) for v in s]) + return [pow(v, INV_ALPHA, P) for v in add_constants(s, ARK2, r)] + + +def ext_round(s, r): + s = add_constants(s, ARK1, r) + return sum((ext_pow(s[3 * e:3 * e + 3], 7) for e in range(4)), []) + + +def final_round(s): + return add_constants(mds(s), ARK1, 6) + + +def permute(s): + for r in range(6): + s = fb_round(s, r) if r % 2 == 0 else ext_round(s, r) + return final_round(s) + + +def fb_round_inv(s, r): + s = add_constants([pow(v, 7, P) for v in s], ARK2, r, -1) + s = [pow(v, INV_ALPHA, P) for v in mds_inv(s)] + return mds_inv(add_constants(s, ARK1, r, -1)) + + +def ext_round_inv(s, r): + s = sum((ext_pow(s[3 * e:3 * e + 3], EXT_INV7) for e in range(4)), []) + return add_constants(s, ARK1, r, -1) + + +def rounds_0_to_5_inv(t): + for r in (5, 4, 3, 2, 1, 0): + t = ext_round_inv(t, r) if r % 2 == 1 else fb_round_inv(t, r) + return t + + +# --- self-check against the header's oracle table before trusting any of the above ---- + +HEADER = (REPO / "crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h").read_text() +body = HEADER.split("RPX_PERMUTATION_VECTORS[NUM_RPX_PERMUTATION_VECTORS] = {")[1].split("};")[0] +rows = re.findall(r'\{"([^"]*)",\s*\{([^}]*)\},\s*\{([^}]*)\}\}', body) +assert len(rows) >= 8, "oracle table has %d rows" % len(rows) +for name, inp, outp in rows: + x = [int(v) for v in re.findall(r"\d+", inp)] + y = [int(v) for v in re.findall(r"\d+", outp)] + assert permute(x) == y, "forward transcription disagrees with the oracle on %r" % name + t = rounds_0_to_5_inv(mds_inv(add_constants(y, ARK1, 6, -1))) + assert t == x, "inverse permutation does not round-trip on %r" % name +print("self-check: %d/%d oracle rows reproduced forward and inverted back" % (len(rows), len(rows))) + +# --- the witness ------------------------------------------------------------------------- + +c0 = ARK1[6][0] +rng = random.Random(0x4B57) # "KW"; one generator, eleven draws +u = [P - c0 + 1] + [rng.randrange(P) for _ in range(11)] +assert P - c0 <= u[0] < P - c0 + (1 << 32) - 1 +x = rounds_0_to_5_inv(mds_inv(u)) +y = permute(x) +assert y[0] == 1 and y == add_constants(u, ARK1, 6) +print("witness input :", ", ".join(str(v) for v in x)) +print("witness output:", ", ".join(str(v) for v in y), " (lane 0 raw on device: %d = p + 1)" % (u[0] + c0)) + +# The generator's hard-coded row must be exactly this input, or the header's +# witness and this derivation have drifted apart. +GENERATOR = (REPO / "prover/tests/rpx_host_kat_vectors.rs").read_text() +block = GENERATOR.split('"canonicalisation witness"', 1)[1].split("]", 1)[0] +assert [int(v) for v in re.findall(r"\d+", block)] == x, "the generator's witness row is not this derivation's" +print("generator row check: prover/tests/rpx_host_kat_vectors.rs carries this exact input") diff --git a/crypto/math-cuda/tests/host_kat/rpx_host_kat.cpp b/crypto/math-cuda/tests/host_kat/rpx_host_kat.cpp new file mode 100644 index 000000000..7cdf8cbd5 --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/rpx_host_kat.cpp @@ -0,0 +1,1197 @@ +// Known-answer tests for `kernels/rpx.cu`, run on the host. +// +// WHY THIS EXISTS. The GPU parity suite runs only where a GPU does, and per-PR +// CI has none — GPU CI is merge_group-only. This compiles the real kernel +// source through `cuda_host_shim.h` and pins its arithmetic in seconds, with no +// GPU and no cargo, exactly as `blake3_host_kat.cpp` does for BLAKE3. +// +// WHAT IT COVERS: the field primitives the kernel is built from, the MDS, both +// S-boxes, the cubic extension, the seven-round schedule, the rate-8 overwrite +// duplex leaf, the Merkle parent, and the raw-vs-canonical representation. +// +// WHAT IT DOES NOT COVER, and what the GPU tests are still required for: +// whether nvcc accepts the file, and every property of execution rather than +// arithmetic — grid indexing, register pressure, local-memory spills from the +// sponge's dynamic indexing. Passing here is necessary, never sufficient. +// +// HOW THE ANCHORING LAYERS. Nothing here is checked only against itself: +// 1. The field primitives (`goldilocks::mul/add`, `ext3::dot3`) against +// schoolbook `__int128` arithmetic — the definition, no shared code. +// 2. The MDS against its per-term definition; the S-boxes against generic +// exponentiation (including `x^{1/7}` as `x^INV_ALPHA`); the cubic +// extension against naive polynomial multiplication reduced by +// `φ³ = φ + 1` — the same independent algorithms `rpx.rs`'s own tests use. +// 3. ★ EXTERNAL: RPX's FB round IS RPO's round with RPO's constants. Seven +// `fb_round(s, r)` compose to RPO256, and that composition is replayed over +// miden-crypto's nineteen `hash_elements` vectors, which nothing in this +// tree produced. That pins ARK1/ARK2, the MDS row and orientation, both +// S-box chains and the sponge lane convention from outside. +// 4. ★ THE ORACLE: the Rust host `Rpx256` (`prover/src/lfm/rpx.rs`), through +// the tables `prover/tests/rpx_host_kat_vectors.rs` prints — the bare +// permutation, the leaf sponge at seven lengths, the parent. miden +// publishes no RPX vector, so the E round and the schedule rest on this +// layer alone, as the Rust module's own provenance note says they must. +// 5. Negative controls: RPX ≠ RPO on the same state; every input lane +// reaches the output; raw (`≥ p`) and canonical inputs agree; outputs are +// canonical. +// 6. The cost model, COUNTED rather than asserted from a comment. +// 7. Every leaf kernel, both Merkle compressors and the permutation probe +// replayed thread by thread through the shim against the CPU leaf spec +// and the host parent — the read patterns and the node encoding, with the +// hash over them anchored by the layers above. +// 8. The proof-of-work grind kernel against the HOST predicate +// `stark::grinding::is_valid_nonce` over `AlgebraicDigest` — +// the nonce, its minimality, that `base` participates, and an endianness +// control (the little-endian reading of the same inner hash finds nothing +// where the big-endian one finds the nonce). +// +// Build and run with `make test-rpx-host-kat`. + +#include +#include +#include +#include +#include + +#include "cuda_host_shim.h" + +// The kernel under test. Included, not linked: the shim turns its device +// functions into host functions, and there is no other way to call them. +// RPX_HOST_OP_COUNT turns on its field-op counters (layer 6). +#define RPX_HOST_OP_COUNT +#include "rpx.cu" + +#include "rpx_kat_vectors.h" + +namespace { + +int failures = 0; + +void check(bool ok, const char *what) { + if (!ok) { + printf("FAIL: %s\n", what); + ++failures; + } +} + +typedef unsigned __int128 u128; +const uint64_t P = 0xFFFFFFFF00000001ull; +// `7^{-1} mod (p − 1)` — rpo.rs:96. Re-derived below rather than trusted. +const uint64_t INV_ALPHA = 10540996611094048183ull; + +uint64_t canon(uint64_t x) { return x >= P ? x - P : x; } + +// =========================================================================== +// Reference arithmetic: schoolbook over `__int128`. It shares no code with the +// kernel — it is the definition the kernel's shortcuts are checked against. +// =========================================================================== + +uint64_t ref_mul(uint64_t a, uint64_t b) { + return (uint64_t)(((u128)canon(a) * (u128)canon(b)) % P); +} + +uint64_t ref_add(uint64_t a, uint64_t b) { + return (uint64_t)(((u128)canon(a) + (u128)canon(b)) % P); +} + +uint64_t ref_pow(uint64_t x, uint64_t e) { + uint64_t r = 1, b = canon(x); + while (e != 0) { + if (e & 1) r = ref_mul(r, b); + b = ref_mul(b, b); + e >>= 1; + } + return r; +} + +struct RefExt { + uint64_t c[3]; +}; + +// Naive polynomial multiplication reduced by `φ³ = φ + 1`, `φ⁴ = φ² + φ` — the +// obvious slow way, as `rpx.rs:341-352` writes it, so it shares no structure +// with the kernel's regrouped closed form. +RefExt ref_ext_mul(const RefExt &a, const RefExt &b) { + uint64_t c[5] = {0, 0, 0, 0, 0}; + for (int i = 0; i < 3; ++i) { + for (int j = 0; j < 3; ++j) c[i + j] = ref_add(c[i + j], ref_mul(a.c[i], b.c[j])); + } + RefExt r; + r.c[0] = ref_add(c[0], c[3]); + r.c[1] = ref_add(ref_add(c[1], c[3]), c[4]); + r.c[2] = ref_add(c[2], c[4]); + return r; +} + +RefExt ref_ext_pow(RefExt a, unsigned e) { + RefExt r = {{1, 0, 0}}; + while (e != 0) { + if (e & 1) r = ref_ext_mul(r, a); + a = ref_ext_mul(a, a); + e >>= 1; + } + return r; +} + +// The MDS as defined: `out_i = Σ_j ROW[(j − i) mod 12] · s_j`, one reduced +// field multiplication per term (rpo.rs:522-524). +void ref_mds(const uint64_t in[12], uint64_t out[12]) { + static const uint64_t ROW[12] = {7, 23, 8, 26, 13, 10, 9, 7, 6, 22, 21, 8}; + for (int i = 0; i < 12; ++i) { + uint64_t acc = 0; + for (int j = 0; j < 12; ++j) acc = ref_add(acc, ref_mul(ROW[(j + 12 - i) % 12], in[j])); + out[i] = acc; + } +} + +// A deterministic value stream. Every fifth value is a RAW representation in +// `[p, 2^64)` — the field's non-canonical storage, which the kernel must read +// as `value − p` — so the reduction paths are exercised rather than assumed. +uint64_t splitmix(uint64_t &seed) { + seed += 0x9E3779B97F4A7C15ull; + uint64_t z = seed; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); +} + +uint64_t sample(uint64_t &seed, uint64_t i) { + uint64_t x = splitmix(seed); + // Raw values above p exist only for canonical values below 2^32 − 1. + return (i % 5 == 0) ? (x % 0xFFFFFFFFull) + P : x % P; +} + +// Values at every edge of the representation: zero, one, the modulus and its +// neighbours (raw zero, raw one), EPSILON and 2^32, the top of the u64 range. +const uint64_t EDGES[] = {0ull, 1ull, 2ull, P - 1, P, + P + 1, 0xFFFFFFFFull, 0x100000000ull, 1ull << 63, ~0ull, + ~0ull - 1, 0x0123456789ABCDEFull}; +const int NUM_EDGES = (int)(sizeof(EDGES) / sizeof(EDGES[0])); + +// =========================================================================== +// Layer 1 — the field primitives the kernel is built from. +// =========================================================================== + +void field_primitives_match_schoolbook_arithmetic() { + int checked = 0; + for (int i = 0; i < NUM_EDGES; ++i) { + for (int j = 0; j < NUM_EDGES; ++j) { + const uint64_t a = EDGES[i], b = EDGES[j]; + check(canon(goldilocks::mul(a, b)) == ref_mul(a, b), "goldilocks::mul at an edge"); + check(canon(goldilocks::add(a, b)) == ref_add(a, b), "goldilocks::add at an edge"); + // Three equal products: the 128-bit sum overflows for the large edges. + const uint64_t want = ref_add(ref_add(ref_mul(a, b), ref_mul(a, b)), ref_mul(a, b)); + check(canon(ext3::dot3(a, b, a, b, a, b)) == want, "ext3::dot3 at an edge (3 equal terms)"); + ++checked; + } + } + // The two-overflow case explicitly: six maximal operands. + { + const uint64_t m = ~0ull; + const uint64_t want = ref_add(ref_add(ref_mul(m, m), ref_mul(m, m)), ref_mul(m, m)); + check(canon(ext3::dot3(m, m, m, m, m, m)) == want, "ext3::dot3 with two 2^128 overflows"); + const uint64_t want1 = ref_add(ref_mul(m, m), ref_mul(m, m)); + check(canon(ext3::dot3(m, m, m, m, 0, 0)) == want1, "ext3::dot3 with one 2^128 overflow"); + } + uint64_t seed = 0xF1E1D; + for (int k = 0; k < 500; ++k) { + uint64_t v[6]; + for (int t = 0; t < 6; ++t) v[t] = sample(seed, (uint64_t)k * 6 + t); + const uint64_t want = + ref_add(ref_add(ref_mul(v[0], v[1]), ref_mul(v[2], v[3])), ref_mul(v[4], v[5])); + check(canon(ext3::dot3(v[0], v[1], v[2], v[3], v[4], v[5])) == want, "ext3::dot3 on random"); + check(canon(goldilocks::mul(v[0], v[1])) == ref_mul(v[0], v[1]), "goldilocks::mul on random"); + ++checked; + } + printf("field primitives vs schoolbook __int128: %d edge pairs + random, mul/add/dot3\n", checked); +} + +// =========================================================================== +// Layer 2 — the building blocks against independent algorithms. +// =========================================================================== + +void mds_matches_its_per_term_definition() { + std::vector> states; + states.push_back(std::vector(12, 0)); + states.push_back(std::vector(12, P - 1)); + states.push_back(std::vector(12, ~0ull)); // the raw maximum: the u128 bound's worst case + for (int k = 0; k < 12; ++k) { // one-hot lanes pin the orientation + std::vector s(12, 0); + s[k] = 1; + states.push_back(s); + } + uint64_t seed = 0x3D5; + for (int k = 0; k < 64; ++k) { + std::vector s(12); + for (int i = 0; i < 12; ++i) s[i] = sample(seed, (uint64_t)k * 12 + i); + states.push_back(s); + } + for (size_t n = 0; n < states.size(); ++n) { + uint64_t got[12], want[12]; + memcpy(got, states[n].data(), sizeof(got)); + rpx::mds(got); + ref_mds(states[n].data(), want); + for (int i = 0; i < 12; ++i) { + if (canon(got[i]) != want[i]) { + printf("FAIL mds state %zu lane %d: got %llu want %llu\n", n, i, + (unsigned long long)canon(got[i]), (unsigned long long)want[i]); + ++failures; + break; + } + } + } + printf("MDS (u128-property port) vs per-term definition: %zu states incl. raw-max and one-hot\n", + states.size()); +} + +void sboxes_are_the_seventh_power_and_its_inverse() { + // `7 · INV_ALPHA ≡ 1 (mod p − 1)`, re-derived as rpo.rs:797-806 does. + const u128 p_minus_one = (u128)P - 1; + check(((u128)7 * (u128)INV_ALPHA) % p_minus_one == 1, "INV_ALPHA must invert 7 in the exponent group"); + + std::vector xs(EDGES, EDGES + NUM_EDGES); + uint64_t seed = 0x5B0; + for (int k = 0; k < 48; ++k) xs.push_back(sample(seed, (uint64_t)k)); + for (size_t n = 0; n < xs.size(); ++n) { + const uint64_t x = xs[n]; + check(canon(rpx::sbox(x)) == ref_pow(x, 7), "sbox(x) must be x^7"); + check(canon(rpx::inv_sbox(x)) == ref_pow(x, INV_ALPHA), "inv_sbox(x) must be x^INV_ALPHA"); + check(canon(rpx::sbox(rpx::inv_sbox(x))) == canon(x), "sbox(inv_sbox(x)) must be x"); + check(canon(rpx::inv_sbox(rpx::sbox(x))) == canon(x), "inv_sbox(sbox(x)) must be x"); + } + check(rpx::inv_sbox(0) == 0, "inv_sbox(0) must be 0 (the padding row's fixed point)"); + check(canon(rpx::inv_sbox(P)) == 0, "inv_sbox(raw zero) must be 0"); + check(canon(rpx::inv_sbox(1)) == 1, "inv_sbox(1) must be 1"); + printf("S-boxes vs generic exponentiation: %zu values, x^7, x^{1/7}, both compositions\n", + xs.size()); +} + +void cubic_extension_matches_naive_polynomial_arithmetic() { + // The reduction rule itself, pinned on the basis: φ·φ² = φ³ = 1 + φ, and + // φ²·φ² = φ⁴ = φ + φ². + { + rpx::CubicExt phi = {0, 1, 0}, phi2 = {0, 0, 1}, one = {1, 0, 0}; + rpx::CubicExt r = rpx::ext_mul(phi, phi2); + check(canon(r.c0) == 1 && canon(r.c1) == 1 && canon(r.c2) == 0, "φ·φ² must be 1 + φ"); + r = rpx::ext_mul(phi2, phi2); + check(canon(r.c0) == 0 && canon(r.c1) == 1 && canon(r.c2) == 1, "φ²·φ² must be φ + φ²"); + r = rpx::ext_mul(phi2, one); + check(canon(r.c0) == 0 && canon(r.c1) == 0 && canon(r.c2) == 1, "1 must be the identity"); + } + std::vector as, bs; + as.push_back(RefExt{{P - 1, P - 1, P - 1}}); + bs.push_back(RefExt{{P - 1, P - 1, P - 1}}); + as.push_back(RefExt{{~0ull, ~0ull, ~0ull}}); // raw maxima + bs.push_back(RefExt{{~0ull, ~0ull, ~0ull}}); + as.push_back(RefExt{{0, 0, 0}}); + bs.push_back(RefExt{{P - 1, 0, 1}}); + uint64_t seed = 0xE3; + for (int k = 0; k < 64; ++k) { + RefExt a, b; + for (int t = 0; t < 3; ++t) { + a.c[t] = sample(seed, (uint64_t)k * 6 + t); + b.c[t] = sample(seed, (uint64_t)k * 6 + 3 + t); + } + as.push_back(a); + bs.push_back(b); + } + for (size_t n = 0; n < as.size(); ++n) { + const rpx::CubicExt a = {as[n].c[0], as[n].c[1], as[n].c[2]}; + const rpx::CubicExt b = {bs[n].c[0], bs[n].c[1], bs[n].c[2]}; + const rpx::CubicExt m = rpx::ext_mul(a, b); + const RefExt mw = ref_ext_mul(as[n], bs[n]); + check(canon(m.c0) == mw.c[0] && canon(m.c1) == mw.c[1] && canon(m.c2) == mw.c[2], + "ext_mul must equal the naive polynomial product"); + const rpx::CubicExt s = rpx::ext_square(a); + const RefExt sw = ref_ext_mul(as[n], as[n]); + check(canon(s.c0) == sw.c[0] && canon(s.c1) == sw.c[1] && canon(s.c2) == sw.c[2], + "ext_square must equal the naive square"); + const rpx::CubicExt p7 = rpx::ext_power7(a); + const RefExt pw = ref_ext_pow(as[n], 7); + check(canon(p7.c0) == pw.c[0] && canon(p7.c1) == pw.c[1] && canon(p7.c2) == pw.c[2], + "ext_power7 must equal generic exponentiation to 7"); + } + printf("cubic extension (φ³ = φ + 1) vs naive polynomial arithmetic: %zu pairs, mul/square/power7\n", + as.size()); +} + +// =========================================================================== +// Layer 3 — ★ the EXTERNAL anchor: seven FB rounds are RPO256. +// =========================================================================== + +// RPO256's permutation composed from the kernel's FB round — rpo.rs:567-583. +void rpo_permute(uint64_t s[12]) { + rpx::fb_round(s, 0); + rpx::fb_round(s, 1); + rpx::fb_round(s, 2); + rpx::fb_round(s, 3); + rpx::fb_round(s, 4); + rpx::fb_round(s, 5); + rpx::fb_round(s, 6); + for (int i = 0; i < 12; ++i) s[i] = goldilocks::canonical(s[i]); +} + +// miden's `hash_elements` in this lane convention — a transcription of the +// test-only `rpo.rs:747-767`: capacity lane 8 takes `len % 8`, the rate is +// OVERWRITTEN, the tail zero-padded, the digest is lanes 0-3. +void miden_hash_elements(const uint64_t *elements, size_t n, uint64_t out[4]) { + uint64_t state[12] = {0}; + state[8] = (uint64_t)(n % 8); + size_t i = 0; + for (size_t k = 0; k < n; ++k) { + state[i++] = elements[k]; + if (i == 8) { + rpo_permute(state); + i = 0; + } + } + if (i > 0) { + for (; i < 8; ++i) state[i] = 0; + rpo_permute(state); + } + for (int d = 0; d < 4; ++d) out[d] = state[d]; +} + +void seven_fb_rounds_reproduce_the_miden_rpo_vectors() { + check(NUM_MIDEN_HASH_ELEMENTS == 19, "miden vector table lost entries"); + int matched = 0; + for (int n = 0; n < NUM_MIDEN_HASH_ELEMENTS; ++n) { + uint64_t elements[19]; + for (int k = 0; k <= n; ++k) elements[k] = (uint64_t)k; + uint64_t got[4]; + miden_hash_elements(elements, (size_t)n + 1, got); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && got[d] == MIDEN_HASH_ELEMENTS[n][d]; + if (!ok) { + printf("FAIL miden hash_elements(0..=%d)\n got %llu %llu %llu %llu\n want %llu %llu %llu %llu\n", + n, (unsigned long long)got[0], (unsigned long long)got[1], + (unsigned long long)got[2], (unsigned long long)got[3], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][0], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][1], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][2], + (unsigned long long)MIDEN_HASH_ELEMENTS[n][3]); + ++failures; + } else { + ++matched; + } + } + // The compress layout, pinned the way rpo.rs:789-795 pins it: one + // permutation of `[0..8 ‖ 0⁴]` is the eight-element vector, so + // `[left ‖ right ‖ zero capacity]` with left = 0..4, right = 4..8 IS + // `Rpo256::merge` — the layout `rpx::compress` builds. + { + uint64_t s[12] = {0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 0, 0}; + rpo_permute(s); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && s[d] == MIDEN_HASH_ELEMENTS[7][d]; + check(ok, "permute([0..8 ‖ 0⁴]) must be miden's eight-element vector (compress layout)"); + } + printf("★ EXTERNAL: seven fb_round(s, r) = RPO256 vs miden-crypto hash_elements: %d/19 matched\n", + matched); +} + +// =========================================================================== +// Layer 4 — ★ the Rust oracle. +// +// ⚠ Every comparison here is RAW: `s[i] == v.output[i]`, never +// `canon(s[i]) == …`. The tables are canonical by construction (the generator +// canonicalises), and `permute` ends in a canonicalisation loop that makes +// digests byte-comparable to the host's; a check that canonicalised the kernel +// side would pass with that loop deleted, and so would a raw check on outputs +// that merely happen to be canonical — all but a 2^-32 slice per lane. The +// "canonicalisation witness" row and `the_canonicalisation_loop_is_pinned…` +// below are what make the loop observable. +// =========================================================================== + +void rpx_permutation_matches_the_rust_oracle() { + check(NUM_RPX_PERMUTATION_VECTORS >= 8, + "Rust-oracle permutation table must hold >= 8 vectors (run the generator, see rpx_kat_vectors.h)"); + bool saw_zero = false, saw_p_minus_one = false; + int matched = 0; + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + const RpxPermutationVector &v = RPX_PERMUTATION_VECTORS[n]; + bool all_zero = true, all_pm1 = true; + uint64_t s[12]; + for (int i = 0; i < 12; ++i) { + s[i] = v.input[i]; + all_zero = all_zero && v.input[i] == 0; + all_pm1 = all_pm1 && v.input[i] == P - 1; + } + saw_zero = saw_zero || all_zero; + saw_p_minus_one = saw_p_minus_one || all_pm1; + rpx::permute(s); + bool ok = true; + for (int i = 0; i < 12; ++i) ok = ok && s[i] == v.output[i]; + if (!ok) { + printf("FAIL rpx permutation vector %d (%s)\n", n, v.name); + for (int i = 0; i < 12; ++i) { + if (s[i] != v.output[i]) { + printf(" lane %2d got %llu (raw) want %llu\n", i, (unsigned long long)s[i], + (unsigned long long)v.output[i]); + } + } + ++failures; + } else { + ++matched; + } + } + check(saw_zero, "the permutation table must include the all-zero state"); + check(saw_p_minus_one, "the permutation table must include the all-(p-1) state"); + printf("★ ORACLE: rpx::permute vs Rust Rpx256::permute: %d/%d vectors matched\n", matched, + NUM_RPX_PERMUTATION_VECTORS); +} + +// The array-form transcription of `algebraic_commit::sponge_leaf` (:169-184), +// over the kernel's permutation — so the STREAMING struct's block bookkeeping +// is checked against the direct transcription at every length, independently +// of which lengths the oracle table carries. +void ref_sponge_leaf(const uint64_t *felts, size_t n, uint64_t digest[4]) { + uint64_t state[12] = {0}; + state[8] = (uint64_t)(n % 8); + state[9] = 0x4C4D464Cull; // u32::from_le_bytes(b"LFML") + if (n == 0) { + for (int d = 0; d < 4; ++d) digest[d] = state[d]; + return; + } + for (size_t start = 0; start < n; start += 8) { + for (size_t lane = 0; lane < 8; ++lane) { + state[lane] = (start + lane < n) ? felts[start + lane] : 0; + } + rpx::permute(state); + } + for (int d = 0; d < 4; ++d) digest[d] = state[d]; +} + +void leaf_sponge_matches_the_rust_oracle() { + // The streaming struct against the array transcription, lengths 0..40. + { + uint64_t seed = 0x1EAF; + std::vector felts(40); + for (size_t i = 0; i < felts.size(); ++i) felts[i] = sample(seed, i); + for (size_t n = 0; n <= felts.size(); ++n) { + uint64_t got[4], want[4]; + rpx::sponge_leaf(felts.data(), n, got); + ref_sponge_leaf(felts.data(), n, want); + check(memcmp(got, want, sizeof(got)) == 0, "rpx::Sponge must equal the sponge_leaf transcription"); + } + uint64_t empty[4] = {1, 1, 1, 1}; + rpx::sponge_leaf(felts.data(), 0, empty); + check(empty[0] == 0 && empty[1] == 0 && empty[2] == 0 && empty[3] == 0, + "the empty leaf's digest is the zero rate lanes, with NO permutation"); + uint64_t one[4]; + rpx::sponge_leaf(felts.data(), 1, one); + check(one[0] != 0 || one[1] != 0 || one[2] != 0 || one[3] != 0, "a one-felt leaf must permute"); + printf("leaf: rpx::Sponge vs sponge_leaf transcription at 41 lengths (0..40)\n"); + } + // The oracle table: exactly the gate's seven lengths. + std::set lengths; + for (int n = 0; n < NUM_RPX_LEAF_VECTORS; ++n) lengths.insert(RPX_LEAF_VECTORS[n].len); + const uint32_t required[7] = {0, 1, 7, 8, 9, 16, 17}; + bool all_present = NUM_RPX_LEAF_VECTORS > 0; + for (int k = 0; k < 7; ++k) all_present = all_present && lengths.count(required[k]) == 1; + check(all_present, + "Rust-oracle leaf table must hold lengths 0, 1, 7, 8, 9, 16, 17 (run the generator, see rpx_kat_vectors.h)"); + int matched = 0; + for (int n = 0; n < NUM_RPX_LEAF_VECTORS; ++n) { + const RpxLeafVector &v = RPX_LEAF_VECTORS[n]; + check(v.len <= (uint32_t)RPX_LEAF_KAT_MAX_FELTS, "leaf vector wider than the table row"); + uint64_t got[4]; + rpx::sponge_leaf(v.felts, v.len, got); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && got[d] == v.digest[d]; + if (!ok) { + printf("FAIL rpx leaf vector len=%u\n got %llu %llu %llu %llu\n want %llu %llu %llu %llu\n", + v.len, (unsigned long long)got[0], (unsigned long long)got[1], + (unsigned long long)got[2], (unsigned long long)got[3], + (unsigned long long)v.digest[0], (unsigned long long)v.digest[1], + (unsigned long long)v.digest[2], (unsigned long long)v.digest[3]); + ++failures; + } else { + ++matched; + } + } + printf("★ ORACLE: rpx::sponge_leaf vs Rust sponge_leaf(Rpx): %d/%d lengths matched\n", matched, + NUM_RPX_LEAF_VECTORS); +} + +void parent_matches_the_rust_oracle() { + check(NUM_RPX_PARENT_VECTORS >= 1, + "Rust-oracle parent table must hold >= 1 vector (run the generator, see rpx_kat_vectors.h)"); + int matched = 0; + for (int n = 0; n < NUM_RPX_PARENT_VECTORS; ++n) { + const RpxParentVector &v = RPX_PARENT_VECTORS[n]; + uint64_t got[4]; + rpx::compress(v.left, v.right, got); + bool ok = true; + for (int d = 0; d < 4; ++d) ok = ok && got[d] == v.digest[d]; + if (!ok) { + printf("FAIL rpx parent vector %d (%s)\n got %llu %llu %llu %llu\n want %llu %llu %llu %llu\n", + n, v.name, (unsigned long long)got[0], (unsigned long long)got[1], + (unsigned long long)got[2], (unsigned long long)got[3], + (unsigned long long)v.digest[0], (unsigned long long)v.digest[1], + (unsigned long long)v.digest[2], (unsigned long long)v.digest[3]); + ++failures; + } else { + ++matched; + } + // Structure: a parent is ONE permutation of `[l ‖ r ‖ 0⁴]`, and the + // order of the children matters. + uint64_t s[12] = {v.left[0], v.left[1], v.left[2], v.left[3], v.right[0], v.right[1], + v.right[2], v.right[3], 0, 0, 0, 0}; + rpx::permute(s); + check(memcmp(s, got, sizeof(got)) == 0, "compress must be permute([l ‖ r ‖ 0⁴]) truncated"); + uint64_t swapped[4]; + rpx::compress(v.right, v.left, swapped); + bool same_children = memcmp(v.left, v.right, sizeof(swapped)) == 0; + check(same_children || memcmp(swapped, got, sizeof(got)) != 0, "compress(r, l) must differ from compress(l, r)"); + } + printf("★ ORACLE: rpx::compress vs Rust HasherKind::Rpx.compress: %d/%d parents matched\n", matched, + NUM_RPX_PARENT_VECTORS); +} + +// ★ The pin on the canonicalisation loop. The witness row's M-round MDS output +// lane 0 is `p − ARK1[6][0] + 1`, so the device's final `add` returns the raw +// twin `p + 1` for a field value of 1 — deterministically, since neither that +// sum nor the MDS reduction can wrap there. Replaying the rounds without the +// loop must therefore show a lane ≥ p (or the witness has gone stale and no +// longer witnesses anything), and `permute` must then return the oracle's +// canonical digits RAW — which a kernel without the loop cannot. +void the_canonicalisation_loop_is_pinned_by_the_witness() { + const RpxPermutationVector *w = nullptr; + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + if (strcmp(RPX_PERMUTATION_VECTORS[n].name, "canonicalisation witness") == 0) { + w = &RPX_PERMUTATION_VECTORS[n]; + } + } + check(w != nullptr, + "the permutation table must carry the 'canonicalisation witness' row (run the generator, see rpx_kat_vectors.h)"); + if (w == nullptr) return; + + uint64_t s[12]; + memcpy(s, w->input, sizeof(s)); + rpx::fb_round(s, 0); + rpx::ext_round(s, 1); + rpx::fb_round(s, 2); + rpx::ext_round(s, 3); + rpx::fb_round(s, 4); + rpx::ext_round(s, 5); + rpx::final_round(s, 6); + int twins = 0; + for (int i = 0; i < 12; ++i) twins += (s[i] >= P) ? 1 : 0; + check(twins > 0, "the witness must leave a raw lane >= p before the canonicalisation loop"); + check(s[0] == P + 1, "the witness's lane 0 must be the raw twin p + 1 before the loop"); + for (int i = 0; i < 12; ++i) { + check(canon(s[i]) == w->output[i], "the witness's field values must be the oracle's"); + } + + uint64_t full[12]; + memcpy(full, w->input, sizeof(full)); + rpx::permute(full); + const bool loop_present = memcmp(full, w->output, sizeof(full)) == 0; + check(loop_present, + "permute must return the witness's digits RAW — the canonicalisation loop is missing"); + if (loop_present) { + printf("★ canonicalisation pin: witness leaves %d raw lane(s) >= p before the loop; permute() returns them canonical\n", + twins); + } +} + +// =========================================================================== +// Layer 5 — negative controls and the representation. +// =========================================================================== + +void rpx_is_not_rpo() { + // rpx.rs:577-580: the two share constants, an MDS and three of seven + // rounds, so a schedule bug could collapse one into the other. + uint64_t a[12], b[12]; + for (int i = 0; i < 12; ++i) a[i] = b[i] = (uint64_t)i; + rpx::permute(a); + rpo_permute(b); + check(memcmp(a, b, sizeof(a)) != 0, "RPX must not be RPO on the same state"); + uint64_t z[12] = {0}; + rpx::permute(z); + bool nonzero = false; + for (int i = 0; i < 12; ++i) nonzero = nonzero || z[i] != 0; + check(nonzero, "with its constants present, permute(0) must not be 0"); + printf("negative control: RPX(0..12) != RPO(0..12); permute(0) != 0\n"); +} + +void raw_and_canonical_inputs_agree_and_outputs_are_canonical() { + uint64_t seed = 0xCA0; + for (int k = 0; k < 32; ++k) { + uint64_t raw[12], can[12]; + for (int i = 0; i < 12; ++i) { + // Canonical values below 2^32 − 1 have a raw twin `c + p`; alternate + // lanes between the twin and a plain canonical value. + const uint64_t c = splitmix(seed) % 0xFFFFFFFFull; + const bool twin = ((k + i) % 3) != 0; + can[i] = twin ? c : splitmix(seed) % P; + raw[i] = twin ? c + P : can[i]; + } + uint64_t r1[12], c1[12]; + memcpy(r1, raw, sizeof(r1)); + memcpy(c1, can, sizeof(c1)); + rpx::permute(r1); + rpx::permute(c1); + check(memcmp(r1, c1, sizeof(r1)) == 0, "permute(raw) must equal permute(canonical)"); + for (int i = 0; i < 12; ++i) check(c1[i] < P, "permute output must be canonical"); + + uint64_t d_raw[4], d_can[4]; + rpx::sponge_leaf(raw, 12, d_raw); + rpx::sponge_leaf(can, 12, d_can); + check(memcmp(d_raw, d_can, sizeof(d_raw)) == 0, "sponge_leaf(raw) must equal sponge_leaf(canonical)"); + + uint64_t p_raw[4], p_can[4]; + rpx::compress(raw, raw + 4, p_raw); + rpx::compress(can, can + 4, p_can); + check(memcmp(p_raw, p_can, sizeof(p_raw)) == 0, "compress(raw) must equal compress(canonical)"); + } + printf("representation: raw [p, 2^64) inputs agree with canonical; outputs canonical (32 states)\n"); +} + +void every_input_lane_reaches_the_output() { + uint64_t seed = 0x1A4E; + uint64_t base[12]; + for (int i = 0; i < 12; ++i) base[i] = splitmix(seed) % P; + uint64_t out0[12]; + memcpy(out0, base, sizeof(out0)); + rpx::permute(out0); + for (int k = 0; k < 12; ++k) { + uint64_t s[12]; + memcpy(s, base, sizeof(s)); + s[k] = (s[k] + 1) % P; + rpx::permute(s); + check(memcmp(s, out0, sizeof(s)) != 0, "changing one input lane must move the output"); + } + printf("negative control: each of the 12 input lanes moves the output\n"); +} + +// =========================================================================== +// Layer 6 — the cost model, counted. +// =========================================================================== + +struct Counted { + unsigned long long mul, dot3, add; +}; + +template +Counted count_ops(F f) { + rpx::g_ops = rpx::OpCount{0, 0, 0}; + f(); + return Counted{rpx::g_ops.mul, rpx::g_ops.dot3, rpx::g_ops.add}; +} + +void the_cost_model_is_what_the_header_claims() { + uint64_t s[12]; + for (int i = 0; i < 12; ++i) s[i] = (uint64_t)i + 1; + const Counted fb = count_ops([&] { rpx::fb_round(s, 0); }); + const Counted ext = count_ops([&] { rpx::ext_round(s, 1); }); + const Counted fin = count_ops([&] { rpx::final_round(s, 6); }); + const Counted all = count_ops([&] { rpx::permute(s); }); + const Counted rpo = count_ops([&] { rpo_permute(s); }); + const Counted inv = count_ops([&] { (void)rpx::inv_sbox(s[0]); }); + const Counted fwd = count_ops([&] { (void)rpx::sbox(s[0]); }); + const Counted emul = count_ops([&] { + rpx::CubicExt a = {s[0], s[1], s[2]}; + (void)rpx::ext_mul(a, a); + }); + + printf("op counts (Goldilocks mul | 3-term dot3 | add); MDS = 288 narrow 32x32 MACs each, uncounted:\n"); + printf(" x^7 (sbox) %4llu | %3llu | %3llu\n", fwd.mul, fwd.dot3, fwd.add); + printf(" x^{1/7} (inv_sbox) %4llu | %3llu | %3llu (63 squarings + 9 products)\n", inv.mul, + inv.dot3, inv.add); + printf(" ext_mul %4llu | %3llu | %3llu (9 wide products in 3 reductions)\n", emul.mul, + emul.dot3, emul.add); + printf(" FB round %4llu | %3llu | %3llu + 2 MDS\n", fb.mul, fb.dot3, fb.add); + printf(" E round %4llu | %3llu | %3llu (4 triples x power7)\n", ext.mul, ext.dot3, + ext.add); + printf(" M round %4llu | %3llu | %3llu + 1 MDS\n", fin.mul, fin.dot3, fin.add); + printf(" RPX permutation %4llu | %3llu | %3llu + 7 MDS (2016 MACs)\n", all.mul, all.dot3, + all.add); + printf(" RPO permutation %4llu | %3llu | %3llu + 14 MDS (4032 MACs), for comparison\n", + rpo.mul, rpo.dot3, rpo.add); + printf(" inverse S-box share of RPX field multiplications: %llu / %llu\n", 3ull * 12ull * inv.mul, + all.mul); + + check(fwd.mul == 4 && inv.mul == 72, "S-box costs must be 4 and 72 multiplications"); + check(emul.mul == 0 && emul.dot3 == 3 && emul.add == 2, "ext_mul must be 3 dot3 + 2 adds"); + check(fb.mul == 912 && fb.dot3 == 0 && fb.add == 48, "FB round must be 912 mul / 48 add"); + check(ext.mul == 0 && ext.dot3 == 48 && ext.add == 44, "E round must be 48 dot3 / 44 add"); + check(fin.mul == 0 && fin.dot3 == 0 && fin.add == 24, "M round must be 24 add"); + check(all.mul == 2736 && all.dot3 == 144 && all.add == 300, "RPX permutation must be 2736 mul / 144 dot3 / 300 add"); + check(rpo.mul == 6384 && rpo.dot3 == 0 && rpo.add == 336, "RPO permutation must be 6384 mul / 336 add"); +} + +// =========================================================================== +// Layer 7 — the leaf kernels, the Merkle compressors and the probe, replayed +// thread by thread through the shim. +// +// What a leaf hashes is the CPU `leaves_bit_reversed_grouped` sequence — +// bit-reversed rows, each column by column, an ext3 element as its three +// components — and the hash over it is the `sponge_leaf` transcription pinned +// in layer 4. So each kernel is checked for its READ PATTERN and its node +// ENCODING (`digest_to_commitment`: four canonical felts, big-endian), with the +// permutation anchored separately above. Raw `[p, 2^64)` values are fed in, +// since that is what an LDE buffer holds. +// =========================================================================== + +uint64_t reverse_index(uint64_t i, uint32_t log_n) { return __brevll(i) >> (64 - log_n); } + +// The host leaf over `felts`: `sponge_leaf`, then `digest_to_commitment`. +void expected_leaf(const std::vector &felts, uint8_t out[32]) { + uint64_t d[4]; + ref_sponge_leaf(felts.data(), felts.size(), d); + for (int i = 0; i < 4; ++i) { + const uint64_t c = canon(d[i]); + for (int b = 0; b < 8; ++b) out[i * 8 + b] = (uint8_t)(c >> (56 - 8 * b)); + } +} + +std::string hex32(const uint8_t *b) { + std::string s(64, '\0'); + for (int i = 0; i < 32; ++i) snprintf(&s[i * 2], 3, "%02x", (unsigned)b[i]); + return s; +} + +void check_leaves(const std::vector &got, const std::vector> &want, + const char *what) { + if (got.size() != want.size() * 32) { + printf("FAIL %s: leaf count %zu vs %zu\n", what, got.size() / 32, want.size()); + ++failures; + return; + } + for (size_t i = 0; i < want.size(); ++i) { + uint8_t expect[32]; + expected_leaf(want[i], expect); + if (memcmp(got.data() + i * 32, expect, 32) != 0) { + printf("FAIL %s: leaf %zu\n got %s\n want %s\n", what, i, hex32(got.data() + i * 32).c_str(), + hex32(expect).c_str()); + ++failures; + return; + } + } +} + +// The two column-major base kernels: one leaf per bit-reversed row, and one per +// bit-reversed row pair. +void base_leaf_kernels_read_the_specified_felts() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 5ull, 8ull, 17ull}) { + const uint64_t n = 1ull << log_n; + std::vector cols(num_cols * n); + uint64_t seed = log_n * 31 + num_cols; + for (size_t i = 0; i < cols.size(); ++i) cols[i] = sample(seed, i); + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + rpx_leaves_base_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + const uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) want[leaf].push_back(cols[c * n + br]); + } + check_leaves(out, want, "rpx_leaves_base_batched"); + } + { + const uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_row_pair_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + const uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < num_cols; ++c) want[leaf].push_back(cols[c * n + br]); + } + } + check_leaves(out, want, "rpx_leaves_base_row_pair_batched"); + } + } + } + printf("base leaf kernels: read pattern + node encoding match the CPU leaf spec\n"); +} + +// The ext3 kernels over the de-interleaved three-slab layout. +void ext3_leaf_kernels_read_the_specified_felts() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t num_cols : {1ull, 3ull, 11ull}) { + const uint64_t n = 1ull << log_n; + std::vector cols(num_cols * 3 * n); + uint64_t seed = log_n * 17 + num_cols; + for (size_t i = 0; i < cols.size(); ++i) cols[i] = sample(seed, i); + { + std::vector out(n * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, n) { + rpx_leaves_ext3_batched(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(n); + for (uint64_t leaf = 0; leaf < n; ++leaf) { + const uint64_t br = reverse_index(leaf, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) want[leaf].push_back(cols[(c * 3 + k) * n + br]); + } + } + check_leaves(out, want, "rpx_leaves_ext3_batched"); + } + { + const uint64_t num_leaves = n / 2; + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_comp_poly_leaves_ext3(cols.data(), n, num_cols, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int j = 0; j < 2; ++j) { + const uint64_t br = reverse_index(2 * leaf + j, log_n); + for (uint64_t c = 0; c < num_cols; ++c) { + for (uint64_t k = 0; k < 3; ++k) want[leaf].push_back(cols[(c * 3 + k) * n + br]); + } + } + } + check_leaves(out, want, "rpx_comp_poly_leaves_ext3"); + } + } + } + printf("ext3 + comp-poly leaf kernels: read pattern + node encoding match the CPU leaf spec\n"); +} + +// ★ COSET leaves — the WHIR shape, and the only two kernels in this file that +// the per-table branch has no twin for, so this is the ONLY place their read +// pattern is pinned without a GPU. +// +// Leaf `j` holds the fold coset of `j`: the `block` codeword positions strided +// by `num_leaves`. Every other leaf kernel here reads a ROW; these read a +// STRIDE, which is exactly the kind of index arithmetic that compiles, runs and +// silently hashes the wrong elements. The expectation below is built from the +// definition — `codeword[j + t * num_leaves]` — not from a second call to the +// kernel. +void coset_leaf_kernels_read_the_specified_felts() { + // `block` is `2^log_folding`; 16 is the shipped posture, the others bracket + // it. `num_leaves` deliberately includes a value that is not a multiple of + // any block, to catch a bound computed from the wrong quantity. + for (uint64_t block : {2ull, 4ull, 16ull}) { + for (uint64_t num_leaves : {1ull, 2ull, 8ull, 13ull}) { + // --- base field ------------------------------------------------- + { + std::vector codeword(num_leaves * block); + uint64_t seed = 0xC05E7; + for (size_t i = 0; i < codeword.size(); ++i) codeword[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_coset(codeword.data(), num_leaves, block, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t j = 0; j < num_leaves; ++j) { + for (uint64_t t = 0; t < block; ++t) { + want[j].push_back(codeword[j + t * num_leaves]); + } + } + check_leaves(out, want, "rpx_leaves_base_coset"); + } + // --- cubic extension -------------------------------------------- + { + std::vector codeword(num_leaves * block * 3); + uint64_t seed = 0xC05E8; + for (size_t i = 0; i < codeword.size(); ++i) codeword[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_ext3_coset(codeword.data(), num_leaves, block, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t j = 0; j < num_leaves; ++j) { + for (uint64_t t = 0; t < block; ++t) { + const uint64_t *at = codeword.data() + (j + t * num_leaves) * 3; + for (int k = 0; k < 3; ++k) want[j].push_back(at[k]); + } + } + check_leaves(out, want, "rpx_leaves_ext3_coset"); + } + } + } + + // ✓ A coset is NOT a contiguous run. At block > 1 the strided read and the + // contiguous one differ, so a kernel that had dropped the stride would have + // passed everything above only if `num_leaves == 1`. This is the control + // that says the stride is really being exercised. + { + const uint64_t num_leaves = 4, block = 4; + std::vector codeword(num_leaves * block); + uint64_t seed = 0xC05E9; + for (size_t i = 0; i < codeword.size(); ++i) codeword[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_coset(codeword.data(), num_leaves, block, out.data()); + } + std::vector contiguous(codeword.begin(), codeword.begin() + block); + uint8_t as_contiguous[32]; + expected_leaf(contiguous, as_contiguous); + if (memcmp(out.data(), as_contiguous, 32) == 0) { + printf("FAIL rpx_leaves_base_coset: leaf 0 equals the CONTIGUOUS run, so the " + "stride is not being read\n"); + ++failures; + } + } + printf("coset leaf kernels: strided read pattern + node encoding match the WHIR leaf spec\n"); +} + +// FRI leaves: two consecutive ext3 values from an interleaved vector, six felts, +// no bit reversal — the Pair backend's `hash_data`. +void fri_leaf_kernel_reads_the_specified_felts() { + for (uint64_t num_leaves : {1ull, 2ull, 8ull, 33ull}) { + std::vector evals(num_leaves * 6); + uint64_t seed = 0xF41; + for (size_t i = 0; i < evals.size(); ++i) evals[i] = sample(seed, i); + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { rpx_fri_leaves_ext3(evals.data(), num_leaves, out.data()); } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int i = 0; i < 6; ++i) want[leaf].push_back(evals[leaf * 6 + i]); + } + check_leaves(out, want, "rpx_fri_leaves_ext3"); + } + printf("FRI leaf kernel: read pattern + node encoding match the Pair backend's leaf\n"); +} + +// The row-major row-pair kernels, plain and column-ranged, every non-empty +// range. +void row_major_leaf_kernels_read_the_specified_felts() { + for (uint32_t log_n : {2u, 4u, 6u}) { + for (uint64_t m : {1ull, 5ull, 13ull}) { + const uint64_t n = 1ull << log_n; + const uint64_t num_leaves = n / 2; + std::vector data(n * m); + uint64_t seed = log_n * 7 + m; + for (size_t i = 0; i < data.size(); ++i) data[i] = sample(seed, i); + { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_row_major_row_pair(data.data(), m, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + const uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = 0; c < m; ++c) want[leaf].push_back(data[br * m + c]); + } + } + check_leaves(out, want, "rpx_leaves_base_row_major_row_pair"); + } + for (uint64_t cs = 0; cs < m; ++cs) { + for (uint64_t ce = cs + 1; ce <= m; ++ce) { + std::vector out(num_leaves * 32, 0); + CUDA_HOST_FOR_EACH_THREAD(t, num_leaves) { + rpx_leaves_base_row_major_row_pair_range(data.data(), m, cs, ce, n, log_n, out.data()); + } + std::vector> want(num_leaves); + for (uint64_t leaf = 0; leaf < num_leaves; ++leaf) { + for (int k = 0; k < 2; ++k) { + const uint64_t br = reverse_index(2 * leaf + k, log_n); + for (uint64_t c = cs; c < ce; ++c) want[leaf].push_back(data[br * m + c]); + } + } + check_leaves(out, want, "rpx_leaves_base_row_major_row_pair_range"); + } + } + } + } + printf("row-major leaf kernels: read pattern + node encoding match the CPU leaf spec, all column ranges\n"); +} + +// The host parent over two nodes: decode big-endian, compress, encode. +void expected_parent(const uint8_t *left, const uint8_t *right, uint8_t out[32]) { + uint64_t l[4], r[4], d[4]; + for (int i = 0; i < 4; ++i) { + l[i] = r[i] = 0; + for (int b = 0; b < 8; ++b) { + l[i] = (l[i] << 8) | left[i * 8 + b]; + r[i] = (r[i] << 8) | right[i * 8 + b]; + } + } + rpx::compress(l, r, d); + for (int i = 0; i < 4; ++i) { + for (int b = 0; b < 8; ++b) out[i * 8 + b] = (uint8_t)(canon(d[i]) >> (56 - 8 * b)); + } +} + +// The Merkle level kernel replayed thread by thread up a 16-leaf tree, and the +// tail kernel replayed as a one-thread block (the shim's barrier is a no-op, +// so a single thread walking every pair in order is the tail's sequential +// meaning), both against the host parent over the same node buffer. +void merkle_compressors_match_the_host_parent() { + const uint64_t num_leaves = 16; + const uint64_t total = 2 * num_leaves - 1; + // Nodes must be VALID digests (canonical big-endian felts) for the decode to + // be meaningful, so the leaves are hashes of random felts, not random bytes. + std::vector leaves(num_leaves * 32); + uint64_t seed = 0x3E11; + for (uint64_t i = 0; i < num_leaves; ++i) { + std::vector f = {sample(seed, i), sample(seed, i + 1000)}; + expected_leaf(f, leaves.data() + i * 32); + } + + std::vector want(total * 32, 0); + memcpy(want.data() + (num_leaves - 1) * 32, leaves.data(), leaves.size()); + for (uint64_t parent = num_leaves - 1; parent-- > 0;) { + expected_parent(want.data() + (2 * parent + 1) * 32, want.data() + (2 * parent + 2) * 32, + want.data() + parent * 32); + } + + // Level by level. + std::vector by_level(total * 32, 0); + memcpy(by_level.data() + (num_leaves - 1) * 32, leaves.data(), leaves.size()); + uint64_t level_begin = num_leaves - 1; + while (level_begin != 0) { + const uint64_t new_begin = level_begin / 2; + const uint64_t n_pairs = level_begin - new_begin; + CUDA_HOST_FOR_EACH_THREAD(t, n_pairs) { rpx_merkle_level(by_level.data(), new_begin, n_pairs); } + level_begin = new_begin; + } + check(by_level == want, "rpx_merkle_level must reproduce the host tree"); + + // The tail, in one go. + std::vector by_tail(total * 32, 0); + memcpy(by_tail.data() + (num_leaves - 1) * 32, leaves.data(), leaves.size()); + blockIdx.x = 0; + threadIdx.x = 0; + blockDim.x = 1; + rpx_merkle_tail(by_tail.data(), num_leaves - 1); + check(by_tail == want, "rpx_merkle_tail must reproduce the host tree"); + printf("Merkle compressors: level and tail kernels reproduce the host parent over a 16-leaf tree\n"); +} + +// The permutation probe replayed over the oracle table: pins its indexing. +void permute_probe_matches_the_oracle_table() { + std::vector in(NUM_RPX_PERMUTATION_VECTORS * 12), out(NUM_RPX_PERMUTATION_VECTORS * 12, 0); + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + for (int i = 0; i < 12; ++i) in[n * 12 + i] = RPX_PERMUTATION_VECTORS[n].input[i]; + } + CUDA_HOST_FOR_EACH_THREAD(t, NUM_RPX_PERMUTATION_VECTORS) { + rpx_permute_probe(in.data(), (uint64_t)NUM_RPX_PERMUTATION_VECTORS, out.data()); + } + bool ok = true; + for (int n = 0; n < NUM_RPX_PERMUTATION_VECTORS; ++n) { + for (int i = 0; i < 12; ++i) ok = ok && out[n * 12 + i] == RPX_PERMUTATION_VECTORS[n].output[i]; + } + check(ok, "rpx_permute_probe must reproduce the oracle table, raw"); + printf("permute probe: %d oracle states reproduced through the kernel entry point\n", + NUM_RPX_PERMUTATION_VECTORS); +} + +// --------------------------------------------------------------------------- +// Layer 8 — the proof-of-work grind kernel. +// +// `rpx_grind_search` is the one kernel whose correctness is a statement about +// a HOST predicate rather than about the permutation: it has to search for the +// nonces `stark::grinding::is_valid_nonce` accepts over +// `AlgebraicDigest`, and the two reach the sponge by different +// routes — the host through a byte buffer and `felts_from_bytes`, the kernel by +// building the five felts directly. Table 5's rows are that agreement, pinned +// on the Rust side (the generator asserts the two routes match over the whole +// scanned range) and reproduced here through the kernel entry point. +// +// Driven single-threaded, so what this covers is the per-candidate arithmetic +// and the loop bounds. That the parallel `atomicMin` reduction returns the same +// answer is a property of a real launch, and belongs to the GPU test. +// --------------------------------------------------------------------------- +uint64_t run_grind(const uint64_t inner[4], uint8_t factor, uint64_t base, uint64_t count) { + const uint64_t limit = (uint64_t)1 << (64 - factor); + uint64_t result = UINT64_MAX; + CUDA_HOST_SINGLE_THREAD(); + rpx_grind_search(inner, limit, base, count, (volatile unsigned long long *)&result); + return result; +} + +void grind_kernel_finds_the_nonce_the_host_predicate_accepts() { + for (int n = 0; n < NUM_RPX_GRIND_VECTORS; ++n) { + const RpxGrindVector &v = RPX_GRIND_VECTORS[n]; + char what[160]; + + // The nonce is in range: the kernel returns exactly it, and it is the + // SMALLEST — a stride or bounds defect would still return a *valid* + // nonce, just not the first one, which plain validity cannot see. + snprintf(what, sizeof what, + "rpx_grind_search must return the host's nonce %llu (seed 0x%02x, factor %u)", + (unsigned long long)v.nonce, v.seed_byte, v.factor); + check(run_grind(v.inner_felts, v.factor, 0, v.nonce + 1) == v.nonce, what); + + // One short of it: nothing in `[0, nonce)` passes, so the kernel must + // leave the sentinel alone. This is what says the nonce above is the + // first — and it exercises the not-found path the launcher's range walk + // depends on. + snprintf(what, sizeof what, + "rpx_grind_search must find nothing below %llu (seed 0x%02x, factor %u)", + (unsigned long long)v.nonce, v.seed_byte, v.factor); + check(run_grind(v.inner_felts, v.factor, 0, v.nonce) == UINT64_MAX, what); + + // Offset base: the same nonce is found when the block starts inside the + // range, which pins that `base` participates rather than being ignored. + if (v.nonce > 0) { + snprintf(what, sizeof what, + "rpx_grind_search must honour base (seed 0x%02x, factor %u)", v.seed_byte, + v.factor); + check(run_grind(v.inner_felts, v.factor, v.nonce, 1) == v.nonce, what); + } + + // ★ THE ENDIANNESS CONTROL. `felts_from_bytes` reads big-endian and + // keccak's `inner_hash_lanes` reads little-endian; feeding the kernel + // the wrong one compiles, runs, and searches a message the host never + // hashes. The same kernel on the byte-swapped inner hash must give the + // oracle's `le_nonce` — which is the sentinel for every row here, i.e. + // it finds NOTHING where the correct reading finds the nonce. + uint64_t swapped[4]; + for (int i = 0; i < 4; ++i) { + uint64_t x = v.inner_felts[i], y = 0; + for (int b = 0; b < 8; ++b) y |= ((x >> (8 * b)) & 0xffull) << (8 * (7 - b)); + swapped[i] = y; + } + snprintf(what, sizeof what, + "the LE reading must not answer the BE one (seed 0x%02x, factor %u)", v.seed_byte, + v.factor); + check(run_grind(swapped, v.factor, 0, v.nonce + 1) == v.le_nonce, what); + } + printf("grind kernel: %d oracle rows — nonce, minimality, base, and the endianness control\n", + NUM_RPX_GRIND_VECTORS); +} + +} // namespace + +int main() { + printf("RPX device-kernel known-answer tests, host-compiled from crypto/math-cuda/kernels/rpx.cu\n\n"); + printf("-- layer 1/2: primitives and building blocks vs independent algorithms --\n"); + field_primitives_match_schoolbook_arithmetic(); + mds_matches_its_per_term_definition(); + sboxes_are_the_seventh_power_and_its_inverse(); + cubic_extension_matches_naive_polynomial_arithmetic(); + printf("\n-- layer 3: the external anchor --\n"); + seven_fb_rounds_reproduce_the_miden_rpo_vectors(); + printf("\n-- layer 4: the Rust oracle --\n"); + rpx_permutation_matches_the_rust_oracle(); + leaf_sponge_matches_the_rust_oracle(); + parent_matches_the_rust_oracle(); + the_canonicalisation_loop_is_pinned_by_the_witness(); + printf("\n-- layer 5: negative controls and representation --\n"); + rpx_is_not_rpo(); + raw_and_canonical_inputs_agree_and_outputs_are_canonical(); + every_input_lane_reaches_the_output(); + printf("\n-- layer 6: cost model --\n"); + the_cost_model_is_what_the_header_claims(); + printf("\n-- layer 7: leaf kernels, Merkle compressors and the probe, replayed thread by thread --\n"); + base_leaf_kernels_read_the_specified_felts(); + ext3_leaf_kernels_read_the_specified_felts(); + fri_leaf_kernel_reads_the_specified_felts(); + coset_leaf_kernels_read_the_specified_felts(); + row_major_leaf_kernels_read_the_specified_felts(); + merkle_compressors_match_the_host_parent(); + permute_probe_matches_the_oracle_table(); + printf("\n-- layer 8: the proof-of-work grind kernel against the host predicate --\n"); + grind_kernel_finds_the_nonce_the_host_predicate_accepts(); + if (failures != 0) { + printf("\n*** %d FAILURE(S) ***\n", failures); + return 1; + } + printf("\nALL HOST KAT CHECKS PASS\n"); + printf("NOTE: arithmetic only. nvcc acceptance and GPU execution are phase 2's GPU tests.\n"); + return 0; +} diff --git a/crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h b/crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h new file mode 100644 index 000000000..c89900c7a --- /dev/null +++ b/crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h @@ -0,0 +1,195 @@ +// Known-answer vectors for the RPX device kernel (`kernels/rpx.cu`), embedded +// rather than parsed at run time — a table cannot have a zero-vector run, and +// `rpx_host_kat.cpp` asserts every count below as well. +// +// This file is DATA. Table 1 is transcribed; Tables 2-4 are printed by the Rust +// oracle and pasted. Nothing here is computed by the harness. +#pragma once +#include + +// --------------------------------------------------------------------------- +// Table 1 — miden-crypto's RPO256 `hash_elements` known-answer vectors. +// EXTERNAL: nothing in this repository produced these seventy-six numbers. +// +// Transcribed mechanically (a script over the source, not by hand) from +// `prover/src/lfm/rpo.rs:624-739` `MIDEN_HASH_ELEMENTS`, itself transcribed +// from miden-crypto `src/hash/algebraic_sponge/rescue/rpo/tests.rs`. Entry `n` +// is the digest of the field elements `[0, 1, …, n]` under miden's convention: +// capacity lane 8 = `len mod 8`, rate OVERWRITTEN, zero-padded tail, digest = +// lanes 0..4 (rpo.rs:741-767). +// +// What they pin in `rpx.cu`: RPX's FB round IS RPO's round, so seven +// `fb_round<0..7>` compose to RPO256 and must reproduce this table. That pins +// ARK1/ARK2 (all seven rows), the MDS row AND its orientation, both S-box +// exponents including the 72-step inverse chain, the u128-property MDS, and the +// sponge lane convention — externally. Entries 1-7 and 9-19 exercise padding, +// 8 and 16 the exact-block path, everything above 8 the capacity carry. +// --------------------------------------------------------------------------- +inline constexpr int NUM_MIDEN_HASH_ELEMENTS = 19; +inline constexpr uint64_t MIDEN_HASH_ELEMENTS[NUM_MIDEN_HASH_ELEMENTS][4] = { + {8563248028282119176ull, 14757918088501470722ull, 14042820149444308297ull, 7607140247535155355ull}, + {8762449007102993687ull, 4386081033660325954ull, 5000814629424193749ull, 8171580292230495897ull}, + {16710087681096729759ull, 10808706421914121430ull, 14661356949236585983ull, 5683478730832134441ull}, + {5309818427047650994ull, 17172251659920546244ull, 8288476618870804357ull, 18080473279382182941ull}, + {3647545403045515695ull, 3358383208908083302ull, 8797161010298072910ull, 2412100201132087248ull}, + {8409780526028662686ull, 214479528340808320ull, 13626616722984122219ull, 13991752159726061594ull}, + {4800410126693035096ull, 8293686005479024958ull, 16849389505608627981ull, 12129312715917897796ull}, + {5421234586123900205ull, 9738602082989433872ull, 7017816005734536787ull, 8635896173743411073ull}, + {11707446879505873182ull, 7588005580730590001ull, 4664404372972250366ull, 17613162115550587316ull}, + {6991094187713033844ull, 10140064581418506488ull, 1235093741254112241ull, 16755357411831959519ull}, + {18007834547781860956ull, 5262789089508245576ull, 4752286606024269423ull, 15626544383301396533ull}, + {5419895278045886802ull, 10747737918518643252ull, 14861255521757514163ull, 3291029997369465426ull}, + {16916426112258580265ull, 8714377345140065340ull, 14207246102129706649ull, 6226142825442954311ull}, + {7320977330193495928ull, 15630435616748408136ull, 10194509925259146809ull, 15938750299626487367ull}, + {9872217233988117092ull, 5336302253150565952ull, 9650742686075483437ull, 8725445618118634861ull}, + {12539853708112793207ull, 10831674032088582545ull, 11090804155187202889ull, 105068293543772992ull}, + {7287113073032114129ull, 6373434548664566745ull, 8097061424355177769ull, 14780666619112596652ull}, + {17147873541222871127ull, 17350918081193545524ull, 5785390176806607444ull, 12480094913955467088ull}, + {17273934282489765074ull, 8007352780590012415ull, 16690624932024962846ull, 8137543572359747206ull}, +}; + +// --------------------------------------------------------------------------- +// Tables 2-4 — THE RUST ORACLE. miden publishes no RPX known-answer table +// (rpx.rs "PROVENANCE"), so the host `Rpx256` is the oracle the kernel is +// pinned to. Printed by `prover/tests/rpx_host_kat_vectors.rs`: +// +// cargo test -p lambda-vm-prover --test rpx_host_kat_vectors -- --ignored --nocapture +// +// and pasted verbatim between the `>>> BEGIN` / `<<< END` markers. Inputs are +// derived there from fixed seeds and printed next to the outputs, so this file +// is self-contained. All values are canonical (`< p`). +// +// Table 2 — the bare permutation: all-zero, all-(p−1), `0..12`, alternating, +// two one-hot lanes, four seeded random states, and the row named +// "canonicalisation witness" — an input whose output lane 0 is the +// raw twin `p + 1` before the kernel's final canonicalisation loop +// (derived by `rpx_canon_witness.py`; the harness replays it). +// Table 3 — the leaf sponge (`algebraic_commit::sponge_leaf`) at 0, 1, 7, 8, +// 9, 16 and 17 felts; `felts[]` is zero beyond `len`. +// Table 4 — the parent `compress(l, r)`. +// Table 5 — the proof-of-work grind: the inner hash's four BIG-endian felts +// and the smallest valid nonce, at three grinding factors, taken +// through the production predicate `stark::grinding::is_valid_nonce` +// over `GrindingDigest`; `le_nonce` is what the same +// kernel answers on the LITTLE-endian reading of the same inner +// hash, and must differ. +// --------------------------------------------------------------------------- +struct RpxPermutationVector { + const char *name; + uint64_t input[12]; + uint64_t output[12]; +}; + +inline constexpr int RPX_LEAF_KAT_MAX_FELTS = 17; +struct RpxLeafVector { + uint32_t len; + uint64_t felts[RPX_LEAF_KAT_MAX_FELTS]; + uint64_t digest[4]; +}; + +struct RpxParentVector { + const char *name; + uint64_t left[4]; + uint64_t right[4]; + uint64_t digest[4]; +}; + +struct RpxGrindVector { + uint8_t seed_byte; // the 32-byte grinding seed is this byte, repeated + uint8_t factor; // limit = 1 << (64 - factor) + uint64_t inner_felts[4]; // BIG-endian reading of the 32-byte inner hash + uint64_t nonce; // the smallest nonce the host predicate accepts + uint64_t le_nonce; // the same kernel's answer on the LE reading +}; + +// >>> BEGIN RUST-ORACLE TABLES — generated by +// cargo test -p lambda-vm-prover --test rpx_host_kat_vectors -- --ignored --nocapture +// (prover/tests/rpx_host_kat_vectors.rs). Paste verbatim; do not edit by hand. + +inline constexpr int NUM_RPX_PERMUTATION_VECTORS = 11; +inline constexpr RpxPermutationVector RPX_PERMUTATION_VECTORS[NUM_RPX_PERMUTATION_VECTORS] = { + {"all-zero", + {0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {8760086638283468260ull, 18228666152919569253ull, 4041825754230271128ull, 16906183286731764961ull, 4664375192219530269ull, 271590372761485506ull, 5612474514543166805ull, 8933101171974180471ull, 1556877437237031065ull, 7026397410864970258ull, 15101742939622740655ull, 4524429088483979565ull}}, + {"all-(p-1)", + {18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull, 18446744069414584320ull}, + {7040074528728887770ull, 10474261017970959672ull, 6160748039461781206ull, 9121740959127811013ull, 7259505444118573102ull, 6771278935515018093ull, 18386914479072470354ull, 17160039764143535473ull, 1815780993504974800ull, 17309055307915657636ull, 5977169316478634398ull, 4250629519753691035ull}}, + {"lanes 0..12", + {0ull, 1ull, 2ull, 3ull, 4ull, 5ull, 6ull, 7ull, 8ull, 9ull, 10ull, 11ull}, + {3614697924784493998ull, 4917065433670799835ull, 12893407190838344317ull, 16769932886818781879ull, 17010299523770013195ull, 9826755761378503206ull, 1872785960340665977ull, 7783788981462778586ull, 45778307605882514ull, 7437259891664617628ull, 17010253034795346176ull, 6863075881906649113ull}}, + {"alternating 0 / p-1", + {0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull, 0ull, 18446744069414584320ull}, + {12839024277220712229ull, 1805658617972785851ull, 11708832562581917975ull, 2207339757364837492ull, 457975798096500050ull, 15656130651128894835ull, 3485815494872446363ull, 10687968103458402677ull, 10384294655078062232ull, 1487178939946482695ull, 12310600107129561463ull, 18388841767871832735ull}}, + {"one-hot lane 0", + {1ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {8423002511501289529ull, 6761734748202534392ull, 17987336675889252592ull, 14012777376234247391ull, 15293807115397414812ull, 15290017247514670316ull, 10548590320248089637ull, 9459855167724924903ull, 10549768014422457033ull, 13045952392708592140ull, 3310663857881768756ull, 7584810783597460418ull}}, + {"one-hot lane 11", + {0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 1ull}, + {18436166275486246010ull, 14000894557392395452ull, 10767551609857089912ull, 12516698445112165012ull, 13131066481882004069ull, 9858979976142754244ull, 11402636824743634507ull, 10600727647028701714ull, 11200928220555719329ull, 7317761145158236061ull, 16857331551667002769ull, 16879508045812612150ull}}, + {"random #1", + {303661977215735624ull, 5244312915552057691ull, 9817756985327366386ull, 15550273871372065883ull, 5764353057648779642ull, 16198122637140758912ull, 7462824619408935181ull, 3819703627846067891ull, 10378249170554155646ull, 11473525795005675318ull, 8246620909628934680ull, 4793144044164964625ull}, + {15068850129045079395ull, 15287067578585128518ull, 13369562146120321575ull, 10561395445440413441ull, 9652992371859647144ull, 4276856065313043669ull, 5527444075954724606ull, 7786060382866009904ull, 16451772069079981395ull, 198876956612152837ull, 15815343923951857286ull, 16122126005548441717ull}}, + {"random #2", + {5204068831683694011ull, 601380814908431653ull, 258667317409904638ull, 8486618912357792900ull, 16418043790810515027ull, 10319906524521615844ull, 8286207029444254408ull, 17770698039797916230ull, 12310900488678790115ull, 11195649432216834664ull, 13332813278057623446ull, 16898620073423657296ull}, + {9523479656024648568ull, 5510889535488554715ull, 8599619832581755346ull, 3318619196771576895ull, 12581966946741818379ull, 12200018864226225973ull, 4385075405488142149ull, 8051813774684357414ull, 3019406547981393239ull, 7453667634993074437ull, 9864259903669275905ull, 6156796699962990553ull}}, + {"random #3", + {13533914130435405040ull, 15234815373149021432ull, 10183913914233800905ull, 9526239132464493568ull, 5375977297676405297ull, 5765388458641153407ull, 4908125521970473579ull, 4421030864271922041ull, 15641279279696351384ull, 16893076439662162884ull, 7253714011824234117ull, 14616467593891397000ull}, + {15514260962038810700ull, 190255547175148079ull, 15766300047716671382ull, 10145444481310349528ull, 6135237967701788176ull, 11361125511081474273ull, 9927005018743801106ull, 17211086950078547559ull, 10833199580085782023ull, 13634008743082439065ull, 6687522208929839355ull, 3545879585555314384ull}}, + {"random #4", + {389113379214421922ull, 1947929307647562990ull, 667333451960644926ull, 3487966933876559811ull, 4195385248066926332ull, 2153180418459341747ull, 2727969323864685845ull, 29633526854483411ull, 990649808851061115ull, 1355410330370587755ull, 11605520071788416946ull, 4884409355120715354ull}, + {7025469669435110295ull, 17270957437800346011ull, 13702589935335807876ull, 3666927270871270796ull, 16666721215101099684ull, 531487850530305024ull, 15550553335698242665ull, 8959489596577675281ull, 11020601500923732075ull, 16110845767020565054ull, 4778394010005480449ull, 7715575140819562371ull}}, + {"canonicalisation witness", + {15055324559807314153ull, 10242425218814686878ull, 9326602342065331773ull, 15451135068213333861ull, 17942679252967467289ull, 9284164080268346300ull, 5090350781253234438ull, 9328738269791029498ull, 18385380985273671691ull, 3238854716908013220ull, 5495049682105235955ull, 15773368383738726538ull}, + {1ull, 9023883145409261355ull, 5839950281880325605ull, 5697668523532261268ull, 13033383890974728246ull, 14801658261553133914ull, 3025695522291518949ull, 12907720598453111556ull, 14827640614007773288ull, 14642633917625231592ull, 3090884930034198616ull, 2894057710100710233ull}}, +}; + +inline constexpr int NUM_RPX_LEAF_VECTORS = 7; +inline constexpr RpxLeafVector RPX_LEAF_VECTORS[NUM_RPX_LEAF_VECTORS] = { + {0u, + {0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {0ull, 0ull, 0ull, 0ull}}, + {1u, + {14681136968691612469ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {16400186102935428425ull, 12817983163740802970ull, 13449009006350391325ull, 2209445548780258712ull}}, + {7u, + {2664695409302073823ull, 17298518342786888931ull, 17367242851809685948ull, 13566833943477212382ull, 6789339537410032387ull, 5202847705797706501ull, 6869254230765949416ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {2289345357069865559ull, 8509266780934512918ull, 13810958145049281723ull, 5769431894700133303ull}}, + {8u, + {3521541860211663897ull, 5585621328801039182ull, 3314063895810834828ull, 6286715337571703139ull, 9272399501810688383ull, 17378448552699642502ull, 9663403628134293866ull, 8225575178453385283ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {14052993739410942603ull, 8384701950754250190ull, 11473922331550289114ull, 16644313465254305812ull}}, + {9u, + {15923052634311126246ull, 10423360080185943333ull, 4604695570423031111ull, 15959212651715575539ull, 4341333374822801132ull, 3169961389438585383ull, 7059846953207312362ull, 6231597079039193598ull, 14413065529971692326ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull, 0ull}, + {15453186885173297365ull, 11395279108043639065ull, 15954005188014354330ull, 2854892578083306874ull}}, + {16u, + {9660685076555889599ull, 4027567791223379602ull, 11432600011703367870ull, 6441517771629429252ull, 8272264386868866348ull, 16565648022353132158ull, 16844837242675693755ull, 12942506659476152817ull, 11839051358503478840ull, 1846358602548732379ull, 118703897581348635ull, 14480592082795401517ull, 12015885875590073011ull, 7433808365622677077ull, 13247077855319202624ull, 17837888200692576115ull, 0ull}, + {18135965004560326100ull, 1948492279228612931ull, 17772968542724134453ull, 12116464713281646840ull}}, + {17u, + {14169068543591784110ull, 12906798066534908639ull, 1898134805181953282ull, 3700382130787856361ull, 10455317549184205797ull, 1564511190292879407ull, 5954886065046464361ull, 10320234224067579215ull, 17095047743397986079ull, 8434180870595516882ull, 17706992797230203878ull, 813257427175065251ull, 13312284969041468023ull, 15899260221184366980ull, 5770785055252949875ull, 11176385994046687487ull, 8142444693260481147ull}, + {430819886588247494ull, 10400188655761849356ull, 3003730485848167815ull, 13484379440855863704ull}}, +}; + +inline constexpr int NUM_RPX_PARENT_VECTORS = 2; +inline constexpr RpxParentVector RPX_PARENT_VECTORS[NUM_RPX_PARENT_VECTORS] = { + {"digits 0..8", + {0ull, 1ull, 2ull, 3ull}, + {4ull, 5ull, 6ull, 7ull}, + {10386438340626196987ull, 10820383641790274229ull, 5711121060683785078ull, 11046870009967209474ull}}, + {"random", + {10430052842846219471ull, 4016318112082366688ull, 17186674839268073878ull, 16606021345024473049ull}, + {1405896845186672283ull, 13799610513837549656ull, 17571522367612218822ull, 18082329703565322844ull}, + {18019606657308693634ull, 10494109104368286361ull, 7943124261980338770ull, 17971490172695632899ull}}, +}; + +inline constexpr int NUM_RPX_GRIND_VECTORS = 3; +inline constexpr RpxGrindVector RPX_GRIND_VECTORS[NUM_RPX_GRIND_VECTORS] = { + {90u, 12u, + {17047917526726690733ull, 2027278666509702433ull, 4678289907902145381ull, 4242003890993108442ull}, + 1342ull, 18446744073709551615ull}, + {17u, 13u, + {3807340077325453675ull, 129745844021573959ull, 15014385560057355003ull, 944573484564438641ull}, + 300ull, 18446744073709551615ull}, + {32u, 14u, + {5597071933014793605ull, 8702110216523445336ull, 2882478612521280078ull, 9429844132731097150ull}, + 705ull, 18446744073709551615ull}, +}; +// <<< END RUST-ORACLE TABLES diff --git a/crypto/math-cuda/tests/whir_commit.rs b/crypto/math-cuda/tests/whir_commit.rs index 8ec0fbc99..ac20f5b91 100644 --- a/crypto/math-cuda/tests/whir_commit.rs +++ b/crypto/math-cuda/tests/whir_commit.rs @@ -8,15 +8,44 @@ //! The reference is `multilinear`'s own pipeline rather than a copy of it: a //! second implementation of the Möbius transform in this file could drift from //! the one the prover runs with every test still green. +//! +//! ⚠ **Both hashes, and the device path asserted TAKEN.** The commit falls back +//! to the host silently when the device declines (`gpu::commit_tree_ext3` +//! returns `None` below a size threshold, on a missing card, or under +//! `LAMBDA_VM_NO_GPU_WHIR_COMMIT`). A parity test that let that happen would be +//! comparing the host against itself and passing — the exact shape recon B +//! found in `whir_fold.rs`, where two host paths were checked against each +//! other. So `commit_codeword_to_host` is called directly, which has no host +//! fallback at all: it either runs the kernels or returns an error this test +//! turns into a failure. use math::field::element::FieldElement; use math::field::goldilocks::GoldilocksField as F; use multilinear::mle::Mle; use multilinear::whir::{self, Domain}; use multilinear::whir_commit::{CodewordCommitment, verify_opening}; +use multilinear::whir_hash::{DeviceHashKey, KeccakWhir, RpxWhir, WhirHash}; type FE = FieldElement; +/// The dispatch key `math-cuda` takes, from the configuration that names it. +/// +/// ⚠ A MIRROR of `DeviceHashKey::into_math_cuda`, not a call to it. That method +/// is `#[cfg(feature = "cuda")]` on `multilinear`, and `multilinear` is a plain +/// dev-dependency here — enabling its cuda feature from this crate would be a +/// dependency cycle through the crate under test. The production bridge is +/// asserted total and bijective at compile time inside `multilinear` +/// (`whir_hash.rs`); what this mirror can still get wrong is being written +/// backwards, which `the_device_dispatch_really_selects_the_kernel_family` +/// would catch — it requires the two keys to produce DIFFERENT trees, and a +/// swapped mirror produces the same two trees in the other order. +fn device_key() -> math_cuda::DeviceHash { + match H::DEVICE { + DeviceHashKey::Keccak256 => math_cuda::DeviceHash::Keccak256, + DeviceHashKey::Rpx256 => math_cuda::DeviceHash::Rpx256, + } +} + /// A polynomial with no structure a kernel could accidentally satisfy. fn poly(num_vars: usize, seed: u64) -> Mle { let evals: Vec = (0..(1u64 << num_vars)) @@ -25,17 +54,18 @@ fn poly(num_vars: usize, seed: u64) -> Mle { Mle::new(evals).expect("power of two") } -fn parity(num_vars: usize, log_blowup: usize, log_folding: usize) { +fn parity(num_vars: usize, log_blowup: usize, log_folding: usize) { let f = poly(num_vars, 1 + num_vars as u64); let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + // No host fallback on this entry point: it hashes on the device or errors. let (device_codeword, nodes) = - math_cuda::whir::commit_codeword_to_host(&raw, log_blowup, log_folding) - .expect("device commit (needs a GPU)"); + math_cuda::whir::commit_codeword_to_host(&raw, log_blowup, log_folding, device_key::()) + .unwrap_or_else(|e| panic!("device commit under {} (needs a GPU): {e:?}", H::NAME)); let domain = Domain::::new(num_vars + log_blowup).expect("domain"); let host_codeword = whir::encode::(&whir::lift_coefficients(&f), &domain).expect("encode"); - let host = CodewordCommitment::new(&host_codeword, log_folding).expect("host commit"); + let host = CodewordCommitment::<_, H>::new(&host_codeword, log_folding).expect("host commit"); assert_eq!(device_codeword.len(), host_codeword.len()); for (i, (device, host)) in device_codeword.iter().zip(&host_codeword).enumerate() { @@ -51,9 +81,9 @@ fn parity(num_vars: usize, log_blowup: usize, log_folding: usize) { .map(|node| node.try_into().expect("32 bytes")) .collect(); let codeword: Vec = device_codeword.into_iter().map(FE::from_raw).collect(); - let device = CodewordCommitment::from_precomputed(codeword, nodes, log_folding) + let device = CodewordCommitment::<_, H>::from_precomputed(codeword, nodes, log_folding) .expect("device commitment"); - assert_eq!(device.root(), host.root(), "roots differ"); + assert_eq!(device.root(), host.root(), "roots differ under {}", H::NAME); assert_eq!(device.num_leaves(), host.num_leaves()); // The root alone would pass on a tree whose inner nodes are garbage below @@ -61,8 +91,9 @@ fn parity(num_vars: usize, log_blowup: usize, log_folding: usize) { for index in [0, 1, device.num_leaves() / 3, device.num_leaves() - 1] { let opening = device.open(index).expect("open"); assert!( - verify_opening(&device.root(), index, &opening), - "device opening at {index} does not verify" + verify_opening::<_, H>(&device.root(), index, &opening), + "device opening at {index} does not verify under {}", + H::NAME ); assert_eq!( opening.values, @@ -72,21 +103,55 @@ fn parity(num_vars: usize, log_blowup: usize, log_folding: usize) { } } +/// The shapes: both sides of the fused-8-level NTT threshold, a fold width that +/// is not the whole blowup, and the Möbius windows — below the contiguous +/// kernel, exactly one window, one window plus a tiled level, and several full +/// tiles with a partial one on top. +fn every_shape() { + parity::(14, 2, 4); + parity::(16, 2, 4); + parity::(11, 1, 1); + parity::(12, 3, 5); + + parity::(5, 2, 3); + parity::(8, 2, 4); + parity::(9, 1, 2); + parity::(13, 2, 5); + parity::(17, 1, 4); +} + #[test] fn device_commit_matches_the_host_pipeline() { - // Both sides of the fused-8-level NTT threshold, and a fold width that is - // not the whole blowup. - parity(14, 2, 4); - parity(16, 2, 4); - parity(11, 1, 1); - parity(12, 3, 5); - - // The Mobius windows: below the contiguous kernel (every level on its own), - // exactly one contiguous window, one window plus a single tiled level, and - // several full tiles with a partial one on top. - parity(5, 2, 3); - parity(8, 2, 4); - parity(9, 1, 2); - parity(13, 2, 5); - parity(17, 1, 4); + every_shape::(); +} + +/// ★ The same, under the algebraic hash — the kernels this branch adds. +#[test] +fn device_commit_matches_the_host_pipeline_under_rpx() { + every_shape::(); +} + +/// ★★ The two hashes really do build DIFFERENT trees on the device. +/// +/// Without this, both tests above would pass on a dispatch that ignored its key +/// and always launched keccak's kernels: the RPX host reference would be +/// compared against a keccak device tree and fail — unless the host side had +/// been mis-wired the same way, which is exactly the failure a single-hash +/// parity test cannot see. Comparing the two device roots directly closes it. +#[test] +fn the_device_dispatch_really_selects_the_kernel_family() { + let f = poly(12, 7); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + + let (_, keccak) = + math_cuda::whir::commit_codeword_to_host(&raw, 2, 4, device_key::()) + .expect("device commit (needs a GPU)"); + let (_, rpx) = math_cuda::whir::commit_codeword_to_host(&raw, 2, 4, device_key::()) + .expect("device commit (needs a GPU)"); + + assert_eq!(keccak.len(), rpx.len(), "the node layout must not change"); + assert_ne!( + keccak, rpx, + "the two kernel families produced identical trees, so the key is not being read" + ); } diff --git a/crypto/math-cuda/tests/whir_fold.rs b/crypto/math-cuda/tests/whir_fold.rs index 999183335..2a3decdcc 100644 --- a/crypto/math-cuda/tests/whir_fold.rs +++ b/crypto/math-cuda/tests/whir_fold.rs @@ -16,6 +16,7 @@ use math::field::goldilocks::GoldilocksField as Gl; use multilinear::mle::Mle; use multilinear::whir::{self, Domain}; use multilinear::whir_commit::CodewordCommitment; +use multilinear::whir_hash::KeccakWhir; type FE3 = FieldElement; type FE = FieldElement; @@ -74,13 +75,19 @@ fn the_device_ext3_commit_matches_the_host() { let (folded, _) = whir::fold_codeword_k::(&cw, &domain, &alphas).expect("device fold"); - let device = CodewordCommitment::from_codeword(folded.clone(), 4).expect("device commit"); - let host = CodewordCommitment::from_codeword_on_host(folded, 4).expect("host commit"); + let device = CodewordCommitment::<_, KeccakWhir>::from_codeword(folded.clone(), 4) + .expect("device commit"); + let host = + CodewordCommitment::<_, KeccakWhir>::from_codeword_on_host(folded, 4).expect("host commit"); assert_eq!(device.root(), host.root(), "roots differ"); for index in [0, 1, device.num_leaves() / 3, device.num_leaves() - 1] { let opening = device.open(index).expect("open"); assert!( - multilinear::whir_commit::verify_opening(&device.root(), index, &opening), + multilinear::whir_commit::verify_opening::<_, KeccakWhir>( + &device.root(), + index, + &opening + ), "device opening at {index} does not verify" ); } diff --git a/crypto/math-cuda/tests/whir_tree_cache.rs b/crypto/math-cuda/tests/whir_tree_cache.rs new file mode 100644 index 000000000..e3e001c27 --- /dev/null +++ b/crypto/math-cuda/tests/whir_tree_cache.rs @@ -0,0 +1,463 @@ +//! ★★ H4's result of record — a commitment does NOT keep its tree, and the +//! bytes it holds are only its codeword. +//! +//! Needs a GPU: +//! +//! ```text +//! cargo test -p math-cuda --release --test whir_tree_cache -- --nocapture +//! ``` +//! +//! # What this is about +//! +//! `commit()` builds the tree, takes the root and drops the buffer; `paths()` +//! rebuilds it to read a kilobyte per query out of it. Every commitment on this +//! path is opened, so every one pays for two leaf-hash passes over its codeword +//! — ~25 s of RPX device hashing on a real block, about half of it that second +//! pass. H4 cached the first tree to remove the second pass. It was measured on +//! the card and it LOST, ~+15 s in both hashes. +//! +//! # Why keeping the tree cannot work here, which is what these tests pin +//! +//! Not because the cache missed — it returned exactly the hashing it promised. +//! Because the retention is one tree per commitment IN THE GROUP, not one tree. +//! `StackedCommitment::commit` builds every chain's commitment before it +//! returns, since all the roots enter the transcript before any query index is +//! drawn, and the openings follow one chain at a time. So the last chain's tree +//! would live from its commit to the end of the proof, and no placement of an +//! eviction call bounds that peak: all N trees exist before the first opening. +//! Ten chains at half a gigabyte put the card at 96%, after which device +//! allocations fail, commits fall back to the host, and the host grows ~1.5 GiB +//! per fallen-back chain. +//! +//! `multilinear::whir_commit`'s `paths` said this in its doc comment before any +//! of it was built, and the reservation in `StackedCommitment::commit` — "nine +//! codewords of room instead of sixteen" — budgets a retained codeword per +//! commitment and no tree. +//! +//! # ⚠ Why the counts are PER CODEWORD +//! +//! An earlier run of this file failed two tests for a reason that was not the +//! code under test: `leaf_hash_calls()` is process-wide, the tests share one +//! binary, and cargo runs them in parallel — so one test read 5 where it +//! expected 1 purely because its neighbours were committing at the same time. +//! An assertion on a global counter is an assertion about every test in the +//! binary. The counting assertions use `DeviceCodeword::tree_builds()`; the +//! process-wide counter keeps one test of its own, and that one takes the lock. +//! +//! # ⚠ Why the memory guard asks the DRIVER +//! +//! `Backend::reserved_bytes()` counts what callers promised, so it is silent +//! about device memory allocated without a reservation — and it reads baseline +//! while the card fills, which is how the H4 arm's retention stayed invisible +//! to a unit test that passed. It is also trivially at baseline now, which is a +//! check that cannot fail. So the guard samples `free_vram_bytes()` across four +//! live, unopened commitments and asserts what they took is their codewords and +//! nothing else. That one fails if a tree is ever held past the call that +//! builds it, wherever the holding is written. + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use math_cuda::DeviceHash; +use math_cuda::whir::{leaf_hash_calls, reset_leaf_hash_calls}; +use multilinear::mle::Mle; +use multilinear::whir::{self, Domain}; +use multilinear::whir_commit::{CodewordCommitment, verify_opening}; +use multilinear::whir_hash::{DeviceHashKey, KeccakWhir, RpxWhir, WhirHash}; +use std::sync::Mutex; + +type FE = FieldElement; + +/// ★ Taken by EVERY test in this file, because every one of them commits, and a +/// commit moves two process-wide quantities: the leaf-hash counter and the +/// device reservation total. +/// +/// The per-codeword counts added earlier make the *counting* assertions immune +/// to the scheduler, but one proposition is irreducibly global — "dropping a +/// codeword gives its promised bytes BACK" is a statement about +/// `Backend::reserved_bytes()`, and there is no per-codeword handle left to ask +/// once the codeword is gone. So these tests take turns. +/// +/// That costs nothing real: they share one card and serialise on it regardless. +/// What the lock buys over `--test-threads=1` is that the property holds however +/// the suite is invoked, rather than only when someone remembers the flag. +/// +/// Poisoning is ignored so one failure does not cascade into unrelated tests — +/// a lesson this branch learned once already, in `hash_metrics_tests`. +static DEVICE_GLOBALS: Mutex<()> = Mutex::new(()); + +fn exclusive() -> std::sync::MutexGuard<'static, ()> { + DEVICE_GLOBALS.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Mirror of `DeviceHashKey::into_math_cuda`, which is cuda-gated on +/// `multilinear` and so unreachable from this crate's dev-dependency. The +/// production bridge is asserted bijective at compile time inside +/// `multilinear`; a swapped mirror here would fail +/// [`the_two_hashes_build_different_trees`] below. +fn key() -> DeviceHash { + match H::DEVICE { + DeviceHashKey::Keccak256 => DeviceHash::Keccak256, + DeviceHashKey::Rpx256 => DeviceHash::Rpx256, + } +} + +/// A polynomial with no structure a kernel could accidentally satisfy. +fn poly(num_vars: usize) -> Mle { + let evals: Vec = (0..(1u64 << num_vars)) + .map(|i| FE::from(i.wrapping_mul(6364136223846793005).wrapping_add(11) >> 11)) + .collect(); + Mle::new(evals).expect("power of two") +} + +/// Commit on the device at `log_blowup = 2`, above `COMMIT_THRESHOLD`, so the +/// device path is the one taken. +fn commit_on_device( + num_vars: usize, + log_folding: usize, + hash: DeviceHash, +) -> (math_cuda::whir::DeviceCodeword, [u8; 32]) { + let f = poly(num_vars); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit (needs a GPU)") +} + +/// ★★ (1) THE COUNT. One leaf-hash pass per tree built: the commit's, and +/// one more for each round that opens it. +/// +/// This is the cost H4 tried to remove and the number that says whether anyone +/// has quietly re-added a cache. It is asserted as an integer in both +/// directions — a commitment that read 1 after an opening would mean a tree is +/// being kept, which is the state this file exists to forbid. +#[test] +fn a_commitment_hashes_its_leaves_once_per_tree_it_builds() { + let _exclusive = exclusive(); + for (name, hash) in [("keccak", key::()), ("rpx", key::())] { + let (codeword, _root) = commit_on_device(14, 4, hash); + assert_eq!( + codeword.tree_builds(), + 1, + "{name}: the commit itself must hash the leaves exactly once" + ); + + let _ = codeword.paths(4, &[0, 1, 7], hash).expect("paths"); + assert_eq!( + codeword.tree_builds(), + 2, + "{name}: an opening builds its own tree — a 1 here means one is kept" + ); + + // …and again, because a cache that served once and then evicted would + // read 2 on the line above too. + let _ = codeword.paths(4, &[2, 3], hash).expect("paths"); + assert_eq!( + codeword.tree_builds(), + 3, + "{name}: and a second opening builds a third" + ); + } +} + +/// ★ (2) THE PATHS ARE RIGHT. Two independent codewords over the same +/// evaluations give the same root and the same paths. +/// +/// The count alone is satisfied by a build that hands back stale or wrong +/// nodes: the paths would be internally consistent and wrong. +#[test] +fn the_paths_are_the_ones_a_fresh_tree_gives() { + let _exclusive = exclusive(); + for (name, hash) in [("keccak", key::()), ("rpx", key::())] { + let num_vars = 12; + let log_folding = 4; + let f = poly(num_vars); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + + let (codeword, root) = math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit (needs a GPU)"); + let leaves = (raw.len() << 2) >> log_folding; + let positions: Vec = [0usize, 1, leaves / 3, leaves - 1] + .iter() + .map(|p| *p as u32) + .collect(); + let first = codeword + .paths(log_folding, &positions, hash) + .expect("paths"); + + // A second, independent codeword over the same evaluations: same + // inputs, a tree built from scratch, nothing shared with the one above. + let (fresh_codeword, fresh_root) = + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, hash) + .expect("device commit"); + let fresh = fresh_codeword + .paths(log_folding, &positions, hash) + .expect("paths"); + + assert_eq!( + root, fresh_root, + "{name}: the two commits disagree on the root" + ); + assert_eq!( + first, fresh, + "{name}: two builds over the same codeword disagree on the paths" + ); + } +} + +/// ★ The same, through the production types, so the openings are checked by +/// the verifier rather than only compared to each other. +#[test] +fn the_openings_verify_against_the_device_commitment() { + let _exclusive = exclusive(); + fn check(name: &str) { + let num_vars = 12; + let log_folding = 4; + let f = poly(num_vars); + let domain = Domain::::new(num_vars + 2).expect("domain"); + let host_codeword = + whir::encode::(&whir::lift_coefficients(&f), &domain).expect("encode"); + let host = + CodewordCommitment::<_, H>::new(&host_codeword, log_folding).expect("host commit"); + + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + let (_device, root) = + math_cuda::whir::commit_codeword(&raw, 2, log_folding, false, key::()) + .expect("device commit (needs a GPU)"); + assert_eq!(root, host.root(), "{name}: device and host roots differ"); + + for index in [0, 1, host.num_leaves() / 3, host.num_leaves() - 1] { + let opening = host.open(index).expect("open"); + assert!( + verify_opening::<_, H>(&root, index, &opening), + "{name}: opening {index} does not verify against the device root" + ); + } + } + check::("keccak"); + check::("rpx"); +} + +/// ★★ (3) THE RESERVATION SEES THE CODEWORD, AND GETS IT BACK. +/// +/// Device memory held outside the accounting is an under-count that nothing +/// reports until a second prover shares the card. What a codeword promises is +/// its own bytes and the folds that halve it — NOT a tree, which is never held +/// past the call that builds it. +#[test] +fn the_codeword_is_inside_the_reservation_and_gives_it_back() { + let _exclusive = exclusive(); + let be = math_cuda::device::backend().expect("a device"); + let hash = key::(); + let num_vars = 14; + let log_folding = 4; + + // ⚠ Read under the lock, and it is a BASELINE rather than an assumed zero: + // a sibling's reservation was what failed this test the first time it ran + // on a card (786,400 B of someone else's). What is asserted below is the + // DELTA this codeword is responsible for. + let before = be.reserved_bytes(); + let (codeword, _root) = commit_on_device(num_vars, log_folding, hash); + + let codeword_bytes = ((1u64 << num_vars) << 2) * 8; + // `2*L - 1` nodes of 32 bytes, from the shapes alone. + let leaves = ((1usize << num_vars) << 2) >> log_folding; + let tree_bytes = (2 * leaves as u64 - 1) * 32; + + // (a) This codeword's OWN promise covers its codeword — no global involved, + // so this half would hold even without the lock. + let held = codeword.reserved_bytes(); + assert!( + held >= codeword_bytes, + "the reservation holds {held} B, which does not cover the {codeword_bytes} B codeword" + ); + + // (b) …and the global grew by exactly that, as a delta. + let grown = be.reserved_bytes() - before; + assert_eq!( + grown, held, + "this codeword's promise and the global's growth must be the same bytes" + ); + + // (c) ★ and a TREE is not in the promise. The codeword has been committed, + // so a kept tree would be sitting in this number; `paths` below builds a + // second one, and neither may appear. This is the half that H4's version of + // this test asserted the other way round. + let after_open = { + let _ = codeword.paths(log_folding, &[0, 1], hash).expect("paths"); + codeword.reserved_bytes() + }; + assert_eq!( + after_open, held, + "a tree was added to the reservation: {after_open} B against {held} B, \ + and a tree here is {tree_bytes} B" + ); + + // (d) Dropping it gives every byte back. The irreducibly global + // proposition, and the reason this test holds the lock. + drop(codeword); + assert_eq!( + be.reserved_bytes(), + before, + "dropping the codeword must return the accounting to its baseline" + ); +} + +/// ★★★ (4) THE GUARD, AT GROUP SCALE. Four commitments, none opened, hold +/// four codewords and nothing else. +/// +/// This is the test H4 needed and did not have. The unit test that shipped +/// dropped ONE bare codeword and asserted the accounting returned to baseline; +/// it passed on the leaking prover, because an O(chains) peak is not a state +/// one codeword can be in and because the accounting it read is blind to bytes +/// nobody promised. Four LIVE, UNOPENED commitments is the state the group +/// actually reaches — `StackedCommitment::commit` builds all of them before the +/// first opening — and the driver's own free-memory count is the instrument +/// that cannot be fooled by where the retention is written. +/// +/// # The margin, and why it is this wide +/// +/// A codeword here is `2^20 << 2` u64 = 32 MiB. At `log_folding = 2` its tree +/// is `2^20` leaves, `(2*2^20 - 1) * 32` B = **64 MiB** — two codewords, not +/// half of one, which is the whole reason for that blocking. So four +/// codewords are 128 MiB and four codewords with their kept trees are 384 MiB, +/// and the bound sits at 256 MiB: 128 MiB of slack above the passing case and +/// 128 MiB below the failing one. +/// +/// ⚠ The slack is not decoration. `free_vram_bytes` reports what the PROCESS +/// has taken from the driver, which includes whatever one-time workspace and +/// twiddle caches the first commit of this size sets up, and those are counted +/// identically in both cases. A bound only a codeword above the passing case +/// would turn any such allocation into a false failure — and a wider blocking, +/// where the tree is half a codeword, would leave no room for one. The warm-up +/// commit below pays those costs before the sample, and the margin absorbs what +/// it misses. +/// +/// Keccak because this is a memory proposition and the two hash families build +/// identically shaped trees; the cheaper kernel keeps the test short. +#[test] +fn a_group_holds_only_its_codewords_before_any_open() { + let _exclusive = exclusive(); + let be = math_cuda::device::backend().expect("a device"); + let hash = key::(); + let num_vars = 20; + // ⚠ Not 4. At `log_folding = 2` the tree is TWO codewords rather than half + // of one, which is what puts 128 MiB between the passing and failing cases + // instead of 32. + let log_folding = 2; + + // One commit of this exact shape before the sample, so the one-time costs + // of the first — twiddles, workspaces, whatever the pool grows to hold them + // — are paid outside the window and not attributed to retention. + drop(commit_on_device(num_vars, log_folding, hash)); + + // ⚠ And without this the measurement is the POOL's, not the caller's: the + // stream-ordered allocator keeps freed blocks, and the warm-up's own + // codeword would silently serve one of the four commits below. Drain, then + // sample. + math_cuda::device::drain_and_trim().expect("drain"); + let free_before = be.free_vram_bytes().expect("cuMemGetInfo"); + + let held: Vec<_> = (0..4) + .map(|_| commit_on_device(num_vars, log_folding, hash)) + .collect(); + + let free_after = be.free_vram_bytes().expect("cuMemGetInfo"); + let taken = free_before.saturating_sub(free_after); + + let codeword_bytes = ((1u64 << num_vars) << 2) * 8; + let leaves = ((1u64 << num_vars) << 2) >> log_folding; + let tree_bytes = (2 * leaves - 1) * 32; + let bound = 8 * codeword_bytes; + let mib = |b: u64| b / (1 << 20); + assert!( + taken < bound, + "four unopened commitments took {} MiB from the device. Four codewords \ + are {} MiB and the bound is {} MiB; a tree is {} MiB, so four of those \ + kept would read {} MiB. Something is held per commitment.", + mib(taken), + mib(4 * codeword_bytes), + mib(bound), + mib(tree_bytes), + mib(4 * (codeword_bytes + tree_bytes)), + ); + + // The commitments are alive up to here, which is the whole point: a `drop` + // any earlier and the assertion would be about a group that had already + // been released. + drop(held); +} + +/// ★ (5) THE BLOCKING IS THE ONE THAT WAS ASKED FOR. +/// +/// The dangerous outcome a cache made reachable — serving a tree that answers a +/// different question, whose paths are internally consistent and wrong — is +/// unreachable once nothing is kept, and this pins that it stays unreachable: +/// each call builds for the `log_folding` it was given, and the shapes differ. +#[test] +fn a_tree_is_built_for_the_blocking_that_is_asked_for() { + let _exclusive = exclusive(); + let hash = key::(); + let (codeword, _root) = commit_on_device(14, 4, hash); + assert_eq!(codeword.tree_builds(), 1); + + // Same codeword, different blocking: its own tree, its own pass. + let at_two = codeword.paths(2, &[0, 1], hash).expect("paths at k=2"); + assert_eq!( + codeword.tree_builds(), + 2, + "the opening must build a tree for the blocking it was given" + ); + + // And the rebuild answered the question that was asked: at k=2 the tree has + // four times the leaves, so each path is two levels deeper. + let at_four = codeword.paths(4, &[0, 1], hash).expect("paths at k=4"); + assert_eq!(codeword.tree_builds(), 3, "and a third for the k=4 opening"); + assert_eq!( + at_two.len(), + at_four.len() + 2 * 2 * 32, + "a k=2 tree's paths must be two levels deeper than a k=4 tree's" + ); +} + +/// ✓ The cache is per hash too — the same codeword under two keys must build +/// two different trees, or the dispatch key is being ignored one level up. +#[test] +fn the_two_hashes_build_different_trees() { + let _exclusive = exclusive(); + let f = poly(12); + let raw: Vec = f.evals().iter().map(|v| *v.value()).collect(); + let (_k, keccak_root) = + math_cuda::whir::commit_codeword(&raw, 2, 4, false, key::()) + .expect("device commit"); + let (_r, rpx_root) = math_cuda::whir::commit_codeword(&raw, 2, 4, false, key::()) + .expect("device commit"); + assert_ne!(keccak_root, rpx_root, "the two kernel families agreed"); +} + +/// ✓ The PROCESS-WIDE counter tracks the same passes — it is what the bench +/// prints, so it needs a test of its own. +/// +/// Takes the lock across the whole window, because every other test in this +/// binary commits too and the counter cannot tell whose work it is counting. +/// That is exactly why the assertions above do not use it. +#[test] +fn the_process_wide_counter_tracks_the_same_passes() { + let _exclusive = exclusive(); + let hash = key::(); + + reset_leaf_hash_calls(); + let (codeword, _root) = commit_on_device(12, 4, hash); + let after_commit = leaf_hash_calls(); + assert_eq!(after_commit, 1, "one commit, one leaf-hash pass"); + + let _ = codeword.paths(4, &[0, 1], hash).expect("paths"); + assert_eq!( + leaf_hash_calls(), + after_commit + 1, + "an opening builds a tree, so the global counter must move by one" + ); + assert_eq!( + codeword.tree_builds(), + leaf_hash_calls(), + "with one codeword in flight the two counters must agree" + ); +} diff --git a/crypto/multilinear/src/constraint_argument.rs b/crypto/multilinear/src/constraint_argument.rs index e91bb2322..155e55310 100644 --- a/crypto/multilinear/src/constraint_argument.rs +++ b/crypto/multilinear/src/constraint_argument.rs @@ -48,6 +48,7 @@ use crate::{ sumcheck::SumcheckProof, whir_chain::ChainConfig, whir_commit::Commitment, + whir_hash::{KeccakWhir, WhirHash}, }; /// The stack width that fits every column in a single polynomial. @@ -139,13 +140,16 @@ fn weave( /// Holds the columns that were committed and, for every trace-level factor, /// where its table comes from. Weight tables are not in here: they depend on /// challenges drawn after the commitments, so the statements bring their own. -pub struct CommittedTrace, E: IsField> -where +pub struct CommittedTrace< + F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, + E: IsField, + H: WhirHash = KeccakWhir, +> where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, { data: TraceData, - stacked: StackedCommitment, + stacked: StackedCommitment, } /// A table's factors: its columns, the public tables, and what each factor @@ -350,7 +354,8 @@ impl TraceData { impl< F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, E: IsField + Send + Sync + 'static, -> CommittedTrace + H: WhirHash, +> CommittedTrace where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, @@ -413,7 +418,7 @@ where layout: StackedLayout, config: &ChainConfig, ) -> Result { - let stacked = StackedCommitment::::commit( + let stacked = StackedCommitment::::commit( layout, &crate::stacking::borrow(&columns), None, @@ -529,8 +534,8 @@ pub struct ConstraintCore { /// /// The caller must have absorbed the commitment roots and drawn whatever /// challenges its statements need, identically on both sides. -pub fn prove_statements( - trace: &CommittedTrace, +pub fn prove_statements( + trace: &CommittedTrace, weights: Vec>, rules: Vec>, claims: &[FieldElement], @@ -543,13 +548,14 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let (core, reduced_point) = prove_core::(&trace.data, weights, rules, claims, transcript)?; // Every column's value at one shared point, so the whole trace is settled // against the stack in one go. - let columns = stacked_eval::prove::( + let columns = stacked_eval::prove::( &trace.stacked, &crate::stacking::borrow(trace.columns()), trace.data().resident(), @@ -654,7 +660,7 @@ where /// the commitment says the prover is consistent with what it committed, not /// that what it committed is right. #[must_use = "the column values are the only place a known column can be checked"] -pub fn verify_statements( +pub fn verify_statements( proof: &ConstraintProof, claim_shape: TraceClaim<'_, F>, rules: &[Rule<'_, E>], @@ -670,6 +676,7 @@ where FieldElement: AsBytes + Sync + Send, T: IsTranscript, P: FnOnce(&[FieldElement]) -> Result>, Error>, + H: WhirHash, { let reduced = verify_core( &proof.core, @@ -682,7 +689,7 @@ where transcript, )?; - stacked_eval::verify::( + stacked_eval::verify::( &proof.columns, claim_shape.layout, claim_shape.roots, @@ -746,8 +753,8 @@ where /// Absorbs the roots, draws `r`, and adds `eq(r, ·)` as one more public factor /// — so `combine` sees exactly the trace's factors, and the weight costs one /// degree. -pub fn prove( - trace: &CommittedTrace, +pub fn prove( + trace: &CommittedTrace, combine: C, degree: usize, config: &ChainConfig, @@ -760,6 +767,7 @@ where FieldElement: AsBytes + Sync + Send, T: IsTranscript, C: Fn(&[FieldElement]) -> FieldElement + Sync, + H: WhirHash, { for root in trace.roots() { transcript.append_bytes(&root); @@ -783,7 +791,7 @@ where } /// Verifies the single-constraint case. See [`prove`]. -pub fn verify( +pub fn verify( proof: &ConstraintProof, claim_shape: TraceClaim<'_, F>, combine: C, @@ -800,6 +808,7 @@ where T: IsTranscript, C: Fn(&[FieldElement]) -> FieldElement + Sync, P: FnOnce(&[FieldElement]) -> Result>, Error>, + H: WhirHash, { for root in claim_shape.roots { transcript.append_bytes(root); @@ -812,7 +821,7 @@ where let rule = Rule::new(degree + 1, move |v: &[FieldElement]| { &v[weight] * combine(&v[..weight]) }); - verify_statements( + verify_statements::( proof, claim_shape, &[rule], @@ -845,10 +854,68 @@ mod tests { gkr::{self, FractionLayer, FractionTree}, selector::Selector, whir_chain::GrindBits, + whir_hash::KeccakWhir, }; type FE = FieldElement; + /// ★ These tests are the KECCAK instantiation, stated once. + /// + /// `prove` and `verify` are generic over [`WhirHash`] on the production + /// path, where the caller supplies it. Shadowing them here with pinned + /// wrappers keeps every test body reading exactly as it did on PR #988 — + /// which is what makes "the existing tests are unchanged and + /// byte-identical" a checkable statement rather than a hopeful one — while + /// still naming the hash in one visible place. + #[allow(clippy::too_many_arguments)] + fn prove( + trace: &CommittedTrace, + combine: C, + degree: usize, + config: &ChainConfig, + transcript: &mut T, + ) -> Result, Error> + where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + C: Fn(&[FieldElement]) -> FieldElement + Sync, + { + super::prove::(trace, combine, degree, config, transcript) + } + + #[allow(clippy::too_many_arguments)] + fn verify( + proof: &ConstraintProof, + claim_shape: TraceClaim<'_, F>, + combine: C, + public_values: P, + degree: usize, + config: &ChainConfig, + transcript: &mut T, + ) -> Result<(), Error> + where + F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, + E: IsField + Send + Sync + 'static, + FieldElement: AsBytes + Sync + Send, + FieldElement: AsBytes + Sync + Send, + T: IsTranscript, + C: Fn(&[FieldElement]) -> FieldElement + Sync, + P: FnOnce(&[FieldElement]) -> Result>, Error>, + { + super::verify::( + proof, + claim_shape, + combine, + public_values, + degree, + config, + transcript, + ) + } + fn transcript() -> DefaultTranscript { DefaultTranscript::::new(b"constraint-argument-test") } @@ -1604,7 +1671,7 @@ mod tests { .map(|_| verifier.sample_field_element()) .collect(); - verify_statements( + verify_statements::( &proof, TraceClaim { roots: &roots, diff --git a/crypto/multilinear/src/gpu.rs b/crypto/multilinear/src/gpu.rs index fafd252a1..b31d605e6 100644 --- a/crypto/multilinear/src/gpu.rs +++ b/crypto/multilinear/src/gpu.rs @@ -10,6 +10,17 @@ use core::sync::atomic::{AtomicU64, Ordering}; /// Successful device commits of a stacked polynomial. static COMMIT_CALLS: AtomicU64 = AtomicU64::new(0); +/// ★ Commits that asked the device and got nothing, and encoded on the host. +/// +/// The counter H4's arm needed and did not have. A device commit that declines +/// is INVISIBLE in every other number here: `COMMIT_CALLS` simply does not +/// rise, and a count that is merely lower than expected says nothing when the +/// expected count is itself derived. It matters because falling back is not a +/// slower way to do the same thing — `from_codeword` then retains a host +/// codeword and a host node array for the rest of the proof, so a card that +/// fills near the end of an epoch turns into gigabytes of host memory and a +/// utilisation figure that looks like a scheduling problem. +static HOST_FALLBACKS: AtomicU64 = AtomicU64::new(0); /// Sumchecks whose rounds ran on device. static SUMCHECK_CALLS: AtomicU64 = AtomicU64::new(0); /// Rounds within them, so a declined tail shows up. @@ -27,6 +38,16 @@ pub fn commit_calls() -> u64 { COMMIT_CALLS.load(Ordering::Relaxed) } +pub fn host_fallbacks() -> u64 { + HOST_FALLBACKS.load(Ordering::Relaxed) +} + +/// Called where a commit gives up on the device. Counts in non-cuda builds +/// too, where every commit takes that path and the number is the commit count. +pub(crate) fn note_host_fallback() { + HOST_FALLBACKS.fetch_add(1, Ordering::Relaxed); +} + pub fn sumcheck_calls() -> u64 { SUMCHECK_CALLS.load(Ordering::Relaxed) } @@ -53,6 +74,7 @@ pub fn open_calls() -> u64 { pub fn reset_call_counters() { COMMIT_CALLS.store(0, Ordering::Relaxed); + HOST_FALLBACKS.store(0, Ordering::Relaxed); SUMCHECK_CALLS.store(0, Ordering::Relaxed); SUMCHECK_ROUNDS.store(0, Ordering::Relaxed); EVALUATE_CALLS.store(0, Ordering::Relaxed); @@ -160,10 +182,27 @@ pub fn reserve_room(_bytes: u64) -> Option { None } +/// What a device sumcheck that runs to the end hands back: the round proofs, +/// and the challenges the rounds were bound at. +/// +/// A named type rather than the tuple written out, because the tuple appears in +/// a return position wrapped in a `Result` and reads as punctuation there. The +/// sibling [`ResidentRounds`] carries a third member — the tables left folded +/// at the crossover — for the path that stops early. +#[cfg(feature = "cuda")] +type ClosedRounds = ( + Vec>, + Vec>, +); + /// A promise held on someone else's behalf. Dropping it gives the room back. +/// +/// The field is never read, and that is the design: it is an RAII guard whose +/// `Drop` returns the reservation, so holding it IS the behaviour. Deleting it +/// to satisfy the lint would delete the promise. #[cfg(feature = "cuda")] #[derive(Debug)] -pub struct DeviceRoom(math_cuda::device::DeviceReservation); +pub struct DeviceRoom(#[allow(dead_code)] math_cuda::device::DeviceReservation); /// A promise no device made. Never constructed. #[cfg(not(feature = "cuda"))] @@ -479,13 +518,7 @@ fn run_rounds( mut reference: impl FnMut( &math_cuda::sumcheck::SumcheckSession, ) -> Option>>, -) -> Result< - ( - Vec>, - Vec>, - ), - crate::Error, -> +) -> Result, crate::Error> where E: math::field::traits::IsField + 'static, { @@ -764,6 +797,7 @@ where pub(crate) fn commit_tree_ext3( codeword: &[math::field::element::FieldElement], log_folding: usize, + hash: crate::whir_hash::DeviceHashKey, ) -> Option> where F: math::field::traits::IsField + 'static, @@ -783,7 +817,8 @@ where // SAFETY: `F == Ext3`, three transparent `u64` limbs per element. let raw = unsafe { core::slice::from_raw_parts(codeword.as_ptr() as *const u64, codeword.len() * 3) }; - let nodes = math_cuda::whir::commit_codeword_ext3(raw, log_folding).ok()?; + let nodes = + math_cuda::whir::commit_codeword_ext3(raw, log_folding, hash.into_math_cuda()).ok()?; let nodes = nodes_in_place(nodes)?; COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); Some(nodes) @@ -793,6 +828,7 @@ where pub(crate) fn commit_tree_ext3( _codeword: &[math::field::element::FieldElement], _log_folding: usize, + _hash: crate::whir_hash::DeviceHashKey, ) -> Option> where F: math::field::traits::IsField + 'static, @@ -851,8 +887,7 @@ where }) .collect(); let values = - math_cuda::sumcheck::evaluate_many_base(columns_at::(resident, &raw), &raw_point) - .ok()?; + math_cuda::sumcheck::evaluate_many_base(columns_at(resident, &raw), &raw_point).ok()?; EVALUATE_CALLS.fetch_add(values.len() as u64, Ordering::Relaxed); Some(values.iter().map(|v| ext3_from_raw::(v)).collect()) } @@ -932,141 +967,6 @@ where None } -#[cfg(test)] -mod tests { - use super::*; - use crate::program::Builder; - use math::field::element::FieldElement; - use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; - use math::field::goldilocks::GoldilocksField as Gl; - - type FE = FieldElement; - - /// The kernel's walk, in Rust: the same slot file, the same node encoding. - /// - /// This is what pins the lowering without a device — a slot freed too early - /// or an operand read from the wrong class shows up here as a wrong value, - /// not as a proof that does not verify an hour later. - fn run_lowered(lowered: &Lowered, values: &[FE]) -> FE { - let mut slots = vec![FE::zero(); lowered.num_slots]; - for node in lowered.nodes.chunks_exact(2) { - let op = (node[0] & 0xFFFF_FFFF) as u32; - let a = (node[0] >> 32) as u32 as usize; - let b = (node[1] & 0xFFFF_FFFF) as u32 as usize; - let res = (node[1] >> 32) as u32 as usize; - slots[res] = match op { - op::FIXED => ext3_from_raw::(&lowered.consts[a * 3..a * 3 + 3]), - op::VAR => values[a], - op::ADD => slots[a] + slots[b], - op::SUB => slots[a] - slots[b], - op::MUL => slots[a] * slots[b], - op::NEG => -slots[a], - _ => panic!("unknown op {op}"), - }; - } - slots[lowered.root_slot as usize] - } - - fn values(n: usize) -> Vec { - (0..n as u64) - .map(|i| { - FE::new([ - FieldElement::::from(i * 31 + 7), - FieldElement::::from(i * 17 + 2), - FieldElement::::from(i + 5), - ]) - }) - .collect() - } - - /// Every op, a constant, and a chain long enough that slots have to be - /// recycled. - fn sample_program() -> crate::program::Program { - let mut b = Builder::::new(); - let mut acc = b.var(0); - for slot in 1..6 { - let v = b.var(slot); - let doubled = b.add(v, v); - let scaled = b.mul(doubled, acc); - let shifted = b.sub(scaled, v); - acc = b.neg(shifted); - } - let seven = b.fixed(FE::from(7u64)); - let root = b.add(acc, seven); - b.finish(root).unwrap() - } - - #[test] - fn the_lowered_program_computes_what_the_program_does() { - let program = sample_program(); - let lowered = lower(&program).expect("lowers"); - let v = values(6); - let mut scratch = Vec::new(); - assert_eq!(run_lowered(&lowered, &v), program.eval(&v, &mut scratch)); - } - - /// A value read twice in the step that kills it — `x·x` — must not free its - /// slot twice, or two later values are handed the same one and the second - /// clobbers the first. Squarings are everywhere in a real constraint - /// program, so this is the shape that matters. - #[test] - fn a_value_read_twice_frees_its_slot_once() { - let mut b = Builder::::new(); - let mut acc = b.var(0); - // Each square kills its operand, and the sums below keep enough values - // live that a doubly-freed slot gets reused while it is still needed. - let mut squares = Vec::new(); - for slot in 1..8 { - let v = b.var(slot); - let squared = b.mul(v, v); - let with_acc = b.add(squared, acc); - squares.push(with_acc); - acc = b.mul(with_acc, with_acc); - } - squares.push(acc); - let root = b.sum(&squares); - let program = b.finish(root).unwrap(); - - let lowered = lower(&program).expect("lowers"); - let v = values(8); - let mut scratch = Vec::new(); - assert_eq!(run_lowered(&lowered, &v), program.eval(&v, &mut scratch)); - } - - /// The point of the slot file: a long chain of dead intermediates does not - /// widen it. - #[test] - fn slots_are_reused_once_a_value_is_dead() { - let lowered = lower(&sample_program()).expect("lowers"); - assert!( - lowered.num_slots < lowered.nodes.len() / 2, - "{} slots for {} steps is no reuse at all", - lowered.num_slots, - lowered.nodes.len() / 2 - ); - } - - /// A program wider than the slot file declines rather than asking a device - /// for scratch it cannot have. - #[test] - fn a_program_past_the_slot_ceiling_declines() { - let mut b = Builder::::new(); - // Every value stays live to the end, so the slots cannot be recycled. - let terms: Vec = (0..=MAX_SLOTS).map(|slot| b.var(slot)).collect(); - let root = b.sum(&terms); - let program = b.finish(root).unwrap(); - assert!(lower(&program).is_none()); - } - - #[test] - fn a_field_the_kernel_does_not_cover_declines() { - let mut b = Builder::::new(); - let root = b.var(0); - let program = b.finish(root).unwrap(); - assert!(lower(&program).is_none()); - } -} - /// Input-layer size below which the host tree wins: the levels are a launch /// each and the fold is a pass a few cores finish in microseconds. #[cfg(feature = "cuda")] @@ -1416,13 +1316,10 @@ where /// Where a run of columns is, for the entry points that take either. #[cfg(feature = "cuda")] -fn columns_at<'a, F>( +fn columns_at<'a>( resident: Option<(&'a ResidentColumns, usize)>, host: &'a [&'a [u64]], -) -> math_cuda::columns::Columns<'a> -where - F: math::field::traits::IsField + 'static, -{ +) -> math_cuda::columns::Columns<'a> { match resident { Some((store, first)) if store.0.is_run(first, host.len()) => { math_cuda::columns::Columns::Device { @@ -1536,7 +1433,7 @@ where .collect(); let uploaded = math_cuda::sumcheck::DeviceFactors::from_columns( - columns_at::(resident, &raw_columns), + columns_at(resident, &raw_columns), &plan, &raw_public, rows, @@ -2035,6 +1932,7 @@ pub(crate) fn commit_parts( log_blowup: usize, log_folding: usize, transient: bool, + hash: crate::whir_hash::DeviceHashKey, ) -> Option<(DeviceCodeword, [u8; 32])> where F: math::field::traits::IsField + 'static, @@ -2068,9 +1966,15 @@ where ) }) .collect(); - let (codeword, root) = - math_cuda::whir::commit_codeword_parts(&raw, log_evals, log_blowup, log_folding, transient) - .ok()?; + let (codeword, root) = math_cuda::whir::commit_codeword_parts( + &raw, + log_evals, + log_blowup, + log_folding, + transient, + hash.into_math_cuda(), + ) + .ok()?; COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); Some((DeviceCodeword(codeword), root)) } @@ -2084,6 +1988,7 @@ pub(crate) fn commit_resident( log_blowup: usize, log_folding: usize, transient: bool, + hash: crate::whir_hash::DeviceHashKey, ) -> Option<(DeviceCodeword, [u8; 32])> { if (1usize << log_evals) << log_blowup < COMMIT_THRESHOLD { return None; @@ -2099,6 +2004,7 @@ pub(crate) fn commit_resident( log_blowup, log_folding, transient, + hash.into_math_cuda(), ) .ok()?; COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); @@ -2113,6 +2019,7 @@ pub(crate) fn commit_resident( _log_blowup: usize, _log_folding: usize, _transient: bool, + _hash: crate::whir_hash::DeviceHashKey, ) -> Option<(DeviceCodeword, [u8; 32])> { None } @@ -2124,6 +2031,7 @@ pub(crate) fn commit_parts( _log_blowup: usize, _log_folding: usize, _transient: bool, + _hash: crate::whir_hash::DeviceHashKey, ) -> Option<(DeviceCodeword, [u8; 32])> where F: math::field::traits::IsField + 'static, @@ -2157,8 +2065,12 @@ impl DeviceCodeword { /// The tree is not kept: the only other thing a proof wants from it is a /// path per query, and [`paths`](Self::paths) rebuilds it then, when the /// queries are known — see the note there. - pub(crate) fn commit(&self, log_folding: usize) -> Option<[u8; 32]> { - let root = self.0.commit(log_folding).ok()?; + pub(crate) fn commit( + &self, + log_folding: usize, + hash: crate::whir_hash::DeviceHashKey, + ) -> Option<[u8; 32]> { + let root = self.0.commit(log_folding, hash.into_math_cuda()).ok()?; COMMIT_CALLS.fetch_add(1, Ordering::Relaxed); Some(root) } @@ -2168,13 +2080,17 @@ impl DeviceCodeword { &self, log_folding: usize, indices: &[usize], + hash: crate::whir_hash::DeviceHashKey, ) -> Option>> { let leaves = self.0.elements() >> log_folding; if indices.iter().any(|index| *index >= leaves) { return None; } let positions: Vec = indices.iter().map(|index| *index as u32).collect(); - let bytes = self.0.paths(log_folding, &positions).ok()?; + let bytes = self + .0 + .paths(log_folding, &positions, hash.into_math_cuda()) + .ok()?; let depth = leaves.trailing_zeros() as usize; let nodes = nodes_in_place(bytes)?; Some(nodes.chunks_exact(depth).map(<[_]>::to_vec).collect()) @@ -2297,7 +2213,11 @@ impl DeviceCodeword { match self.0 {} } - pub(crate) fn commit(&self, _log_folding: usize) -> Option<[u8; 32]> { + pub(crate) fn commit( + &self, + _log_folding: usize, + _hash: crate::whir_hash::DeviceHashKey, + ) -> Option<[u8; 32]> { match self.0 {} } @@ -2305,6 +2225,7 @@ impl DeviceCodeword { &self, _log_folding: usize, _indices: &[usize], + _hash: crate::whir_hash::DeviceHashKey, ) -> Option>> { match self.0 {} } @@ -2328,3 +2249,138 @@ impl DeviceCodeword { match self.0 {} } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::program::Builder; + use math::field::element::FieldElement; + use math::field::extensions_goldilocks::Degree3GoldilocksExtensionField as Ext3; + use math::field::goldilocks::GoldilocksField as Gl; + + type FE = FieldElement; + + /// The kernel's walk, in Rust: the same slot file, the same node encoding. + /// + /// This is what pins the lowering without a device — a slot freed too early + /// or an operand read from the wrong class shows up here as a wrong value, + /// not as a proof that does not verify an hour later. + fn run_lowered(lowered: &Lowered, values: &[FE]) -> FE { + let mut slots = vec![FE::zero(); lowered.num_slots]; + for node in lowered.nodes.chunks_exact(2) { + let op = (node[0] & 0xFFFF_FFFF) as u32; + let a = (node[0] >> 32) as u32 as usize; + let b = (node[1] & 0xFFFF_FFFF) as u32 as usize; + let res = (node[1] >> 32) as u32 as usize; + slots[res] = match op { + op::FIXED => ext3_from_raw::(&lowered.consts[a * 3..a * 3 + 3]), + op::VAR => values[a], + op::ADD => slots[a] + slots[b], + op::SUB => slots[a] - slots[b], + op::MUL => slots[a] * slots[b], + op::NEG => -slots[a], + _ => panic!("unknown op {op}"), + }; + } + slots[lowered.root_slot as usize] + } + + fn values(n: usize) -> Vec { + (0..n as u64) + .map(|i| { + FE::new([ + FieldElement::::from(i * 31 + 7), + FieldElement::::from(i * 17 + 2), + FieldElement::::from(i + 5), + ]) + }) + .collect() + } + + /// Every op, a constant, and a chain long enough that slots have to be + /// recycled. + fn sample_program() -> crate::program::Program { + let mut b = Builder::::new(); + let mut acc = b.var(0); + for slot in 1..6 { + let v = b.var(slot); + let doubled = b.add(v, v); + let scaled = b.mul(doubled, acc); + let shifted = b.sub(scaled, v); + acc = b.neg(shifted); + } + let seven = b.fixed(FE::from(7u64)); + let root = b.add(acc, seven); + b.finish(root).unwrap() + } + + #[test] + fn the_lowered_program_computes_what_the_program_does() { + let program = sample_program(); + let lowered = lower(&program).expect("lowers"); + let v = values(6); + let mut scratch = Vec::new(); + assert_eq!(run_lowered(&lowered, &v), program.eval(&v, &mut scratch)); + } + + /// A value read twice in the step that kills it — `x·x` — must not free its + /// slot twice, or two later values are handed the same one and the second + /// clobbers the first. Squarings are everywhere in a real constraint + /// program, so this is the shape that matters. + #[test] + fn a_value_read_twice_frees_its_slot_once() { + let mut b = Builder::::new(); + let mut acc = b.var(0); + // Each square kills its operand, and the sums below keep enough values + // live that a doubly-freed slot gets reused while it is still needed. + let mut squares = Vec::new(); + for slot in 1..8 { + let v = b.var(slot); + let squared = b.mul(v, v); + let with_acc = b.add(squared, acc); + squares.push(with_acc); + acc = b.mul(with_acc, with_acc); + } + squares.push(acc); + let root = b.sum(&squares); + let program = b.finish(root).unwrap(); + + let lowered = lower(&program).expect("lowers"); + let v = values(8); + let mut scratch = Vec::new(); + assert_eq!(run_lowered(&lowered, &v), program.eval(&v, &mut scratch)); + } + + /// The point of the slot file: a long chain of dead intermediates does not + /// widen it. + #[test] + fn slots_are_reused_once_a_value_is_dead() { + let lowered = lower(&sample_program()).expect("lowers"); + assert!( + lowered.num_slots < lowered.nodes.len() / 2, + "{} slots for {} steps is no reuse at all", + lowered.num_slots, + lowered.nodes.len() / 2 + ); + } + + /// A program wider than the slot file declines rather than asking a device + /// for scratch it cannot have. + #[test] + fn a_program_past_the_slot_ceiling_declines() { + let mut b = Builder::::new(); + // Every value stays live to the end, so the slots cannot be recycled. + let terms: Vec = (0..=MAX_SLOTS).map(|slot| b.var(slot)).collect(); + let root = b.sum(&terms); + let program = b.finish(root).unwrap(); + assert!(lower(&program).is_none()); + } + + #[test] + fn a_field_the_kernel_does_not_cover_declines() { + let mut b = Builder::::new(); + let root = b.var(0); + let program = b.finish(root).unwrap(); + assert!(lower(&program).is_none()); + } +} diff --git a/crypto/multilinear/src/lib.rs b/crypto/multilinear/src/lib.rs index 706f1039a..a269ae4a6 100644 --- a/crypto/multilinear/src/lib.rs +++ b/crypto/multilinear/src/lib.rs @@ -15,6 +15,7 @@ pub mod logup; pub mod mle; pub mod poly; pub mod program; +pub mod query_count; pub mod selector; pub mod stacked_eval; pub mod stacking; @@ -26,6 +27,7 @@ pub mod whir; pub mod whir_chain; pub mod whir_commit; pub mod whir_eval; +pub mod whir_hash; pub mod whir_round; pub mod zerocheck; diff --git a/crypto/multilinear/src/query_count.rs b/crypto/multilinear/src/query_count.rs new file mode 100644 index 000000000..c7da37282 --- /dev/null +++ b/crypto/multilinear/src/query_count.rs @@ -0,0 +1,316 @@ +//! ★ The query count, derived in INTEGERS. +//! +//! `ChainConfig::with_security` computed `num_queries` with `f64` `sqrt`, +//! `log2` and `ceil`. That is fine on a host and a problem everywhere else this +//! protocol is going: a field-native verifier has no floating point, and a +//! prover and verifier that disagree by one query do not fail gracefully — the +//! transcript diverges and every later challenge is different. +//! +//! So the derivation lives here, in fixed-point integer arithmetic, and the +//! `f64` version survives only as the reference the tests enumerate against +//! over the whole parameter grid. +//! +//! # The formula, unchanged +//! +//! ```text +//! rate = 2^-log_blowup +//! proximity = 1 - sqrt(rate) - 1/300 (the Johnson bound) +//! bits_per_query = -log2(1 - proximity) +//! = -log2(sqrt(rate) + 1/300) +//! target = security_bits + log2(rounds) (a union bound over rounds) +//! num_queries = max(ceil(max(target - grind, 0) / bits_per_query), 1) +//! ``` +//! +//! ⚠ **This is not a soundness analysis and this module does not make it one.** +//! It is the conservative mirror of the parameters the univariate prover ships, +//! as `with_security`'s own doc says. Moving it to integers changes who can +//! evaluate it, not what it claims. +//! +//! # Precision, and why the answers are the same +//! +//! Everything is Q62 — 62 fractional bits in a `u128` — against `f64`'s 53 bits +//! of mantissa, so this form is strictly more precise than the one it replaces. +//! Where they could still differ is a grid point whose true ratio sits within a +//! rounding step of an integer, and `ceil` then goes either way. That is not +//! argued here, it is ENUMERATED: the grid test walks every point the protocol +//! can reach and fails on any disagreement. +//! +//! Q62 rather than Q64 for one concrete reason: the log2 loop squares its +//! running value, and a Q64 mantissa in `[1, 2)` squares to 130 bits, which a +//! `u128` does not hold. At Q62 the square fits with two bits to spare. + +/// Fractional bits in the fixed-point representation. +const FRAC: u32 = 62; +/// The value `1.0`. +const ONE: u128 = 1 << FRAC; + +/// Integer square root of a `u128`, by bit-by-bit restoring subtraction. +/// +/// Exact: returns `floor(sqrt(n))`. Written out rather than reached for through +/// a float, which is the thing this module exists to avoid. +fn isqrt(n: u128) -> u128 { + if n == 0 { + return 0; + } + // The largest power of four not exceeding `n`. + let mut bit: u128 = 1u128 << ((127 - n.leading_zeros()) & !1u32); + let mut rem = n; + let mut root: u128 = 0; + while bit != 0 { + if rem >= root + bit { + rem -= root + bit; + root = (root >> 1) + bit; + } else { + root >>= 1; + } + bit >>= 2; + } + root +} + +/// `log2(x / 2^FRAC)` in Q62, for `x > 0`. +/// +/// The integer part comes from the leading bit; the fraction from the classic +/// squaring loop — square the mantissa, and a result at or above two both emits +/// a one bit and halves the value. +fn log2_fixed(x: u128) -> i128 { + debug_assert!(x > 0, "log2 of zero is not a number this protocol uses"); + + // Normalise the mantissa into `[1, 2)`, i.e. `[2^FRAC, 2^(FRAC+1))`. + let bits = 128 - x.leading_zeros(); // position of the leading one, 1-based + let exponent = bits as i128 - 1 - FRAC as i128; + let mut mantissa = if exponent >= 0 { + x >> (exponent as u32) + } else { + x << ((-exponent) as u32) + }; + debug_assert!((ONE..ONE << 1).contains(&mantissa)); + + let mut fraction: u128 = 0; + let mut weight = ONE >> 1; + for _ in 0..FRAC { + // `mantissa` is in `[1, 2)`, so the square is in `[1, 4)` and fits. + mantissa = (mantissa * mantissa) >> FRAC; + if mantissa >= ONE << 1 { + mantissa >>= 1; + fraction |= weight; + } + weight >>= 1; + } + + (exponent << FRAC) + fraction as i128 +} + +/// `sqrt(2^-log_blowup)` in Q62. +/// +/// `sqrt(2^-b) * 2^62 = sqrt(2^(124 - b))`, so one integer square root does it +/// — exactly when `124 - b` is even, floored otherwise. +fn sqrt_rate(log_blowup: usize) -> u128 { + assert!( + log_blowup < 124, + "a rate of 2^-{log_blowup} is not a code this protocol can use" + ); + isqrt(1u128 << (124 - log_blowup as u32)) +} + +/// ★ Queries needed for `security_bits` under the Johnson bound, given the +/// round count and the proof of work spent on the query challenge. +/// +/// The integer twin of what `ChainConfig::with_security` used to compute in +/// `f64`, and the one the configuration now uses. +pub fn num_queries(log_blowup: usize, rounds: usize, security_bits: u8, grind_query: u8) -> usize { + // 1 - proximity = sqrt(rate) + 1/300. + let one_over_300 = ONE / 300; + let w = sqrt_rate(log_blowup) + one_over_300; + + // A rate whose `w` reached 1 would buy nothing per query. `log_blowup >= 1` + // keeps `w <= 0.708`. + assert!(w < ONE, "each query must buy a positive number of bits"); + let bits_per_query = -log2_fixed(w); + debug_assert!(bits_per_query > 0); + + let rounds = rounds.max(1); + let target = ((security_bits as i128) << FRAC) + log2_fixed((rounds as u128) << FRAC); + let left = (target - ((grind_query as i128) << FRAC)).max(0); + + // Both sides are Q62, so the ratio is a plain integer one. Written out + // rather than `div_ceil`, which is unstable for `i128`; both operands are + // known non-negative here (`left` is clamped, `bits_per_query` asserted + // positive), so the rounding has no sign case to get wrong. + let queries = (left + bits_per_query - 1) / bits_per_query; + (queries as usize).max(1) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The `f64` derivation this replaces, verbatim from `with_security` as it + /// stood — the reference, kept only so the integer form can be checked + /// against it. + fn f64_reference( + log_blowup: usize, + rounds: usize, + security_bits: u8, + grind_query: u8, + ) -> usize { + let rounds = rounds.max(1); + let rate = 1.0 / (1u64 << log_blowup) as f64; + let proximity = 1.0 - rate.sqrt() - 1.0 / 300.0; + let bits_per_query = -(1.0 - proximity).log2(); + + let target = security_bits as f64 + (rounds as f64).log2(); + let left = (target - grind_query as f64).max(0.0); + (left / bits_per_query).ceil().max(1.0) as usize + } + + /// ★★ The whole realistic grid, enumerated. Roughly 4.3 million points. + /// + /// Not a spot check and not an argument: the one way the two forms can + /// differ is a ratio landing within a rounding step of an integer, and the + /// only honest way to know whether that happens anywhere the protocol can + /// reach is to look at every point it can reach. + #[test] + fn the_integer_derivation_agrees_with_the_f64_reference() { + let mut checked = 0u64; + for log_blowup in 1..=4usize { + for rounds in 1..=64usize { + for security_bits in 0..=255u8 { + for grind_query in 0..=64u8 { + let got = num_queries(log_blowup, rounds, security_bits, grind_query); + let want = f64_reference(log_blowup, rounds, security_bits, grind_query); + assert_eq!( + got, want, + "blowup 2^{log_blowup}, {rounds} rounds, {security_bits} bits, \ + grind {grind_query}" + ); + checked += 1; + } + } + } + } + assert_eq!(checked, 4 * 64 * 256 * 65, "the grid must be fully walked"); + } + + /// ✓ The grid above is worth walking only if the answers vary across it. + #[test] + fn the_grid_is_not_one_answer_everywhere() { + let mut seen = std::collections::BTreeSet::new(); + for log_blowup in 1..=4usize { + for security_bits in [0u8, 64, 128, 255] { + for grind_query in [0u8, 20, 64] { + seen.insert(num_queries(log_blowup, 6, security_bits, grind_query)); + } + } + } + assert!( + seen.len() > 10, + "only {} distinct counts across the sample: the grid is degenerate", + seen.len() + ); + } + + /// ★ The shipped posture, pinned to its literal: blowup 4, 128 bits, 20 + /// bits of query grinding, one round — the same 110 the univariate + /// prover's own accounting gives. + #[test] + fn the_shipped_posture_is_110_at_one_round() { + assert_eq!(num_queries(2, 1, 128, 20), 110); + } + + /// ★ And 112 / 113 at the round counts a real proof reaches. + #[test] + fn the_shipped_posture_is_112_then_113_as_the_rounds_grow() { + for rounds in 4..=7 { + assert_eq!(num_queries(2, rounds, 128, 20), 112, "{rounds} rounds"); + } + for rounds in 8..=15 { + assert_eq!(num_queries(2, rounds, 128, 20), 113, "{rounds} rounds"); + } + } + + /// More grinding buys fewer queries, and a wider blowup buys more per + /// query — the two monotonicities the formula is supposed to have, checked + /// rather than assumed. + #[test] + fn the_count_moves_the_way_the_parameters_say_it_should() { + let at = |b, g| num_queries(b, 6, 128, g); + assert!( + at(2, 30) < at(2, 20), + "grinding must reduce the query count" + ); + assert!( + at(3, 20) < at(2, 20), + "a wider blowup must buy more per query" + ); + assert!(at(4, 20) < at(3, 20)); + for g in 0..64u8 { + assert!( + at(2, g + 1) <= at(2, g), + "the count must not rise with grinding at {g}" + ); + } + } + + /// The floor: a configuration that has already ground past its target still + /// checks one position. + #[test] + fn at_least_one_query_is_always_drawn() { + assert_eq!(num_queries(2, 1, 0, 64), 1); + assert_eq!(num_queries(2, 1, 10, 64), 1); + } + + /// `isqrt` against the definition: `r^2 <= n < (r+1)^2`. + #[test] + fn the_integer_square_root_is_the_floor_of_the_real_one() { + for n in [0u128, 1, 2, 3, 4, 5, 99, 100, 101, 1 << 40, (1 << 62) + 7] { + let r = isqrt(n); + assert!(r * r <= n, "isqrt({n}) = {r} is too large"); + assert!((r + 1).checked_mul(r + 1).is_none_or(|s| s > n)); + } + // The shapes `sqrt_rate` actually asks for. + for b in 1..=4u32 { + let n = 1u128 << (124 - b); + let r = isqrt(n); + assert!(r * r <= n && (r + 1) * (r + 1) > n, "blowup 2^{b}"); + } + } + + /// `log2_fixed` against `f64::log2` — a different algorithm for the same + /// number, to within the precision the fixed point carries. + #[test] + fn the_fixed_point_log2_agrees_with_the_floating_one() { + for v in [ + 1.0f64, + 1.5, + 2.0, + 3.0, + 7.0, + 64.0, + 0.5, + 0.25, + 0.1, + 0.708, + 1.0 / 300.0, + ] { + let x = (v * (ONE as f64)) as u128; + let got = log2_fixed(x) as f64 / ONE as f64; + let want = (x as f64 / ONE as f64).log2(); + assert!((got - want).abs() < 1e-15, "log2({v}): {got} vs {want}"); + } + } + + /// ✓ Exact on the powers of two, where the answer is an integer and any + /// drift in the squaring loop would show as a fraction. + #[test] + fn the_fixed_point_log2_is_exact_on_powers_of_two() { + for k in -30i32..=30 { + let x = if k >= 0 { ONE << k } else { ONE >> (-k) }; + assert_eq!( + log2_fixed(x), + (k as i128) << FRAC, + "log2(2^{k}) must be exactly {k}" + ); + } + } +} diff --git a/crypto/multilinear/src/stacked_eval.rs b/crypto/multilinear/src/stacked_eval.rs index 45d62db14..8ba8ddc7d 100644 --- a/crypto/multilinear/src/stacked_eval.rs +++ b/crypto/multilinear/src/stacked_eval.rs @@ -44,15 +44,16 @@ use crate::{ whir::Domain, whir_chain::{self, ChainConfig, ChainProof}, whir_commit::{CodewordCommitment, Commitment}, + whir_hash::WhirHash, }; /// The stacked polynomials, committed. Base-field, like the trace they hold. -pub struct StackedCommitment +pub struct StackedCommitment where FieldElement: AsBytes + Sync + Send, { layout: StackedLayout, - commitments: Vec>, + commitments: Vec>, domain: Domain, /// The room the commits and the openings take turns with, promised once /// for the whole group. Lives as long as the commitments do, because the @@ -60,7 +61,7 @@ where _room: Option, } -impl StackedCommitment +impl StackedCommitment where FieldElement: AsBytes + Sync + Send, { @@ -118,7 +119,7 @@ where // tree on the device while it runs. Every polynomial has the same // variable count, so they share a domain. let commit = |poly: &whir_chain::Stacked<'_, F>| { - whir_chain::commit_stacked::(poly, config, transient) + whir_chain::commit_stacked::(poly, config, transient) }; let mut domain = None; let mut commitments = Vec::with_capacity(sources.len()); @@ -333,8 +334,8 @@ fn claimed( /// /// The claims are absorbed before the batching challenge, so the prover cannot /// pick them after seeing it. -pub fn prove( - stacked: &StackedCommitment, +pub fn prove( + stacked: &StackedCommitment, columns: &[&Mle], resident: Option<(&crate::gpu::ResidentColumns, usize)>, point: &Claimed<'_, E>, @@ -348,6 +349,7 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let layout = &stacked.layout; if values.len() != layout.placements().len() { @@ -385,7 +387,7 @@ where }; // The weight goes down as its shares: a device writes them into its own // buffer, and the host materializes the table only if none does. - polys.push(whir_chain::prove_shared::( + polys.push(whir_chain::prove_shared::( &poly, &weight_shares(layout, i, point, &weights)?, layout.n_stack(), @@ -404,7 +406,7 @@ where /// The layout is public and derived from the column heights, so it is not part /// of the proof. #[allow(clippy::too_many_arguments)] -pub fn verify( +pub fn verify( proof: &StackedProof, layout: &StackedLayout, roots: &[Commitment], @@ -420,6 +422,7 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { if values.len() != layout.placements().len() { return Err(Error::QueryCountMismatch { @@ -439,7 +442,7 @@ where let weights = challenge_powers(&transcript.sample_field_element(), values.len()); for (i, (eval_proof, root)) in proof.polys.iter().zip(roots).enumerate() { - whir_chain::verify_weighted::( + whir_chain::verify_weighted::( eval_proof, root, |at: &[FieldElement]| weight_at(layout, i, point, &weights, at), @@ -461,7 +464,7 @@ mod tests { use crypto::fiat_shamir::default_transcript::DefaultTranscript; use math::field::goldilocks::GoldilocksField as F; - use crate::whir_chain::GrindBits; + use crate::{whir_chain::GrindBits, whir_hash::KeccakWhir}; type FE = FieldElement; @@ -510,7 +513,7 @@ mod tests { at: &[FE], claimed: &[FE], ) -> Result { - let stacked = StackedCommitment::::commit( + let stacked = StackedCommitment::::commit( layout, &crate::stacking::borrow(columns), None, @@ -527,7 +530,7 @@ mod tests { &mut transcript(), )?; - verify( + verify::( &proof, stacked.layout(), &roots, @@ -673,7 +676,7 @@ mod tests { let at = point(num_vars); let claimed = values(&columns, &at); - let stacked = StackedCommitment::::commit( + let stacked = StackedCommitment::::commit( layout, &crate::stacking::borrow(&columns), None, @@ -705,7 +708,7 @@ mod tests { let at = point(num_vars); let claimed = values(&columns, &at); - let stacked = StackedCommitment::::commit( + let stacked = StackedCommitment::::commit( layout, &crate::stacking::borrow(&columns), None, @@ -726,7 +729,7 @@ mod tests { let mut other = DefaultTranscript::::new(b"a-different-statement"); assert!( - verify( + verify::( &proof, stacked.layout(), &roots, @@ -767,7 +770,7 @@ mod tests { .map(|c| c.evaluate_in(&at).unwrap()) .collect(); - let stacked = StackedCommitment::::commit( + let stacked = StackedCommitment::::commit( layout, &crate::stacking::borrow(&columns), None, @@ -778,7 +781,7 @@ mod tests { assert_eq!(roots.len(), 1); let mut prover = DefaultTranscript::::new(b"tower"); - let proof = prove::( + let proof = prove::( &stacked, &crate::stacking::borrow(&columns), None, @@ -790,7 +793,7 @@ mod tests { .unwrap(); let mut verifier = DefaultTranscript::::new(b"tower"); - verify::( + verify::( &proof, stacked.layout(), &roots, @@ -828,7 +831,7 @@ mod tests { .map(|(c, p)| c.evaluate(p).unwrap()) .collect(); - let stacked = StackedCommitment::::commit( + let stacked = StackedCommitment::::commit( layout, &crate::stacking::borrow(&columns), None, @@ -848,7 +851,7 @@ mod tests { &mut transcript(), ) .unwrap(); - verify( + verify::( &proof, stacked.layout(), &roots, @@ -875,7 +878,7 @@ mod tests { ) .unwrap(); assert!( - verify( + verify::( &proof, stacked.layout(), &roots, diff --git a/crypto/multilinear/src/whir_chain.rs b/crypto/multilinear/src/whir_chain.rs index 0ff9390bb..37e90a718 100644 --- a/crypto/multilinear/src/whir_chain.rs +++ b/crypto/multilinear/src/whir_chain.rs @@ -56,6 +56,7 @@ use crate::{ sumcheck::{self, RoundProof as SumcheckRoundProof}, whir::{Domain, encode, fold_codeword_k, lift_coefficients}, whir_commit::{Codeword, CodewordCommitment, Commitment, fold_coset, verify_opening}, + whir_hash::{GrindingDigest, WhirHash}, whir_round::{self, RoundCommitments, RoundConfig, RoundProof}, }; @@ -103,31 +104,34 @@ where /// /// Retrying that challenge then costs `2^bits` hashes. Zero bits is a no-op, so /// a caller that has not chosen its parameters yet pays nothing. -fn grind(transcript: &mut T, bits: u8) -> Result +fn grind(transcript: &mut T, bits: u8) -> Result where E: IsField + Send + Sync + 'static, T: IsTranscript, + H: WhirHash, { if bits == 0 { return Ok(0); } - let nonce = crypto::grinding::generate_nonce_maybe_gpu(&transcript.state(), bits) - .ok_or(Error::GrindingFailed { bits })?; + let nonce = + crypto::grinding::generate_nonce_maybe_gpu::>(&transcript.state(), bits) + .ok_or(Error::GrindingFailed { bits })?; transcript.append_bytes(&nonce.to_be_bytes()); Ok(nonce) } /// The verifier's half: the nonce must pass against the same state, and it is /// absorbed the same way. -fn check_grind(transcript: &mut T, bits: u8, nonce: u64) -> Result<(), Error> +fn check_grind(transcript: &mut T, bits: u8, nonce: u64) -> Result<(), Error> where E: IsField + Send + Sync + 'static, T: IsTranscript, + H: WhirHash, { if bits == 0 { return Ok(()); } - if !crypto::grinding::is_valid_nonce(&transcript.state(), nonce, bits) { + if !crypto::grinding::is_valid_nonce::>(&transcript.state(), nonce, bits) { return Err(Error::GrindingRejected { bits }); } transcript.append_bytes(&nonce.to_be_bytes()); @@ -207,13 +211,13 @@ impl ChainConfig { grind: GrindBits, ) -> Self { let rounds = num_vars.div_ceil(log_folding.max(1)).max(1); - let rate = 1.0 / (1u64 << log_blowup) as f64; - let proximity = 1.0 - rate.sqrt() - 1.0 / 300.0; - let bits_per_query = -(1.0 - proximity).log2(); - - let target = security_bits as f64 + (rounds as f64).log2(); - let left = (target - grind.query as f64).max(0.0); - let num_queries = (left / bits_per_query).ceil().max(1.0) as usize; + // ★ Integers, not `f64`. The arithmetic and its provenance are in + // [`crate::query_count`]; what matters here is that the count a + // verifier has to reproduce no longer needs floating point to + // reproduce it, and that the answers did not move — the shipped + // posture's 110 / 112 / 113 are pinned in both places. + let num_queries = + crate::query_count::num_queries(log_blowup, rounds, security_bits, grind.query); Self { log_blowup, @@ -383,13 +387,14 @@ impl Stacked<'_, F> { } /// A whole polynomial is a stacked one of a single part at offset zero. -pub fn commit( +pub fn commit( f: &Mle, config: &ChainConfig, transient: bool, -) -> Result<(CodewordCommitment, Domain), Error> +) -> Result<(CodewordCommitment, Domain), Error> where F: IsFFTField + IsPrimeField + Send + Sync + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, { commit_stacked( @@ -403,13 +408,14 @@ where ) } -pub fn commit_stacked( +pub fn commit_stacked( f: &Stacked<'_, F>, config: &ChainConfig, transient: bool, -) -> Result<(CodewordCommitment, Domain), Error> +) -> Result<(CodewordCommitment, Domain), Error> where F: IsFFTField + IsPrimeField + Send + Sync + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, { let num_vars = f.num_vars(); @@ -419,17 +425,40 @@ where // On a device the codeword stays there: the chain folds it and opens a // handful of its values, and it is the biggest array the proof holds. let attempt = match &f.resident { - Some((store, parts)) => { - crate::gpu::commit_resident(store, parts, num_vars, config.log_blowup, first, transient) - } - None => crate::gpu::commit_parts(&f.parts, num_vars, config.log_blowup, first, transient), + Some((store, parts)) => crate::gpu::commit_resident( + store, + parts, + num_vars, + config.log_blowup, + first, + transient, + H::DEVICE, + ), + None => crate::gpu::commit_parts( + &f.parts, + num_vars, + config.log_blowup, + first, + transient, + H::DEVICE, + ), }; let commitment = match attempt { Some((codeword, nodes)) => CodewordCommitment::from_device(codeword, nodes, first)?, - None => CodewordCommitment::from_codeword( - encode::(&lift_coefficients(&f.assemble()?), &domain)?, - first, - )?, + // ⚠ COUNTED, because this arm is otherwise silent. The device declining + // is not a slower path to the same place: the codeword is assembled, + // lifted and encoded here, and the commitment then holds that codeword + // AND its node array on the host until the proof ends. An epoch that + // fills the card partway through lands here for every commit after, + // and the only visible symptoms are host memory and a utilisation + // figure — neither of which names the cause. + None => { + crate::gpu::note_host_fallback(); + CodewordCommitment::from_codeword( + encode::(&lift_coefficients(&f.assemble()?), &domain)?, + first, + )? + } }; Ok((commitment, domain)) } @@ -569,20 +598,20 @@ where /// /// Only the first round's is base-field. Folding it with an extension challenge /// is what lifts it, so every later round is `Extension`. -enum Current<'a, F: IsField, E: IsField> +enum Current<'a, F: IsField + 'static, E: IsField + 'static, H: WhirHash> where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, { - Base(&'a CodewordCommitment), - Extension(CodewordCommitment), + Base(&'a CodewordCommitment), + Extension(CodewordCommitment), } /// Proves `f(z) = y`. -pub fn prove( +pub fn prove( f: &Mle, z: &[FieldElement], - commitment: &CodewordCommitment, + commitment: &CodewordCommitment, domain: &Domain, config: &ChainConfig, transcript: &mut T, @@ -593,19 +622,20 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { - prove_weighted::(f, eq_mle(z)?, commitment, domain, config, transcript) + prove_weighted::(f, eq_mle(z)?, commitment, domain, config, transcript) } /// The same for a weight given as the shares of a stacked polynomial's /// columns, which a device writes into its own buffer and the host /// materializes only if none does. #[allow(clippy::too_many_arguments)] -pub fn prove_shared( +pub fn prove_shared( f: &Stacked<'_, F>, shares: &[crate::stacked_eval::WeightShare<'_, E>], n_stack: usize, - commitment: &CodewordCommitment, + commitment: &CodewordCommitment, domain: &Domain, config: &ChainConfig, transcript: &mut T, @@ -616,9 +646,10 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let factors = Factors::::from_shares(f, shares, n_stack)?; - prove_with_factors::( + prove_with_factors::( f.num_vars(), factors, commitment, @@ -629,10 +660,10 @@ where } /// Proves `Σ_x w(x)·f(x) = y` for a weight the verifier can evaluate itself. -pub fn prove_weighted( +pub fn prove_weighted( f: &Mle, weight: Mle, - commitment: &CodewordCommitment, + commitment: &CodewordCommitment, domain: &Domain, config: &ChainConfig, transcript: &mut T, @@ -643,9 +674,10 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let factors = Factors::::new(f, weight)?; - prove_with_factors::( + prove_with_factors::( f.num_vars(), factors, commitment, @@ -656,10 +688,10 @@ where } /// The chain itself, over factors that are wherever they are. -fn prove_with_factors( +fn prove_with_factors( num_vars: usize, mut factors: Factors, - commitment: &CodewordCommitment, + commitment: &CodewordCommitment, domain: &Domain, config: &ChainConfig, transcript: &mut T, @@ -670,11 +702,12 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let schedule = config.schedule(num_vars); // The codeword comes out of the commitment rather than being encoded // again: it is the same array, and the NTT is not cheap. - let mut current = Current::::Base(commitment); + let mut current = Current::::Base(commitment); let mut current_domain = domain.clone(); let mut rounds = Vec::with_capacity(schedule.len()); @@ -682,7 +715,7 @@ where for (r, &k) in schedule.iter().enumerate() { let mut nonces = RoundNonces { - folding: grind(transcript, config.grind.folding)?, + folding: grind::(transcript, config.grind.folding)?, ..RoundNonces::default() }; @@ -702,7 +735,7 @@ where // be chosen to match them. let next = match schedule.get(r + 1) { Some(&next_k) => { - let next = commit_folded::(folded, next_k)?; + let next = commit_folded::(folded, next_k)?; transcript.append_bytes(&next.root()); Some(next) } @@ -723,7 +756,7 @@ where let y0 = factors.evaluate_message(&point)?; transcript.append_field_element(&y0); - nonces.ood = grind(transcript, config.grind.ood)?; + nonces.ood = grind::(transcript, config.grind.ood)?; let gamma: FieldElement = transcript.sample_field_element(); factors.add_scaled_eq(&point, &gamma)?; Some(y0) @@ -731,7 +764,7 @@ where None }; - nonces.query = grind(transcript, config.grind.query)?; + nonces.query = grind::(transcript, config.grind.query)?; let round_config = RoundConfig { num_queries: config.num_queries, log_folding: k, @@ -740,15 +773,21 @@ where (Current::Base(held), Some(next)) => { RoundOpenings::Base(whir_round::prove(*held, next, &round_config, transcript)?) } - (Current::Base(held), None) => { - RoundOpenings::Base(final_openings::(held, &round_config, transcript)?) - } + (Current::Base(held), None) => RoundOpenings::Base(final_openings::( + held, + &round_config, + transcript, + )?), (Current::Extension(held), Some(next)) => { RoundOpenings::Extension(whir_round::prove(held, next, &round_config, transcript)?) } - (Current::Extension(held), None) => RoundOpenings::Extension( - final_openings::(held, &round_config, transcript)?, - ), + (Current::Extension(held), None) => { + RoundOpenings::Extension(final_openings::( + held, + &round_config, + transcript, + )?) + } }; rounds.push(ChainRound { @@ -802,19 +841,20 @@ where } /// Commits a folded codeword where it is. -fn commit_folded( +fn commit_folded( codeword: Codeword, log_folding: usize, -) -> Result, Error> +) -> Result, Error> where N: IsField + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, { match codeword { Codeword::Host(values) => CodewordCommitment::from_codeword(values, log_folding), Codeword::Device(device) => { let nodes = device - .commit(log_folding) + .commit(log_folding, H::DEVICE) .ok_or(Error::DeviceFailed { stage: "fold tree" })?; CodewordCommitment::from_device(device, nodes, log_folding) } @@ -839,8 +879,8 @@ where /// /// Mirrors [`whir_round`]'s query draw, so both sides sample the same /// positions. -fn final_openings( - current: &CodewordCommitment, +fn final_openings( + current: &CodewordCommitment, config: &RoundConfig, transcript: &mut T, ) -> Result, Error> @@ -849,6 +889,7 @@ where N: IsField, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let queries: Vec = (0..config.num_queries) .map(|_| transcript.sample_u64(current.num_leaves() as u64) as usize) @@ -860,7 +901,7 @@ where } /// Verifies `f(z) = y`. -pub fn verify( +pub fn verify( proof: &ChainProof, root: &Commitment, z: &[FieldElement], @@ -875,8 +916,9 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { - verify_weighted::( + verify_weighted::( proof, root, |alphas: &[FieldElement]| eq_eval(z, alphas), @@ -893,7 +935,7 @@ where /// `weight_at` is the weight's closed form, evaluated at the concatenation of /// every round's challenges. #[allow(clippy::too_many_arguments)] -pub fn verify_weighted( +pub fn verify_weighted( proof: &ChainProof, root: &Commitment, weight_at: W, @@ -910,6 +952,7 @@ where FieldElement: AsBytes + Sync + Send, T: IsTranscript, W: FnOnce(&[FieldElement]) -> Result, Error>, + H: WhirHash, { let schedule = config.schedule(num_vars); if proof.rounds.len() != schedule.len() { @@ -943,7 +986,7 @@ where got: r, }); } - check_grind(transcript, config.grind.folding, round.nonces.folding)?; + check_grind::(transcript, config.grind.folding, round.nonces.folding)?; // The weight raises the degree of the plain `f` term to two. let group = sumcheck::verify_rounds(&round.sumcheck, claim, 2, transcript)?; claim = group.expected_evaluation; @@ -971,19 +1014,19 @@ where let point = ood_point(&z0, num_vars - bound); transcript.append_field_element(y0); - check_grind(transcript, config.grind.ood, round.nonces.ood)?; + check_grind::(transcript, config.grind.ood, round.nonces.ood)?; let gamma: FieldElement = transcript.sample_field_element(); claim += &gamma * y0; ood.push((gamma, point, bound)); - check_grind(transcript, config.grind.query, round.nonces.query)?; + check_grind::(transcript, config.grind.query, round.nonces.query)?; let commitments = RoundCommitments { current_root: ¤t_root, next_root, next_num_leaves: next_domain.size() >> next_k, }; match &round.openings { - RoundOpenings::Base(openings) => whir_round::verify::( + RoundOpenings::Base(openings) => whir_round::verify::( openings, commitments, ¤t_domain, @@ -991,7 +1034,7 @@ where &round_config, transcript, )?, - RoundOpenings::Extension(openings) => whir_round::verify::( + RoundOpenings::Extension(openings) => whir_round::verify::( openings, commitments, ¤t_domain, @@ -1004,9 +1047,9 @@ where } (None, None, None) => { transcript.append_field_element(&proof.final_value); - check_grind(transcript, config.grind.query, round.nonces.query)?; + check_grind::(transcript, config.grind.query, round.nonces.query)?; match &round.openings { - RoundOpenings::Base(openings) => verify_final::( + RoundOpenings::Base(openings) => verify_final::( openings, ¤t_root, ¤t_domain, @@ -1015,7 +1058,7 @@ where &proof.final_value, transcript, )?, - RoundOpenings::Extension(openings) => verify_final::( + RoundOpenings::Extension(openings) => verify_final::( openings, ¤t_root, ¤t_domain, @@ -1062,7 +1105,7 @@ where } /// The last round: every queried block must fold to the constant that was sent. -fn verify_final( +fn verify_final( openings: &RoundProof, current_root: &Commitment, current_domain: &Domain, @@ -1077,6 +1120,7 @@ where N: IsField + 'static, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { if openings.current.len() != config.num_queries || !openings.next.is_empty() { return Err(Error::QueryCountMismatch { @@ -1088,7 +1132,7 @@ where for (i, opening) in openings.current.iter().enumerate() { let q = transcript.sample_u64(num_leaves as u64) as usize; - if !verify_opening::(current_root, q, opening) { + if !verify_opening::(current_root, q, opening) { return Err(Error::OpeningRejected { query: i }); } if fold_coset::(&opening.values, current_domain, q, alphas)? != *final_value { @@ -1105,7 +1149,7 @@ mod tests { use crypto::fiat_shamir::default_transcript::DefaultTranscript; use math::field::goldilocks::GoldilocksField as F; - use crate::{eq::eq_evals, whir_eval}; + use crate::{eq::eq_evals, whir_eval, whir_hash::KeccakWhir}; type FE = FieldElement; @@ -1165,9 +1209,10 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true)?; - let proof = prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript())?; - verify::( + let (commitment, domain) = commit::(&f, &cfg, true)?; + let proof = + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript())?; + verify::( &proof, &commitment.root(), &z, @@ -1269,16 +1314,18 @@ mod tests { let f = pseudo_mle(num_vars, 13); let z = point(num_vars); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let chained = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); let one_round_cfg = whir_eval::EvalConfig { log_blowup: cfg.log_blowup, num_queries: cfg.num_queries, }; - let (one_commitment, one_domain) = whir_eval::commit::(&f, &one_round_cfg).unwrap(); - let one_round = whir_eval::prove::( + let (one_commitment, one_domain) = + whir_eval::commit::(&f, &one_round_cfg).unwrap(); + let one_round = whir_eval::prove::( &f, &z, &one_commitment, @@ -1307,12 +1354,13 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); assert!( - verify::( + verify::( &proof, &commitment.root(), &z, @@ -1333,13 +1381,14 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); proof.final_value += FE::one(); assert!( - verify::( + verify::( &proof, &commitment.root(), &z, @@ -1360,13 +1409,14 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); assert!(proof.rounds.len() >= 3); current_blocks_mut(&mut proof.rounds[1].openings)[0].values[0] += FE::one(); - let err = verify::( + let err = verify::( &proof, &commitment.root(), &z, @@ -1389,13 +1439,14 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); assert!(matches!(proof.rounds[0].openings, RoundOpenings::Base(_))); current_blocks_mut(&mut proof.rounds[0].openings)[0].values[0] += FE::one(); - let err = verify::( + let err = verify::( &proof, &commitment.root(), &z, @@ -1418,11 +1469,12 @@ mod tests { let g = pseudo_mle(num_vars, 31); let z = point(num_vars); - let (f_commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (f_commitment, domain) = commit::(&f, &cfg, true).unwrap(); let proof = - prove::(&g, &z, &f_commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&g, &z, &f_commitment, &domain, &cfg, &mut transcript()) + .unwrap(); - let err = verify::( + let err = verify::( &proof, &f_commitment.root(), &z, @@ -1446,13 +1498,14 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); proof.rounds[0].next_root = None; assert!( - verify::( + verify::( &proof, &commitment.root(), &z, @@ -1473,14 +1526,23 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); let mut other = DefaultTranscript::::new(b"a-different-statement"); assert!( - verify::(&proof, &commitment.root(), &z, y, &domain, &cfg, &mut other) - .is_err() + verify::( + &proof, + &commitment.root(), + &z, + y, + &domain, + &cfg, + &mut other + ) + .is_err() ); } @@ -1504,12 +1566,18 @@ mod tests { let weight = Mle::new(table).unwrap(); let y = f.evaluate(&a).unwrap() + gamma * f.evaluate(&b).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); - let proof = - prove_weighted::(&f, weight, &commitment, &domain, &cfg, &mut transcript()) - .unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let proof = prove_weighted::( + &f, + weight, + &commitment, + &domain, + &cfg, + &mut transcript(), + ) + .unwrap(); - verify_weighted::( + verify_weighted::( &proof, &commitment.root(), |at: &[FE]| Ok(eq_eval(&a, at)? + gamma * eq_eval(&b, at)?), @@ -1546,9 +1614,10 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); // `F` and `E` coincide here, so the same blocks fit the other variant. if let RoundOpenings::Base(openings) = proof.rounds[0].openings.clone() { @@ -1556,7 +1625,7 @@ mod tests { } assert!( - verify::( + verify::( &proof, &commitment.root(), &z, @@ -1635,13 +1704,14 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); proof.rounds[0].ood_value = Some(proof.rounds[0].ood_value.unwrap() + FE::one()); assert!( - verify::( + verify::( &proof, &commitment.root(), &z, @@ -1662,13 +1732,14 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut proof = - prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()).unwrap(); + prove::(&f, &z, &commitment, &domain, &cfg, &mut transcript()) + .unwrap(); proof.rounds[0].ood_value = None; assert!( - verify::( + verify::( &proof, &commitment.root(), &z, @@ -1712,8 +1783,10 @@ mod tests { let f = pseudo_mle(num_vars, 71); let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, cfg, true).unwrap(); - let proof = prove::(&f, &z, &commitment, &domain, cfg, &mut transcript()).unwrap(); + let (commitment, domain) = commit::(&f, cfg, true).unwrap(); + let proof = + prove::(&f, &z, &commitment, &domain, cfg, &mut transcript()) + .unwrap(); (proof, commitment.root(), domain, y) } @@ -1725,7 +1798,7 @@ mod tests { cfg: &ChainConfig, num_vars: usize, ) -> Result<(), Error> { - verify::( + verify::( proof, root, &point(num_vars), @@ -1848,12 +1921,13 @@ mod tests { let z: Vec = (0..num_vars).map(|i| ExtE::from(101 + i as u64)).collect(); let y = f.evaluate_in(&z).unwrap(); - let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); + let (commitment, domain) = commit::(&f, &cfg, true).unwrap(); let mut prover = DefaultTranscript::::new(b"tower"); - let proof = prove::(&f, &z, &commitment, &domain, &cfg, &mut prover).unwrap(); + let proof = prove::(&f, &z, &commitment, &domain, &cfg, &mut prover) + .unwrap(); let mut verifier = DefaultTranscript::::new(b"tower"); - verify::( + verify::( &proof, &commitment.root(), &z, diff --git a/crypto/multilinear/src/whir_commit.rs b/crypto/multilinear/src/whir_commit.rs index 09371e214..f8aed99aa 100644 --- a/crypto/multilinear/src/whir_commit.rs +++ b/crypto/multilinear/src/whir_commit.rs @@ -3,10 +3,7 @@ //! The pre-image of folded index `j` is the stride-`N/2^k` coset //! `{ j, j + N/2^k, …, j + (2^k - 1)·N/2^k }`. -use crypto::merkle_tree::{ - backends::types::BatchKeccak256Backend, merkle::MerkleTree, proof::Proof, - traits::IsMerkleTreeBackend, -}; +use crypto::merkle_tree::{merkle::MerkleTree, proof::Proof, traits::IsMerkleTreeBackend}; use math::{ field::{ element::FieldElement, @@ -18,19 +15,29 @@ use math::{ #[cfg(feature = "parallel")] use rayon::prelude::*; -use crate::{Error, whir::Domain}; +use crate::{ + Error, + whir::Domain, + whir_hash::{KeccakWhir, WhirHash}, +}; -/// 32-byte Keccak commitments, matching the rest of the prover. +/// 32-byte commitments, matching the rest of the prover. +/// +/// ★ **The width is the same for every [`WhirHash`]** — a keccak digest is 32 +/// bytes and an algebraic digest is four canonical Goldilocks felts, which is +/// also 32 bytes. That is what keeps a hash swap out of the proof format: every +/// type below and above this one keeps its layout, its rkyv derives and its +/// serialized length. pub type Commitment = [u8; 32]; -type Backend = BatchKeccak256Backend; -type Tree = MerkleTree>; +type Backend = ::Backend; +type Tree = MerkleTree>; /// A committed codeword and the tree needed to open it. -pub struct CodewordCommitment +pub struct CodewordCommitment where FieldElement: AsBytes + Sync + Send, { - tree: Tree, + tree: Tree, codeword: Codeword, log_folding: usize, log_domain_size: usize, @@ -113,7 +120,7 @@ pub fn leaf_and_slot(position: usize, num_leaves: usize) -> (usize, usize) { (position % num_leaves, position / num_leaves) } -impl std::fmt::Debug for CodewordCommitment +impl std::fmt::Debug for CodewordCommitment where FieldElement: AsBytes + Sync + Send, { @@ -127,7 +134,7 @@ where } } -impl CodewordCommitment +impl CodewordCommitment where FieldElement: AsBytes + Sync + Send, { @@ -155,8 +162,8 @@ where }); } - if let Some(nodes) = crate::gpu::commit_tree_ext3(&codeword, log_folding) { - let tree = Tree::::from_precomputed_nodes(nodes).ok_or(Error::EmptyPolynomial)?; + if let Some(nodes) = crate::gpu::commit_tree_ext3(&codeword, log_folding, H::DEVICE) { + let tree = Tree::::from_precomputed_nodes(nodes).ok_or(Error::EmptyPolynomial)?; return Ok(Self { tree, codeword: Codeword::Host(codeword), @@ -190,7 +197,7 @@ where let hash_leaf = |buffer: &mut Vec>, j: usize| { buffer.clear(); buffer.extend((0..block).map(|t| codeword[j + t * num_leaves].clone())); - Backend::::hash_data(buffer) + Backend::::hash_data(buffer) }; #[cfg(feature = "parallel")] let hashed: Vec<_> = (0..num_leaves) @@ -203,7 +210,7 @@ where (0..num_leaves).map(|j| hash_leaf(&mut buffer, j)).collect() }; - let tree = Tree::::build_from_hashed_leaves(hashed).ok_or(Error::EmptyPolynomial)?; + let tree = Tree::::build_from_hashed_leaves(hashed).ok_or(Error::EmptyPolynomial)?; Ok(Self { tree, codeword: Codeword::Host(codeword), @@ -231,7 +238,7 @@ where n_stack: log_domain_size, }); } - let tree = Tree::::from_precomputed_nodes(nodes).ok_or(Error::EmptyPolynomial)?; + let tree = Tree::::from_precomputed_nodes(nodes).ok_or(Error::EmptyPolynomial)?; Ok(Self { tree, codeword: Codeword::Host(codeword), @@ -262,7 +269,7 @@ where }); } Ok(Self { - tree: Tree::::from_root(root), + tree: Tree::::from_root(root), codeword: Codeword::Device(codeword), log_folding, log_domain_size, @@ -354,7 +361,7 @@ where // device: the tree has to be rebuilt there because that is // where the codeword is. Ok(device - .paths(self.log_folding, indices) + .paths(self.log_folding, indices, H::DEVICE) .ok_or(Error::DeviceFailed { stage: "opening paths", })? @@ -399,15 +406,22 @@ where } } -/// Checks an opening against a root. -pub fn verify_opening(root: &Commitment, index: usize, opening: &CosetOpening) -> bool +/// Checks an opening against a root, under `H`'s hash. +/// +/// `H` is explicit at every call site rather than defaulted, because a free +/// function's type parameter cannot carry a default and — more to the point — +/// because "which hash authenticated this path" is the whole content of the +/// call. A verifier reading a proof under the wrong `H` gets `false` here, not +/// a different-but-plausible answer. +pub fn verify_opening(root: &Commitment, index: usize, opening: &CosetOpening) -> bool where F: IsField + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, { opening .proof - .verify::>(root, index, &opening.values) + .verify::>(root, index, &opening.values) } /// One level of a block's fold. @@ -504,6 +518,7 @@ mod tests { use crate::{ mle::Mle, whir::{encode, fold_codeword_k, monomial_coefficients}, + whir_hash::KeccakWhir, }; type FE = FieldElement; @@ -529,7 +544,7 @@ mod tests { #[test] fn leaves_cover_the_codeword_exactly_once() { let (cw, _) = pseudo_codeword(3, 2, 1); - let commitment = CodewordCommitment::new(&cw, 2).unwrap(); + let commitment = CodewordCommitment::::new(&cw, 2).unwrap(); assert_eq!(commitment.num_leaves(), cw.len() / 4); let mut seen = vec![0usize; cw.len()]; @@ -544,40 +559,43 @@ mod tests { #[test] fn an_opening_verifies_against_the_root() { let (cw, _) = pseudo_codeword(3, 2, 7); - let commitment = CodewordCommitment::new(&cw, 1).unwrap(); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); let root = commitment.root(); for j in 0..commitment.num_leaves() { let opening = commitment.open(j).unwrap(); assert_eq!(opening.values.len(), 2); - assert!(verify_opening::(&root, j, &opening), "leaf {j}"); + assert!( + verify_opening::(&root, j, &opening), + "leaf {j}" + ); } } #[test] fn a_tampered_opening_is_rejected() { let (cw, _) = pseudo_codeword(3, 2, 9); - let commitment = CodewordCommitment::new(&cw, 1).unwrap(); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); let root = commitment.root(); let mut opening = commitment.open(2).unwrap(); opening.values[0] += FE::one(); - assert!(!verify_opening::(&root, 2, &opening)); + assert!(!verify_opening::(&root, 2, &opening)); } #[test] fn an_opening_does_not_verify_at_another_index() { let (cw, _) = pseudo_codeword(3, 2, 11); - let commitment = CodewordCommitment::new(&cw, 1).unwrap(); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); let root = commitment.root(); let opening = commitment.open(2).unwrap(); - assert!(!verify_opening::(&root, 3, &opening)); + assert!(!verify_opening::(&root, 3, &opening)); } #[test] fn a_query_beyond_the_leaves_is_an_error() { let (cw, _) = pseudo_codeword(2, 1, 3); - let commitment = CodewordCommitment::new(&cw, 1).unwrap(); + let commitment = CodewordCommitment::::new(&cw, 1).unwrap(); let out = commitment.num_leaves(); assert!(matches!( commitment.open(out).unwrap_err(), @@ -595,7 +613,7 @@ mod tests { let alphas: Vec = (0..k).map(|i| FE::from(13 + i as u64)).collect(); let (folded, _) = fold_codeword_k(&cw, &domain, &alphas).unwrap(); - let commitment = CodewordCommitment::new(&cw, k).unwrap(); + let commitment = CodewordCommitment::::new(&cw, k).unwrap(); for (j, expected) in folded.iter().enumerate() { let opening = commitment.open(j).unwrap(); @@ -618,7 +636,7 @@ mod tests { #[test] fn folding_by_zero_returns_the_single_value() { let (cw, domain) = pseudo_codeword(3, 2, 5); - let commitment = CodewordCommitment::new(&cw, 0).unwrap(); + let commitment = CodewordCommitment::::new(&cw, 0).unwrap(); assert_eq!(commitment.num_leaves(), cw.len()); let opening = commitment.open(6).unwrap(); assert_eq!(fold_coset(&opening.values, &domain, 6, &[]).unwrap(), cw[6]); @@ -628,7 +646,7 @@ mod tests { fn a_codeword_that_is_not_a_power_of_two_is_rejected() { let values = vec![FE::one(); 6]; assert!(matches!( - CodewordCommitment::new(&values, 1).unwrap_err(), + CodewordCommitment::::new(&values, 1).unwrap_err(), Error::NotPowerOfTwo(6) )); } @@ -647,7 +665,7 @@ mod tests { #[test] fn a_position_resolves_to_the_value_it_holds() { let (cw, _) = pseudo_codeword(3, 2, 21); - let commitment = CodewordCommitment::new(&cw, 2).unwrap(); + let commitment = CodewordCommitment::::new(&cw, 2).unwrap(); let num_leaves = commitment.num_leaves(); for (position, value) in cw.iter().enumerate() { diff --git a/crypto/multilinear/src/whir_eval.rs b/crypto/multilinear/src/whir_eval.rs index 032358cd6..63c6c8e11 100644 --- a/crypto/multilinear/src/whir_eval.rs +++ b/crypto/multilinear/src/whir_eval.rs @@ -34,6 +34,7 @@ use crate::{ virtual_poly::{Term, VirtualPolynomial}, whir::{Domain, encode, fold_codeword_k, lift_coefficients}, whir_commit::{CodewordCommitment, Commitment, CosetOpening, fold_coset, verify_opening}, + whir_hash::WhirHash, }; /// Blowup and query count. @@ -64,13 +65,14 @@ pub struct EvalProof { } /// Commits to `f`, ready to answer evaluation claims. -pub fn commit( +pub fn commit( f: &Mle, config: &EvalConfig, -) -> Result<(CodewordCommitment, Domain), Error> +) -> Result<(CodewordCommitment, Domain), Error> where F: IsFFTField + IsPrimeField + IsSubFieldOf + 'static, E: IsField + Send + Sync + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, { let domain = Domain::::new(f.num_vars() + config.log_blowup)?; @@ -93,10 +95,10 @@ fn weighted( /// /// The caller must have absorbed the commitment root and `z` into `transcript` /// already; both sides must do the same. -pub fn prove( +pub fn prove( f: &Mle, z: &[FieldElement], - commitment: &CodewordCommitment, + commitment: &CodewordCommitment, domain: &Domain, config: &EvalConfig, transcript: &mut T, @@ -106,15 +108,16 @@ where E: IsField + Send + Sync + 'static, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { - prove_weighted::(f, eq_mle(z)?, commitment, domain, config, transcript) + prove_weighted::(f, eq_mle(z)?, commitment, domain, config, transcript) } /// Proves `Σ_x w(x)·f(x) = y` for a weight the verifier can evaluate itself. -pub fn prove_weighted( +pub fn prove_weighted( f: &Mle, weight: Mle, - commitment: &CodewordCommitment, + commitment: &CodewordCommitment, domain: &Domain, config: &EvalConfig, transcript: &mut T, @@ -124,6 +127,7 @@ where E: IsField + Send + Sync + 'static, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let (sumcheck, alphas) = sumcheck::prove(weighted(f, weight)?, transcript)?; @@ -162,7 +166,7 @@ where } /// Verifies `f(z) = y` against a commitment. -pub fn verify( +pub fn verify( proof: &EvalProof, root: &Commitment, z: &[FieldElement], @@ -176,8 +180,9 @@ where E: IsField + Send + Sync + 'static, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { - verify_weighted::( + verify_weighted::( proof, root, |alphas: &[FieldElement]| eq_eval(z, alphas), @@ -194,7 +199,7 @@ where /// `weight_at` is the weight's closed form; the verifier evaluates it at the /// sumcheck point rather than holding its table. #[allow(clippy::too_many_arguments)] -pub fn verify_weighted( +pub fn verify_weighted( proof: &EvalProof, root: &Commitment, weight_at: W, @@ -210,6 +215,7 @@ where FieldElement: AsBytes + Sync + Send, T: IsTranscript, W: FnOnce(&[FieldElement]) -> Result, Error>, + H: WhirHash, { // The weight raises the degree of the plain `f` term to two. let claim = sumcheck::verify(&proof.sumcheck, y, num_vars, 2, transcript)?; @@ -240,7 +246,7 @@ where let queries = sample_queries(transcript, config.num_queries, num_leaves); for (i, (&q, opening)) in queries.iter().zip(&proof.openings).enumerate() { - if !verify_opening::(root, q, opening) { + if !verify_opening::(root, q, opening) { return Err(Error::OpeningRejected { query: i }); } if fold_coset::(&opening.values, domain, q, alphas)? != proof.final_value { @@ -269,6 +275,8 @@ mod tests { use crypto::fiat_shamir::default_transcript::DefaultTranscript; use math::field::goldilocks::GoldilocksField as F; + use crate::whir_hash::KeccakWhir; + type FE = FieldElement; fn transcript() -> DefaultTranscript { @@ -309,12 +317,18 @@ mod tests { let z = point(num_vars); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &config()).unwrap(); - let proof = - prove::(&f, &z, &commitment, &domain, &config(), &mut transcript()) - .unwrap(); + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); - verify::( + verify::( &proof, &commitment.root(), &z, @@ -336,11 +350,18 @@ mod tests { let z = point(3); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &config()).unwrap(); - let proof = - prove::(&f, &z, &commitment, &domain, &config(), &mut transcript()).unwrap(); + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); - let err = verify::( + let err = verify::( &proof, &commitment.root(), &z, @@ -361,12 +382,19 @@ mod tests { let z = point(3); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &config()).unwrap(); - let mut proof = - prove::(&f, &z, &commitment, &domain, &config(), &mut transcript()).unwrap(); + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let mut proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); proof.final_value += FE::one(); - let err = verify::( + let err = verify::( &proof, &commitment.root(), &z, @@ -385,12 +413,19 @@ mod tests { let z = point(3); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &config()).unwrap(); - let mut proof = - prove::(&f, &z, &commitment, &domain, &config(), &mut transcript()).unwrap(); + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let mut proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); proof.openings[0].values[0] += FE::one(); - let err = verify::( + let err = verify::( &proof, &commitment.root(), &z, @@ -410,12 +445,19 @@ mod tests { let g = pseudo_mle(3, 29); let z = point(3); - let (f_commitment, domain) = commit::(&f, &config()).unwrap(); + let (f_commitment, domain) = commit::(&f, &config()).unwrap(); // Argue g's evaluation while presenting f's commitment. - let proof = - prove::(&g, &z, &f_commitment, &domain, &config(), &mut transcript()).unwrap(); + let proof = prove::( + &g, + &z, + &f_commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); - let err = verify::( + let err = verify::( &proof, &f_commitment.root(), &z, @@ -437,13 +479,20 @@ mod tests { let z = point(3); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &config()).unwrap(); - let proof = - prove::(&f, &z, &commitment, &domain, &config(), &mut transcript()).unwrap(); + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); let mut other = DefaultTranscript::::new(b"a-different-statement"); assert!( - verify::( + verify::( &proof, &commitment.root(), &z, @@ -462,12 +511,19 @@ mod tests { let z = point(3); let y = f.evaluate(&z).unwrap(); - let (commitment, domain) = commit::(&f, &config()).unwrap(); - let mut proof = - prove::(&f, &z, &commitment, &domain, &config(), &mut transcript()).unwrap(); + let (commitment, domain) = commit::(&f, &config()).unwrap(); + let mut proof = prove::( + &f, + &z, + &commitment, + &domain, + &config(), + &mut transcript(), + ) + .unwrap(); proof.openings.pop(); - let err = verify::( + let err = verify::( &proof, &commitment.root(), &z, @@ -484,7 +540,7 @@ mod tests { fn the_commitment_has_one_block_per_fold_target() { let f = pseudo_mle(4, 41); let cfg = config(); - let (commitment, domain) = commit::(&f, &cfg).unwrap(); + let (commitment, domain) = commit::(&f, &cfg).unwrap(); assert_eq!(commitment.num_leaves(), 1 << cfg.log_blowup); assert_eq!(domain.log_size(), 4 + cfg.log_blowup); } diff --git a/crypto/multilinear/src/whir_hash.rs b/crypto/multilinear/src/whir_hash.rs new file mode 100644 index 000000000..645250ee0 --- /dev/null +++ b/crypto/multilinear/src/whir_hash.rs @@ -0,0 +1,244 @@ +//! ★ The hash the WHIR path runs on — the one name a proof's Merkle trees, its +//! Fiat-Shamir sponge and its proof-of-work all answer to. +//! +//! # Why one trait and not three parameters +//! +//! A WHIR proof has three hash consumers: the Merkle backend that builds its +//! roots, the transcript sponge that draws its challenges, and the grind that +//! gates each redrawable challenge. Parameterising them separately would make +//! the **half-flip** spellable — one hash's trees under another hash's sponge. +//! That configuration is self-consistent between prover and verifier, so it +//! verifies, so nothing fails; it is silent by construction, and the only thing +//! wrong with it is that no single name describes the proof. Here there is one +//! name to write, so there is nothing to assert against: the bad state is +//! unreachable rather than checked. +//! +//! # What a hash may NOT change +//! +//! [`Commitment`] stays `[u8; 32]` for every implementation. A keccak digest is +//! 32 bytes and an algebraic digest is four canonical Goldilocks felts, which is +//! also 32 bytes — so every proof type on this path (`ChainProof`, +//! `StackedProof`, `CosetOpening`, `Proof`, and `MultiProof` above +//! them) keeps its layout, its rkyv derives and its serialized length. **A hash +//! swap is not a proof-format change**, and `stacked_eval`'s +//! `the_two_hashes_serialize_to_the_same_length` is what holds that to it. +//! +//! # The device +//! +//! Under `cuda` the leaf and parent hashing happens in KERNELS, and the host +//! backend is only the label on the tree they built. So a configuration must +//! also name which kernel family the device has to run: +//! [`WhirHash::DEVICE`] is that name, handed down to `math-cuda`'s tree entry +//! points, which match on it exhaustively. A tree labelled `Self` was therefore +//! hashed by `Self`'s kernels or was not built on the device at all — never by +//! another hash's kernels wearing this name. +//! +//! The key is a type of this crate's own rather than `math_cuda::DeviceHash` +//! directly, because `math-cuda` is an optional dependency and the trait has to +//! exist on a build without it. [`DeviceHashKey::into_math_cuda`] is the bridge, +//! and it is total in both directions with the pairing asserted at compile time, +//! so the two enums cannot drift apart or be cross-wired. + +use crypto::fiat_shamir::transcript_hash::{ + KeccakTranscriptHash, RpxTranscriptHash, TranscriptHash, +}; +use crypto::merkle_tree::backends::types::{BatchKeccak256Backend, BatchRpx256Backend}; +use crypto::merkle_tree::traits::IsMerkleTreeBackend; +use math::field::{element::FieldElement, traits::IsField}; +use math::traits::AsBytes; + +use crate::whir_commit::Commitment; + +/// The digest a configuration grinds over: its transcript's hash, because the +/// grinding seed is `transcript.state()`. +pub type GrindingDigest = <::Transcript as TranscriptHash>::Digest; + +/// ★ Which kernel family the device must run for a configuration's trees. +/// +/// Mirrors `math_cuda::DeviceHash` and exists separately only so this trait +/// compiles without the optional `math-cuda` dependency. The two are kept in +/// step by [`DeviceHashKey::into_math_cuda`] plus the compile-time pairing +/// assertion beside it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DeviceHashKey { + /// Keccak-256 at both the leaf and the parent layer. + Keccak256, + /// RPX256 (XHash12) at both layers. + Rpx256, +} + +impl DeviceHashKey { + /// The key `math-cuda` dispatches on. + /// + /// Total, and exhaustive in both directions: a variant added on either side + /// without its twin is a compile error here rather than a silent + /// fallthrough to whichever hash happened to be first. + #[cfg(feature = "cuda")] + pub const fn into_math_cuda(self) -> math_cuda::DeviceHash { + match self { + Self::Keccak256 => math_cuda::DeviceHash::Keccak256, + Self::Rpx256 => math_cuda::DeviceHash::Rpx256, + } + } + + /// The name a tree built under this key may be called by. + pub const fn name(self) -> &'static str { + match self { + Self::Keccak256 => "keccak256", + Self::Rpx256 => "rpx256", + } + } +} + +/// ✓ The bridge is a bijection, checked at compile time rather than by reading +/// it: every key maps to the twin of the same name, and the names agree. +#[cfg(feature = "cuda")] +const _: () = { + const fn paired(key: DeviceHashKey, twin: math_cuda::DeviceHash) -> bool { + key.into_math_cuda() as u8 == twin as u8 + } + assert!(paired( + DeviceHashKey::Keccak256, + math_cuda::DeviceHash::Keccak256 + )); + assert!(paired(DeviceHashKey::Rpx256, math_cuda::DeviceHash::Rpx256)); +}; + +/// ★ One WHIR hash configuration. +/// +/// Implementing it is the whole of adding a hash to this path: a unit struct, a +/// Merkle backend and a Fiat-Shamir configuration. Nothing in `whir_commit`, +/// `whir_round`, `whir_chain` or `stacked_eval` knows which one it has. +pub trait WhirHash: Copy + Clone + Default + Send + Sync + 'static { + /// The name a proof's roots may be called by — for banners, KATs and + /// diagnostics. Never absorbed into the transcript: the sponge IS this + /// hash, so two configurations' challenge streams diverge at the first + /// squeeze and a tag would separate nothing that is not already separate. + const NAME: &'static str; + + /// The kernel family the device must run for trees labelled `Self`. + /// + /// Not `#[cfg(feature = "cuda")]`: a configuration names its device hash on + /// every build, so a non-cuda build cannot define a configuration that would + /// have had nothing to dispatch on. + const DEVICE: DeviceHashKey; + + /// The Fiat-Shamir configuration this commitment hash is paired with — the + /// sponge's hash, and the one the proof-of-work grind computes over. + type Transcript: TranscriptHash; + + /// The Merkle backend: one leaf per fold block, 32-byte nodes. + type Backend: IsMerkleTreeBackend>> + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; +} + +/// The keccak-256 configuration — the default everywhere on this path, and +/// byte-for-byte what PR #988 produces. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct KeccakWhir; + +impl WhirHash for KeccakWhir { + const NAME: &'static str = "keccak256"; + + const DEVICE: DeviceHashKey = DeviceHashKey::Keccak256; + + type Transcript = KeccakTranscriptHash; + + type Backend + = BatchKeccak256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; +} + +/// ★ The RPX256 configuration — the algebraic hash, and the only reason this +/// trait exists. +/// +/// Slower than keccak on a host by a wide margin, and that is not a defect to +/// be fixed: the lever it pulls is elsewhere. A keccak-f[1600] costs roughly +/// 73,700 trace cells in a field-native verifier against RPX's 325, so a WHIR +/// proof verified inside another proof pays about 227x less for its hashing +/// under this configuration. A proof that will only ever be checked by a host +/// should use [`KeccakWhir`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct RpxWhir; + +impl WhirHash for RpxWhir { + const NAME: &'static str = "rpx256"; + + const DEVICE: DeviceHashKey = DeviceHashKey::Rpx256; + + type Transcript = RpxTranscriptHash; + + type Backend + = BatchRpx256Backend + where + F: IsField + 'static, + FieldElement: AsBytes + Sync + Send; +} + +/// ✓ Both configurations are INHABITED at the fields the prover actually +/// commits over — the base field for traces, the cubic extension for the folded +/// codewords — within one proof. +/// +/// A `WhirHash` impl that type-checks in isolation can still be unusable: the +/// associated type is generic over `F`, and the bound that matters is the one +/// the prover instantiates it at. This is that instantiation, as a compile-time +/// check rather than a comment claiming it holds. +const _: fn() = || { + fn assert_usable() + where + H::Backend: + IsMerkleTreeBackend, + H::Backend: + IsMerkleTreeBackend, + { + } + + assert_usable::(); + assert_usable::(); +}; + +/// ✓ Each configuration's transcript is ITS OWN, not the other's. +/// +/// The half-flip is unspellable because one trait supplies both halves — but +/// only if the two impls actually name different transcripts. A copy-paste that +/// left `RpxWhir` on `KeccakTranscriptHash` would be exactly the silent +/// configuration this design exists to rule out, so it is made a compile error. +const _: fn() = || { + fn assert_same(_: core::marker::PhantomData<(T, T)>) {} + + assert_same::( + core::marker::PhantomData::<(KeccakTranscriptHash, ::Transcript)>, + ); + assert_same::( + core::marker::PhantomData::<(RpxTranscriptHash, ::Transcript)>, + ); +}; + +/// ✓ A configuration and its device key answer to the SAME name. +/// +/// Both are string constants written by hand, so nothing but this stops +/// `RpxWhir::NAME` from saying `rpx256` while its kernels are filed under +/// `keccak256` — which is precisely the mislabelling the key exists to prevent, +/// reintroduced one level up. +const _: () = { + const fn same(a: &str, b: &str) -> bool { + let (a, b) = (a.as_bytes(), b.as_bytes()); + if a.len() != b.len() { + return false; + } + let mut i = 0; + while i < a.len() { + if a[i] != b[i] { + return false; + } + i += 1; + } + true + } + assert!(same(KeccakWhir::NAME, KeccakWhir::DEVICE.name())); + assert!(same(RpxWhir::NAME, RpxWhir::DEVICE.name())); +}; diff --git a/crypto/multilinear/src/whir_round.rs b/crypto/multilinear/src/whir_round.rs index e2abe21a6..3cffbcd93 100644 --- a/crypto/multilinear/src/whir_round.rs +++ b/crypto/multilinear/src/whir_round.rs @@ -17,6 +17,7 @@ use crate::{ Error, whir::Domain, whir_commit::{CodewordCommitment, Commitment, CosetOpening, fold_coset, leaf_and_slot}, + whir_hash::WhirHash, }; /// How hard a round is to cheat. @@ -78,9 +79,9 @@ where /// /// `current` and `next` must already be committed, and `next` must be the fold /// of `current` by `alphas` — [`verify`] is what checks that claim. -pub fn prove( - current: &CodewordCommitment, - next: &CodewordCommitment, +pub fn prove( + current: &CodewordCommitment, + next: &CodewordCommitment, config: &RoundConfig, transcript: &mut T, ) -> Result, Error> @@ -90,6 +91,7 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { let queries = sample_queries(transcript, config.num_queries, current.num_leaves()); @@ -108,7 +110,7 @@ where /// /// Re-derives the queries from the transcript, so the prover could not have /// chosen them. -pub fn verify( +pub fn verify( proof: &RoundProof, commitments: RoundCommitments<'_>, domain: &Domain, @@ -123,6 +125,7 @@ where FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, T: IsTranscript, + H: WhirHash, { if alphas.len() != config.log_folding { return Err(Error::VariableCountMismatch { @@ -145,11 +148,11 @@ where .zip(proof.current.iter().zip(&proof.next)) .enumerate() { - if !crate::whir_commit::verify_opening::(commitments.current_root, q, cur) { + if !crate::whir_commit::verify_opening::(commitments.current_root, q, cur) { return Err(Error::OpeningRejected { query: i }); } let (leaf, slot) = leaf_and_slot(q, commitments.next_num_leaves); - if !crate::whir_commit::verify_opening::(commitments.next_root, leaf, nxt) { + if !crate::whir_commit::verify_opening::(commitments.next_root, leaf, nxt) { return Err(Error::OpeningRejected { query: i }); } @@ -175,6 +178,7 @@ mod tests { use crate::{ mle::Mle, whir::{encode, fold_codeword_k, monomial_coefficients}, + whir_hash::KeccakWhir, }; type FE = FieldElement; @@ -216,7 +220,7 @@ mod tests { } fn run(fx: &Fixture, proof: &RoundProof) -> Result<(), Error> { - verify::( + verify::( proof, RoundCommitments { current_root: &fx.current.root(), @@ -314,7 +318,7 @@ mod tests { run(&fx, &proof).unwrap(); let mut other = DefaultTranscript::::new(b"a-different-statement"); - let result = verify::( + let result = verify::( &proof, RoundCommitments { current_root: &fx.current.root(), diff --git a/crypto/multilinear/tests/host_fallback_counter.rs b/crypto/multilinear/tests/host_fallback_counter.rs new file mode 100644 index 000000000..d807a1c75 --- /dev/null +++ b/crypto/multilinear/tests/host_fallback_counter.rs @@ -0,0 +1,84 @@ +//! ★ Every commit is accounted for on exactly one side: it ran on the device, +//! or it fell back to the host. Neither is silent. +//! +//! No GPU needed, and that is the point — without a card every commit takes the +//! fallback arm, so the arm this file is about is the one that always runs here. +//! +//! ```text +//! cargo test -p multilinear --test host_fallback_counter +//! ``` +//! +//! # Why this exists +//! +//! A device commit that declines does not report anything. `commit_calls()` +//! simply does not rise, and a count merely lower than expected says nothing +//! when the expected count is itself derived from the table census. That +//! silence hid a real regression: keeping a Merkle tree per commitment filled +//! the card, every commit after the ceiling encoded on the host instead, and +//! the only visible symptoms were host memory and a GPU-utilisation figure +//! that read like a scheduling problem. The arm was taken thousands of times +//! and no line of output named it. +//! +//! Falling back is not a slower route to the same place. `from_codeword` +//! retains the host codeword AND its node array for the rest of the proof, so +//! a card that fills partway through an epoch converts into gigabytes of host +//! memory that never comes back. +//! +//! # Why this assertion and not `host_fallbacks() == 3` +//! +//! Because that one would be true on this laptop and false on the box, and a +//! test whose truth depends on which machine ran it is not pinning anything. +//! The SUM is the invariant on both: three commits, three accounted for, +//! wherever they ran. It fails if the fallback arm forgets to count — the sum +//! reads 0 on a GPU-less host and short on a card, which is exactly the bug. +//! +//! Its own integration binary so the process-wide counters belong to it alone. +//! Read `crypto/math-cuda/tests/whir_tree_cache.rs` for what a global counter +//! costs when several tests share a binary. + +use math::field::element::FieldElement; +use math::field::goldilocks::GoldilocksField as F; +use multilinear::mle::Mle; +use multilinear::whir_chain::{ChainConfig, GrindBits, commit}; +use multilinear::whir_hash::KeccakWhir; + +fn poly(num_vars: usize, seed: u64) -> Mle { + Mle::new( + (0..(1u64 << num_vars)) + .map(|i| FieldElement::from(i.wrapping_mul(6364136223846793005).wrapping_add(seed))) + .collect(), + ) + .expect("power of two") +} + +#[test] +fn every_commit_is_counted_on_exactly_one_side() { + let config = ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + }; + + multilinear::gpu::reset_call_counters(); + assert_eq!( + multilinear::gpu::commit_calls() + multilinear::gpu::host_fallbacks(), + 0, + "the reset must clear both counters, or the count below is someone else's" + ); + + const COMMITS: u64 = 3; + for seed in 0..COMMITS { + let f = poly(8, seed * 7 + 1); + let _ = commit::(&f, &config, false).expect("commit"); + } + + let on_device = multilinear::gpu::commit_calls(); + let on_host = multilinear::gpu::host_fallbacks(); + assert_eq!( + on_device + on_host, + COMMITS, + "{COMMITS} commits were made; {on_device} on the device and {on_host} \ + on the host — an arm is silent" + ); +} diff --git a/crypto/stark/src/grinding.rs b/crypto/stark/src/grinding.rs index 0e38a9c42..21207137b 100644 --- a/crypto/stark/src/grinding.rs +++ b/crypto/stark/src/grinding.rs @@ -1,7 +1,19 @@ //! The grinding primitive and its device dispatch, re-exported so existing //! call sites read unchanged. Both live in [`crypto::grinding`], which the //! multilinear prover can also reach. +//! +//! ⚠ The primitive is generic over its hash and carries **no default** — a +//! proof-of-work hash that defaulted would silently keep grinding on keccak for +//! a configuration that had moved everything else. This crate's univariate +//! prover and verifier are keccak throughout, so they name +//! [`StarkGrindingDigest`] at each call site: one token, but it is a statement +//! rather than an omission, and it is the thing that has to change if this path +//! ever gains a hash parameter of its own. pub use crypto::grinding::{ generate_nonce, generate_nonce_maybe_gpu, inner_hash_lanes, is_valid_nonce, }; + +/// The hash the univariate prover and verifier grind with: keccak-256, the same +/// hash their Merkle trees and their transcript use. +pub type StarkGrindingDigest = crypto::hash::platform_keccak::PlatformKeccak256; diff --git a/crypto/stark/src/multilinear_air.rs b/crypto/stark/src/multilinear_air.rs index 38814618d..60a4ee11e 100644 --- a/crypto/stark/src/multilinear_air.rs +++ b/crypto/stark/src/multilinear_air.rs @@ -1666,6 +1666,7 @@ mod tests { use multilinear::{ constraint_argument::{self, CommittedTrace, TraceClaim}, whir_chain::{ChainConfig, GrindBits}, + whir_hash::KeccakWhir, }; let (prog, meta) = fib_program(); @@ -1699,7 +1700,7 @@ mod tests { let betas = beta_powers(&ExtE::from(5), shape.num_roots()); let mut prover_transcript = DefaultTranscript::::new(b"air-argument"); - let proof = constraint_argument::prove::( + let proof = constraint_argument::prove::( &trace, |v: &[ExtE]| shape.combine(&betas, v), degree, @@ -1708,7 +1709,7 @@ mod tests { )?; let mut verifier_transcript = DefaultTranscript::::new(b"air-argument"); - constraint_argument::verify::( + constraint_argument::verify::( &proof, TraceClaim { roots: &roots, diff --git a/crypto/stark/src/multilinear_table.rs b/crypto/stark/src/multilinear_table.rs index b82516f66..266a8670a 100644 --- a/crypto/stark/src/multilinear_table.rs +++ b/crypto/stark/src/multilinear_table.rs @@ -35,6 +35,7 @@ use multilinear::{ whir::Domain, whir_chain::ChainConfig, whir_commit::Commitment, + whir_hash::{KeccakWhir, WhirHash}, }; use crate::constraint_ir::ir::ConstraintProgram; @@ -323,10 +324,11 @@ where /// ends at its own point. /// /// [`Claimed::PerColumn`]: multilinear::stacked_eval::Claimed::PerColumn -pub struct CommittedTables<'a, F, E> +pub struct CommittedTables<'a, F, E, H = KeccakWhir> where F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, E: IsField + Send + Sync + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, { @@ -338,7 +340,7 @@ where /// case; more than one exists so a table can have a commitment of its own — /// which is what binds the same table across two proofs, since a table has /// no root of its own when it shares a stack. - groups: Vec>, + groups: Vec>, /// How many tables each group holds, in order. sizes: Vec, roots: Vec, @@ -415,10 +417,11 @@ pub fn global_layouts( Ok(layouts) } -impl<'a, F, E> CommittedTables<'a, F, E> +impl<'a, F, E, H> CommittedTables<'a, F, E, H> where F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, E: IsField + Send + Sync + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, { @@ -479,7 +482,7 @@ where // By reference: the stack copies every column into its own buffer, // and the trace holds the originals for the rest of the proof. let columns: Vec<&Mle> = group.iter().flat_map(|t| t.columns()).collect(); - let stacked = StackedCommitment::::commit( + let stacked = StackedCommitment::::commit( layout, &columns, store.as_ref().map(|store| (&**store, firsts[at])), @@ -520,7 +523,7 @@ where } /// The stacks, one per group. - pub fn groups(&self) -> &[StackedCommitment] { + pub fn groups(&self) -> &[StackedCommitment] { &self.groups } } @@ -838,17 +841,33 @@ where /// Each table's sumcheck leaves its columns claimed at a point of its own. /// Those go into **one** opening at the end, which is what makes a proof of /// many tables cost about what a proof of one does. -pub fn multi_prove( - committed: &CommittedTables<'_, F, E>, +pub fn multi_prove( + committed: &CommittedTables<'_, F, E, H>, config: &ChainConfig, transcript: &mut T, ) -> Result, MlError> where F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, E: IsField + Send + Sync + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, - T: crypto::fiat_shamir::is_transcript::IsTranscript, + // ★★ The transcript's hash must BE the configuration's. Not "should": a + // caller that passes a keccak transcript under an RPX `H` does not compile. + // + // `DefaultTranscript`'s hash parameter has a default, so `DefaultTranscript::` + // is a keccak transcript that looks like it names no hash. Every call site + // wrote exactly that, and the RPX configuration therefore ran an RPX Merkle + // backend, an RPX grind and a KECCAK sponge — through four measured A/Bs, + // with no instrument disagreeing, because nothing failed: the proofs were + // valid and the arms did differ from each other. + // + // Requiring each site to NAME a hash would not have caught it; a site can + // name the wrong one. An equality the compiler checks is what makes the + // half-configured arm unspellable, and it costs the several hundred STARK + // call sites nothing, because they are not generic over `H`. + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, { for root in committed.roots() { transcript.append_bytes(root); @@ -887,7 +906,7 @@ where .iter() .flat_map(|t| t.trace.columns()) .collect(); - columns.push(stacked_eval::prove::( + columns.push(stacked_eval::prove::( group, &group_columns, committed.store.as_ref().map(|store| (&**store, column_at)), @@ -916,7 +935,7 @@ where /// bus carrying the program's public output — `expected` is zero exactly when /// the program outputs nothing. #[allow(clippy::too_many_arguments)] -pub fn multi_verify( +pub fn multi_verify( proof: &MultiProof, statements: &[TableStatement<'_, F, E>], layouts: &[StackedLayout], @@ -929,9 +948,25 @@ pub fn multi_verify( where F: IsFFTField + IsPrimeField + IsSubFieldOf + Send + Sync + 'static, E: IsField + Send + Sync + 'static, + H: WhirHash, FieldElement: AsBytes + Sync + Send, FieldElement: AsBytes + Sync + Send, - T: crypto::fiat_shamir::is_transcript::IsTranscript, + // ★★ The transcript's hash must BE the configuration's. Not "should": a + // caller that passes a keccak transcript under an RPX `H` does not compile. + // + // `DefaultTranscript`'s hash parameter has a default, so `DefaultTranscript::` + // is a keccak transcript that looks like it names no hash. Every call site + // wrote exactly that, and the RPX configuration therefore ran an RPX Merkle + // backend, an RPX grind and a KECCAK sponge — through four measured A/Bs, + // with no instrument disagreeing, because nothing failed: the proofs were + // valid and the arms did differ from each other. + // + // Requiring each site to NAME a hash would not have caught it; a site can + // name the wrong one. An equality the compiler checks is what makes the + // half-configured arm unspellable, and it costs the several hundred STARK + // call sites nothing, because they are not generic over `H`. + T: crypto::fiat_shamir::is_transcript::IsTranscript + + crypto::fiat_shamir::transcript_hash::HasTranscriptHash::Transcript>, { if proof.tables.len() != statements.len() { return Err(MlError::QueryCountMismatch { @@ -990,7 +1025,7 @@ where expected: root_at + layout.num_polys(), got: proof.roots.len(), })?; - stacked_eval::verify::( + stacked_eval::verify::( opening, layout, roots, @@ -1173,7 +1208,7 @@ mod tests { mul_cols: Vec>, ) -> Result<(), MlError> { let (cpu_air, add_air, mul_air) = airs(); - let committed = CommittedTables::commit( + let committed = CommittedTables::<_, _, KeccakWhir>::commit( vec![ table(&cpu_air, &cpu_cols)?, table(&add_air, &add_cols)?, @@ -1203,7 +1238,7 @@ mod tests { committed.tables().iter().map(|t| t.statement()).collect(); let mut verifier = DefaultTranscript::::new(b"multilinear-table"); - multi_verify( + multi_verify::<_, _, _, KeccakWhir>( &proof, &statements, std::slice::from_ref(committed.groups()[0].layout()), diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index faf512a72..4829149b0 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -2286,9 +2286,11 @@ pub trait IsStarkProver< let grinding_factor = air.context().proof_options.grinding_factor; let mut nonce = None; if grinding_factor > 0 { - let nonce_value = - grinding::generate_nonce_maybe_gpu(&transcript.state(), grinding_factor) - .expect("nonce not found"); + let nonce_value = grinding::generate_nonce_maybe_gpu::( + &transcript.state(), + grinding_factor, + ) + .expect("nonce not found"); transcript.append_bytes(&nonce_value.to_be_bytes()); nonce = Some(nonce_value); } diff --git a/crypto/stark/src/tests/grinding_tests.rs b/crypto/stark/src/tests/grinding_tests.rs index 49c47e81f..5296a664e 100644 --- a/crypto/stark/src/tests/grinding_tests.rs +++ b/crypto/stark/src/tests/grinding_tests.rs @@ -1,4 +1,4 @@ -use crate::grinding::is_valid_nonce; +use crate::grinding::{StarkGrindingDigest as GrindDigest, is_valid_nonce}; #[test] fn test_invalid_nonce_grinding_factor_6() { @@ -10,7 +10,11 @@ fn test_invalid_nonce_grinding_factor_6() { ]; let nonce = 4; let grinding_factor = 6; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::( + &seed, + nonce, + grinding_factor + )); } #[test] @@ -23,7 +27,11 @@ fn test_invalid_nonce_grinding_factor_9() { ]; let nonce = 287; let grinding_factor = 9; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::( + &seed, + nonce, + grinding_factor + )); } #[test] @@ -34,7 +42,7 @@ fn test_is_valid_nonce_grinding_factor_10() { ]; let nonce = 0x5ba; let grinding_factor = 10; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -45,7 +53,7 @@ fn test_is_valid_nonce_grinding_factor_20() { ]; let nonce = 0x2c5db8; let grinding_factor = 20; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -59,7 +67,11 @@ fn test_invalid_nonce_grinding_factor_19() { ]; let nonce = 0x2c5db8; let grinding_factor = 19; - assert!(!is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(!is_valid_nonce::( + &seed, + nonce, + grinding_factor + )); } #[test] @@ -70,7 +82,7 @@ fn test_is_valid_nonce_grinding_factor_30() { ]; let nonce = 0x1ae839e1; let grinding_factor = 30; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } #[test] @@ -81,5 +93,5 @@ fn test_is_valid_nonce_grinding_factor_33() { ]; let nonce = 0x4cc3123f; let grinding_factor = 33; - assert!(is_valid_nonce(&seed, nonce, grinding_factor)); + assert!(is_valid_nonce::(&seed, nonce, grinding_factor)); } diff --git a/crypto/stark/src/verifier.rs b/crypto/stark/src/verifier.rs index 44add9c21..6b55fc545 100644 --- a/crypto/stark/src/verifier.rs +++ b/crypto/stark/src/verifier.rs @@ -1665,7 +1665,11 @@ pub trait IsStarkVerifier< let grinding_factor = air.context().proof_options.grinding_factor; if grinding_factor > 0 { let nonce_is_valid = proof.nonce().is_some_and(|nonce_value| { - grinding::is_valid_nonce(&challenges.grinding_seed, nonce_value, grinding_factor) + grinding::is_valid_nonce::( + &challenges.grinding_seed, + nonce_value, + grinding_factor, + ) }); if !nonce_is_valid { diff --git a/prover/Cargo.toml b/prover/Cargo.toml index cf7b0b6b2..61a91fe34 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -48,6 +48,11 @@ serde_json = { version = "1.0", optional = true } [dev-dependencies] env_logger = "*" +# Test-only: the transcript-count pin in `multilinear_bench_tests` guards on the +# guest ELF's sha256, because two builds of the same guest from the same sources +# share a name and differ in bytes — which is how a 0.23% count difference was +# once read as a model error. +sha2 = { version = "0.10", default-features = false } # Test-only: `serde_json` is behind `shape-profile` for the guest's sake, and a # proof round-trip test should not have to turn that on. serde_json = "1.0" diff --git a/prover/src/lib.rs b/prover/src/lib.rs index 7aeec23ef..7841a0f97 100644 --- a/prover/src/lib.rs +++ b/prover/src/lib.rs @@ -30,6 +30,8 @@ pub mod tables; pub mod test_utils; #[cfg(test)] pub mod tests; +pub mod whir_hash_knob; +pub mod whir_identity; use std::fmt; use std::sync::Arc; diff --git a/prover/src/multilinear_continuation.rs b/prover/src/multilinear_continuation.rs index 80eb24934..5382fc7d0 100644 --- a/prover/src/multilinear_continuation.rs +++ b/prover/src/multilinear_continuation.rs @@ -45,7 +45,7 @@ use crate::{Error, TableCounts}; /// /// Distinct from both the univariate epoch tag and the monolithic multilinear /// one: no two of the three may ever share a transcript prefix. -const MULTILINEAR_EPOCH_TAG: &[u8] = b"LAMBDAVM_MULTILINEAR_CONTINUATION_EPOCH_V1"; +pub(crate) const MULTILINEAR_EPOCH_TAG: &[u8] = b"LAMBDAVM_MULTILINEAR_CONTINUATION_EPOCH_V1"; /// One epoch's proof and everything a standalone verifier re-binds. /// @@ -108,14 +108,17 @@ pub fn l2g_commitment( )]; let layout = multilinear_table::global_layout(&shape).map_err(|e| Error::Prover(format!("{e:?}")))?; - let stacked = multilinear::stacked_eval::StackedCommitment::::commit( - layout, - &multilinear::stacking::borrow(&columns), - None, - config, - ) - .map_err(|e| Error::Prover(format!("{e:?}")))?; - let roots = stacked.roots().to_vec(); + let roots = crate::with_whir_hash!(|H| { + multilinear::stacked_eval::StackedCommitment::::commit( + layout, + &multilinear::stacking::borrow(&columns), + None, + config, + ) + .map_err(|e| Error::Prover(format!("{e:?}")))? + .roots() + .to_vec() + }); if roots.is_empty() { return Err(Error::Prover("the bookend commits to nothing".to_string())); } @@ -127,8 +130,14 @@ pub fn l2g_commitment( /// The monolithic multilinear statement plus the epoch's position. A /// continuation epoch never has private-input pages (the bookend replaces /// PAGE), so that count is not stated — it is zero by construction. -fn absorb_epoch( - t: &mut DefaultTranscript, +/// +/// ★ The length is accumulated beside the absorbs, never written as a constant: +/// a `FIXED` the caller has to keep in step is the same class of defect as the +/// pad this function exists to compute. See +/// [`statement::absorb_statement_padding`] for why the roots that follow have to +/// start on a field element boundary and why the pad cannot be a literal. +pub(crate) fn absorb_epoch( + t: &mut impl crypto::fiat_shamir::is_transcript::IsTranscript, elf_digest: &[u8; 32], public_output: &[u8], table_counts: &TableCounts, @@ -136,17 +145,26 @@ fn absorb_epoch( table_num_vars: &[u8], config: &ChainConfig, ) { + let mut len = 0usize; + t.append_bytes(MULTILINEAR_EPOCH_TAG); + len += MULTILINEAR_EPOCH_TAG.len(); t.append_bytes(elf_digest); + len += elf_digest.len(); t.append_bytes(&epoch_label.to_le_bytes()); + len += size_of_val(&epoch_label); t.append_bytes(&(public_output.len() as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(public_output); + len += public_output.len(); - statement::absorb_table_counts(t, table_counts); + len += statement::absorb_table_counts(t, table_counts); t.append_bytes(&(table_num_vars.len() as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(table_num_vars); + len += table_num_vars.len(); let &ChainConfig { log_blowup, @@ -156,8 +174,21 @@ fn absorb_epoch( } = config; for value in [log_blowup as u64, log_folding as u64, num_queries as u64] { t.append_bytes(&value.to_le_bytes()); + len += size_of_val(&value); } - t.append_bytes(&[grind.folding, grind.ood, grind.query]); + let trailer = [grind.folding, grind.ood, grind.query]; + t.append_bytes(&trailer); + len += trailer.len(); + + statement::absorb_statement_padding( + t, + "epoch", + len, + &[ + ("public_output", public_output.len()), + ("table_num_vars", table_num_vars.len()), + ], + ); } /// How an epoch's tables are split across commitments: everything together, @@ -189,11 +220,11 @@ fn layout_of<'a>( /// What the epoch's tables owe the statement: the COMMIT bus's counterparty, /// counted from the commit index this epoch carried in. -fn owed( +fn owed( public_output: &[u8], register_init: &[u32], roots: &[Commitment], - transcript: &DefaultTranscript, + transcript: &DefaultTranscript, ) -> Option> { let start_index = *register_init.get(register::X254_INDEX)? as u64; let mut probe = transcript.clone(); @@ -206,7 +237,7 @@ fn owed( } /// Domain tag for the multilinear cross-epoch proof. -const MULTILINEAR_GLOBAL_TAG: &[u8] = b"LAMBDAVM_MULTILINEAR_CONTINUATION_GLOBAL_V1"; +pub(crate) const MULTILINEAR_GLOBAL_TAG: &[u8] = b"LAMBDAVM_MULTILINEAR_CONTINUATION_GLOBAL_V1"; /// The one cross-epoch proof: every epoch's bookend and the global-memory /// tables, in one transcript. @@ -242,8 +273,14 @@ impl GlobalProof { } /// Binds the cross-epoch statement: what the run was, not what any epoch was. -fn absorb_global( - t: &mut DefaultTranscript, +/// +/// ⚠ This statement is padded for the same reason the epoch statement is, and +/// it needs it for the same reason: `table_num_vars` is one byte per table +/// (every epoch's bookend plus the global-memory tables), so it is a +/// variable-length field sitting between the fixed prefix and the roots. +/// `page_bases` is eight bytes an entry and does not move the alignment. +pub(crate) fn absorb_global( + t: &mut impl crypto::fiat_shamir::is_transcript::IsTranscript, elf_digest: &[u8; 32], num_epochs: usize, num_private_input_pages: usize, @@ -251,16 +288,26 @@ fn absorb_global( table_num_vars: &[u8], config: &ChainConfig, ) { + let mut len = 0usize; + t.append_bytes(MULTILINEAR_GLOBAL_TAG); + len += MULTILINEAR_GLOBAL_TAG.len(); t.append_bytes(elf_digest); + len += elf_digest.len(); t.append_bytes(&(num_epochs as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(&(num_private_input_pages as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(&(page_bases.len() as u64).to_le_bytes()); + len += size_of::(); for base in page_bases { t.append_bytes(&base.to_le_bytes()); + len += size_of_val(base); } t.append_bytes(&(table_num_vars.len() as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(table_num_vars); + len += table_num_vars.len(); let &ChainConfig { log_blowup, log_folding, @@ -269,8 +316,21 @@ fn absorb_global( } = config; for value in [log_blowup as u64, log_folding as u64, num_queries as u64] { t.append_bytes(&value.to_le_bytes()); + len += size_of_val(&value); } - t.append_bytes(&[grind.folding, grind.ood, grind.query]); + let trailer = [grind.folding, grind.ood, grind.query]; + t.append_bytes(&trailer); + len += trailer.len(); + + statement::absorb_statement_padding( + t, + "global", + len, + &[ + ("page_bases", page_bases.len()), + ("table_num_vars", table_num_vars.len()), + ], + ); } /// How the cross-epoch proof's tables are split: every bookend alone — so its @@ -355,17 +415,6 @@ pub fn prove_global( let table_num_vars: Vec = shapes.iter().map(|&(_, n)| n as u8).collect(); let config = chain_config(&shapes); - let mut transcript = DefaultTranscript::::new(&[]); - absorb_global( - &mut transcript, - &statement::elf_digest(elf_bytes), - boundaries.len(), - num_private_input_pages, - page_bases, - &table_num_vars, - &config, - ); - let mut committed = Vec::with_capacity(pairs.len()); for ((air, trace, _), &(width, num_vars)) in pairs.iter_mut().zip(&shapes) { let layout = layout_of(*air, width, num_vars) @@ -385,10 +434,26 @@ pub fn prove_global( ); } let sizes = global_groups(boundaries.len(), gm_configs.len()); - let committed = CommittedTables::commit_grouped(committed, &sizes, &config) - .map_err(|e| Error::Prover(format!("{e:?}")))?; - let proof = multilinear_table::multi_prove(&committed, &config, &mut transcript) - .map_err(|e| Error::Prover(format!("{e:?}")))?; + let proof = crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb_global( + &mut transcript, + &statement::elf_digest(elf_bytes), + boundaries.len(), + num_private_input_pages, + page_bases, + &table_num_vars, + &config, + ); + let committed = CommittedTables::<_, _, H>::commit_grouped(committed, &sizes, &config) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + multilinear_table::multi_prove(&committed, &config, &mut transcript) + .map_err(|e| Error::Prover(format!("{e:?}")))? + }); Ok(GlobalProof { proof, @@ -468,17 +533,6 @@ fn verify_global_bookends( .collect(); let config = chain_config(&shapes); - let mut transcript = DefaultTranscript::::new(&[]); - absorb_global( - &mut transcript, - &statement::elf_digest(elf_bytes), - num_epochs, - num_private_input_pages, - page_bases, - &global.table_num_vars, - &config, - ); - let layouts: Vec> = air_refs .iter() .zip(&shapes) @@ -508,18 +562,33 @@ fn verify_global_bookends( let polys: Vec = stacks[..num_epochs].iter().map(|l| l.num_polys()).collect(); // The cross-epoch bus has no counterparty in the statement: it must vanish. - if multilinear_table::multi_verify( - &global.proof, - &statements, - &stacks, - &domains, - &sizes, - &FieldElement::::zero(), - &config, - &mut transcript, - ) - .is_err() - { + let verdict = crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb_global( + &mut transcript, + &statement::elf_digest(elf_bytes), + num_epochs, + num_private_input_pages, + page_bases, + &global.table_num_vars, + &config, + ); + multilinear_table::multi_verify::<_, _, _, H>( + &global.proof, + &statements, + &stacks, + &domains, + &sizes, + &FieldElement::::zero(), + &config, + &mut transcript, + ) + }); + if verdict.is_err() { return Ok(None); } Ok(global @@ -585,17 +654,6 @@ pub fn prove_epoch( let table_num_vars: Vec = shapes.iter().map(|&(_, n)| n as u8).collect(); let config = chain_config(&shapes); - let mut transcript = DefaultTranscript::::new(&[]); - absorb_epoch( - &mut transcript, - &statement::elf_digest(elf_bytes), - &public_output, - &table_counts, - label, - &table_num_vars, - &config, - ); - let mut committed = Vec::with_capacity(pairs.len()); for ((air, trace, _), &(width, num_vars)) in pairs.iter_mut().zip(&shapes) { let layout = layout_of(*air, width, num_vars) @@ -617,10 +675,26 @@ pub fn prove_epoch( ); } let sizes = epoch_groups(committed.len()); - let committed = CommittedTables::commit_grouped(committed, &sizes, &config) - .map_err(|e| Error::Prover(format!("{e:?}")))?; - let proof = multilinear_table::multi_prove(&committed, &config, &mut transcript) - .map_err(|e| Error::Prover(format!("{e:?}")))?; + let proof = crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb_epoch( + &mut transcript, + &statement::elf_digest(elf_bytes), + &public_output, + &table_counts, + label, + &table_num_vars, + &config, + ); + let committed = CommittedTables::<_, _, H>::commit_grouped(committed, &sizes, &config) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + multilinear_table::multi_prove(&committed, &config, &mut transcript) + .map_err(|e| Error::Prover(format!("{e:?}")))? + }); Ok(EpochProof { proof, @@ -913,17 +987,6 @@ fn verify_epoch_bookend( .collect(); let config = chain_config(&shapes); - let mut transcript = DefaultTranscript::::new(&[]); - absorb_epoch( - &mut transcript, - &statement::elf_digest(elf_bytes), - &epoch.public_output, - &epoch.table_counts, - label, - &epoch.table_num_vars, - &config, - ); - let layouts: Vec> = air_refs .iter() .zip(&shapes) @@ -946,33 +1009,51 @@ fn verify_epoch_bookend( .map(|(layout, cols)| layout.statement_with_preprocessed(cols)) .collect(); - let Some(owed) = owed( - &epoch.public_output, - register_init, - &epoch.proof.roots, - &transcript, - ) else { - return Ok(None); - }; - let sizes = epoch_groups(shapes.len()); let (layouts, domains) = crate::multilinear_prove::stacks(&shapes, &sizes, &config)?; // The bookend is committed in the last group, alone, so its roots are that // group's — as many as the stack split it into. let num_polys = layouts.last().map(|l| l.num_polys()).unwrap_or(0); - if multilinear_table::multi_verify( - &epoch.proof, - &statements, - &layouts, - &domains, - &sizes, - &owed, - &config, - &mut transcript, - ) - .is_err() - { + let verdict = crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb_epoch( + &mut transcript, + &statement::elf_digest(elf_bytes), + &epoch.public_output, + &epoch.table_counts, + label, + &epoch.table_num_vars, + &config, + ); + // ★ Inside the dispatch with the transcript it forks: `owed` replays + // this transcript to draw `z` and `alpha`, which are a function of the + // configuration's sponge. Computing them against a transcript of a + // different hash is the same defect one level down, and just as quiet. + let Some(owed) = owed( + &epoch.public_output, + register_init, + &epoch.proof.roots, + &transcript, + ) else { + return Ok(None); + }; + multilinear_table::multi_verify::<_, _, _, H>( + &epoch.proof, + &statements, + &layouts, + &domains, + &sizes, + &owed, + &config, + &mut transcript, + ) + }); + if verdict.is_err() { return Ok(None); } Ok(epoch.l2g_roots(num_polys).map(<[_]>::to_vec)) diff --git a/prover/src/multilinear_prove.rs b/prover/src/multilinear_prove.rs index 9a71eca8f..54ef6621b 100644 --- a/prover/src/multilinear_prove.rs +++ b/prover/src/multilinear_prove.rs @@ -98,9 +98,15 @@ pub fn chain_config(shapes: &[Shape]) -> ChainConfig { /// The univariate encoding, under a tag of its own — a WHIR proof and a FRI /// proof must never share a transcript prefix — followed by what only this path /// states: the table heights and the parameters the argument runs at. +/// +/// ⚠ Padded like the continuation statements, and for the same reason: the +/// roots absorbed next have to start on a field element boundary, and two +/// variable-length fields (`public_output`, `table_num_vars`) sit between them +/// and the fixed prefix. `runtime_page_ranges` is sixteen bytes an entry and +/// does not move the alignment. #[allow(clippy::too_many_arguments)] -fn absorb( - t: &mut DefaultTranscript, +pub(crate) fn absorb( + t: &mut impl crypto::fiat_shamir::is_transcript::IsTranscript, elf_digest: &[u8; 32], public_output: &[u8], table_counts: &TableCounts, @@ -109,26 +115,38 @@ fn absorb( table_num_vars: &[u8], config: &ChainConfig, ) { + let mut len = 0usize; + t.append_bytes(MULTILINEAR_TAG); + len += MULTILINEAR_TAG.len(); t.append_bytes(elf_digest); + len += elf_digest.len(); t.append_bytes(&(public_output.len() as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(public_output); + len += public_output.len(); - statement::absorb_table_counts(t, table_counts); + len += statement::absorb_table_counts(t, table_counts); t.append_bytes(&(num_private_input_pages as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(&(runtime_page_ranges.len() as u64).to_le_bytes()); + len += size_of::(); for r in runtime_page_ranges { let &RuntimePageRange { base, count } = r; t.append_bytes(&base.to_le_bytes()); + len += size_of_val(&base); t.append_bytes(&count.to_le_bytes()); + len += size_of_val(&count); } // Every table's height. A prover who shrank a table would have to state the // smaller height here, which moves every challenge. t.append_bytes(&(table_num_vars.len() as u64).to_le_bytes()); + len += size_of::(); t.append_bytes(table_num_vars); + len += table_num_vars.len(); // The parameters the argument runs at: derived from the heights above, but // absorbed rather than assumed, so the two sides agree in the transcript and @@ -141,8 +159,22 @@ fn absorb( } = config; for value in [log_blowup as u64, log_folding as u64, num_queries as u64] { t.append_bytes(&value.to_le_bytes()); + len += size_of_val(&value); } - t.append_bytes(&[grind.folding, grind.ood, grind.query]); + let trailer = [grind.folding, grind.ood, grind.query]; + t.append_bytes(&trailer); + len += trailer.len(); + + statement::absorb_statement_padding( + t, + "monolithic", + len, + &[ + ("public_output", public_output.len()), + ("runtime_page_ranges", runtime_page_ranges.len()), + ("table_num_vars", table_num_vars.len()), + ], + ); } /// Every table's `(main width, height in variables)`, checked to be what the @@ -223,18 +255,6 @@ pub fn prove_with_options_and_inputs( let table_num_vars: Vec = shapes.iter().map(|&(_, n)| n as u8).collect(); let config = chain_config(&shapes); - let mut transcript = DefaultTranscript::::new(&[]); - absorb( - &mut transcript, - &statement::elf_digest(elf_bytes), - &public_output, - &table_counts, - num_private_input_pages, - &runtime_page_ranges, - &table_num_vars, - &config, - ); - // Commit every table against the layout the verifier will rebuild. let mut committed = Vec::with_capacity(pairs.len()); for ((air, trace, _), &(width, num_vars)) in pairs.iter_mut().zip(&shapes) { @@ -262,10 +282,27 @@ pub fn prove_with_options_and_inputs( // One commitment for every table in the proof: the opening is nearly all of // a proof's bytes, and one settles them all. - let committed = - CommittedTables::commit(committed, &config).map_err(|e| Error::Prover(format!("{e:?}")))?; - let proof = multilinear_table::multi_prove(&committed, &config, &mut transcript) - .map_err(|e| Error::Prover(format!("{e:?}")))?; + let proof = crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb( + &mut transcript, + &statement::elf_digest(elf_bytes), + &public_output, + &table_counts, + num_private_input_pages, + &runtime_page_ranges, + &table_num_vars, + &config, + ); + let committed = CommittedTables::<_, _, H>::commit(committed, &config) + .map_err(|e| Error::Prover(format!("{e:?}")))?; + multilinear_table::multi_prove(&committed, &config, &mut transcript) + .map_err(|e| Error::Prover(format!("{e:?}")))? + }); Ok(MultilinearVmProof { proof, @@ -410,18 +447,6 @@ pub fn verify_with_options( .collect(); let config = chain_config(&shapes); - let mut transcript = DefaultTranscript::::new(&[]); - absorb( - &mut transcript, - &statement::elf_digest(elf_bytes), - &proof.public_output, - &proof.table_counts, - proof.num_private_input_pages, - &proof.runtime_page_ranges, - &proof.table_num_vars, - &config, - ); - let layouts: Vec> = air_refs .iter() .zip(&shapes) @@ -443,34 +468,56 @@ pub fn verify_with_options( .map(|(layout, cols)| layout.statement_with_preprocessed(cols)) .collect(); - // What the tables owe: the COMMIT bus's counterparty is the statement, and - // its offset depends on the very challenges `multi_verify` is about to draw - // — so the transcript is replayed to that point on a fork. - let mut probe = transcript.clone(); - for root in &proof.proof.roots { - probe.append_bytes(root); - } - let z: FieldElement = probe.sample_field_element(); - let alpha: FieldElement = probe.sample_field_element(); - // `start_index` is the carried x254: zero for a monolithic proof. - let Some(owed) = crate::compute_commit_bus_offset(&proof.public_output, 0, &z, &alpha) else { - return Ok(false); - }; - // The stack every table's columns share, rebuilt from the shapes alone. A // monolithic proof commits them all together, so there is one group. let sizes = [shapes.len()]; let (layouts, domains) = stacks(&shapes, &sizes, &config)?; - Ok(multilinear_table::multi_verify( - &proof.proof, - &statements, - &layouts, - &domains, - &sizes, - &owed, - &config, - &mut transcript, - ) - .is_ok()) + Ok(crate::with_whir_hash!(|H| { + // ★ Inside the dispatch, because the transcript's hash is part of + // the configuration and `H` does not exist outside this block. The + // bound on `multi_prove`/`multi_verify` rejects any other spelling. + let mut transcript = + DefaultTranscript::::Transcript>::new(&[]); + absorb( + &mut transcript, + &statement::elf_digest(elf_bytes), + &proof.public_output, + &proof.table_counts, + proof.num_private_input_pages, + &proof.runtime_page_ranges, + &proof.table_num_vars, + &config, + ); + // What the tables owe: the COMMIT bus's counterparty is the statement, + // and its offset depends on the very challenges `multi_verify` is about + // to draw — so the transcript is replayed to that point on a fork. + // + // ★ Inside the dispatch with the transcript it forks. Those challenges + // are a function of the configuration's sponge, so computing them + // against a transcript of a different hash is the same defect one level + // down, and just as quiet. + let mut probe = transcript.clone(); + for root in &proof.proof.roots { + probe.append_bytes(root); + } + let z: FieldElement = probe.sample_field_element(); + let alpha: FieldElement = probe.sample_field_element(); + // `start_index` is the carried x254: zero for a monolithic proof. + let Some(owed) = crate::compute_commit_bus_offset(&proof.public_output, 0, &z, &alpha) + else { + return Ok(false); + }; + multilinear_table::multi_verify::<_, _, _, H>( + &proof.proof, + &statements, + &layouts, + &domains, + &sizes, + &owed, + &config, + &mut transcript, + ) + .is_ok() + })) } diff --git a/prover/src/statement.rs b/prover/src/statement.rs index dca2619a9..a1bf09963 100644 --- a/prover/src/statement.rs +++ b/prover/src/statement.rs @@ -67,6 +67,13 @@ pub(crate) fn absorb_statement( /// hold the digest reuse it instead of a second full-ELF Keccak pass — the /// recursion attestation path shares one digest between the transcript absorb /// and the `program_id` fold (a full-ELF hash is expensive in-guest). +/// +/// ⛔ **This statement is NOT padded to a field element boundary**, unlike the +/// three WHIR ones — see [`absorb_statement_padding`]. Its transcript is +/// keccak, which absorbs a byte stream and never re-slices it into field +/// elements, so there is no straddling value to prevent; and its bytes are the +/// univariate pipeline's, which the block identity lines pin. Padding here +/// would move a record for nothing. #[allow(clippy::too_many_arguments)] pub(crate) fn absorb_statement_with_digest( t: &mut impl IsTranscript, @@ -93,7 +100,8 @@ pub(crate) fn absorb_statement_with_digest( t.append_bytes(&(public_output.len() as u64).to_le_bytes()); t.append_bytes(public_output); - absorb_table_counts(t, table_counts); + // The width is for callers that pad from it; this one does not. + let _ = absorb_table_counts(t, table_counts); t.append_bytes(&(num_private_input_pages as u64).to_le_bytes()); @@ -117,12 +125,105 @@ pub(crate) fn absorb_statement_with_digest( } } -/// The table layout, as fixed-width u64s in declared order. +/// Bytes per field element, which is the granularity the algebraic transcript +/// slices its absorbed buffer at (`rpx::sponge_leaf_bytes`). +pub(crate) const FELT_BYTES: usize = 8; + +/// Zeroes for [`absorb_statement_padding`]. A pad is at most `FELT_BYTES - 1`. +const PAD_ZEROS: [u8; FELT_BYTES - 1] = [0u8; FELT_BYTES - 1]; + +/// Zero bytes that bring a statement of `len` bytes up to a multiple of +/// [`FELT_BYTES`]. +pub(crate) const fn statement_padding(len: usize) -> usize { + (FELT_BYTES - len % FELT_BYTES) % FELT_BYTES +} + +/// Closes a WHIR statement so that whatever is absorbed next starts on a field +/// element boundary, and returns the number of bytes it added. +/// +/// # Why a statement is padded at all +/// +/// The transcript hashes BYTES, and the algebraic configuration's sponge +/// re-slices everything absorbed since the last squeeze into field elements +/// every [`FELT_BYTES`] bytes (`crypto::hash::rpx::sponge_leaf_bytes`). A value +/// absorbed at an offset that is not a multiple of 8 therefore STRADDLES two +/// field elements, and a field-machine verifier replaying the transcript has to +/// bit-decompose it to reproduce the absorb. +/// +/// Every window after the first is already aligned: a squeeze leaves the buffer +/// holding its own 32-byte output, and everything absorbed afterwards — roots +/// 32, extension elements 24, grind nonces 8, final values 24 — is a multiple of +/// 8. The FIRST window is the exception, because it opens with the statement, +/// and the roots that follow it land wherever the statement ended. +/// +/// # Why the pad is COMPUTED and not a constant +/// +/// A statement's roots do not sit at the end of its fixed prefix: two +/// variable-length fields sit in between (the public output and the per-table +/// heights). Padding the fixed prefix to a multiple of 8 would leave the roots +/// at `(|public_output| + |table_num_vars|) mod 8` — 2 mod 8 at the shape this +/// system runs — so it would align nothing while moving every pinned constant. +/// The length is accumulated beside the absorbs that produce it, and the pad +/// follows from that length. +/// +/// # Why it is called even when the pad is empty +/// +/// So that "one padding absorb per statement" holds for every shape. The +/// transcript counts an empty `append_bytes` as an absorb, so an absorb count +/// stays a function of the statement's FIELDS rather than of its lengths, and +/// the pinned pair moves by exactly one per statement instead of by a number +/// nobody can predict without the shapes. +pub(crate) fn absorb_statement_padding( + t: &mut impl IsTranscript, + kind: &str, + len: usize, + shape: &[(&str, usize)], +) -> usize { + #[cfg(not(feature = "hash-metrics"))] + let _ = (kind, shape); + + let pad = statement_padding(len); + t.append_bytes(&PAD_ZEROS[..pad]); + + // A diagnostic, not a gate: it prints every variable length the pad is a + // function of beside the pad itself, so a run that reports a total number of + // padding bytes can be checked against the shapes that produced it instead + // of against a premise nobody measured. + #[cfg(feature = "hash-metrics")] + { + // ⚠ Its own label, not `WHIR`: the bench's per-arm lines already start + // with that, and a box launcher counting `WHIR` lines would silently + // pick these up as well. + let mut line = format!("{:<12} statement {kind}", "WHIR-PAD"); + for (name, value) in shape { + line.push_str(&format!(" {name}={value}")); + } + println!("{line} len={len} pad={pad}"); + } + + pad +} + +/// How many per-table counts a statement binds — one `u64` each. +/// +/// ★ Read, never written as a literal by a caller. The transcript pin's +/// expected absorb counts are `base + epochs * NUM_TABLE_KINDS`, because the +/// per-table campaign adds a count to this list (`TableCounts::blake3`) and a +/// pin carrying `14` would then be a constant describing one branch while +/// claiming to describe the protocol. +/// +/// Two compile errors guard it together, and neither alone is enough: the +/// exhaustive destructure in [`table_count_values`] fails when a field is added +/// to [`TableCounts`], and that function's return type fails when the new field +/// is pushed into the array without bumping this constant. +pub(crate) const NUM_TABLE_KINDS: usize = 14; + +/// Every per-table count, in declared order. /// /// The exhaustive destructure makes any field added to [`TableCounts`] a -/// compile error here — that's the signal to extend the loop and bump the +/// compile error here — that's the signal to extend the array and bump the /// domain tag of every statement that absorbs it. -pub(crate) fn absorb_table_counts(t: &mut impl IsTranscript, table_counts: &TableCounts) { +pub(crate) fn table_count_values(table_counts: &TableCounts) -> [u64; NUM_TABLE_KINDS] { let &TableCounts { cpu, lt, @@ -139,24 +240,39 @@ pub(crate) fn absorb_table_counts(t: &mut impl IsTranscript, table_counts: &T store, cpu32, } = table_counts; - for count in [ - cpu, - lt, - memw, - memw_aligned, - load, - mul, - dvrm, - shift, - branch, - memw_register, - eq, - bytewise, - store, - cpu32, - ] { - t.append_bytes(&(count as u64).to_le_bytes()); + [ + cpu as u64, + lt as u64, + memw as u64, + memw_aligned as u64, + load as u64, + mul as u64, + dvrm as u64, + shift as u64, + branch as u64, + memw_register as u64, + eq as u64, + bytewise as u64, + store as u64, + cpu32 as u64, + ] +} + +/// The table layout, as fixed-width u64s in declared order. +/// +/// Returns the number of BYTES it absorbed, so a caller accumulating a +/// statement's length does not have to know — or track — how many counts there +/// are. A caller that does not need the length (the univariate path) ignores it. +#[must_use] +pub(crate) fn absorb_table_counts( + t: &mut impl IsTranscript, + table_counts: &TableCounts, +) -> usize { + let counts = table_count_values(table_counts); + for count in counts { + t.append_bytes(&count.to_le_bytes()); } + counts.len() * size_of::() } /// Domain tag for the multilinear path. A WHIR proof and a FRI proof must diff --git a/prover/src/tables/decode.rs b/prover/src/tables/decode.rs index 8d31fb1fa..ba34e2d70 100644 --- a/prover/src/tables/decode.rs +++ b/prover/src/tables/decode.rs @@ -100,13 +100,38 @@ pub type PcToRow = U64HashMap; pub fn generate_decode_trace( instructions: &U64HashMap, ) -> (TraceTable, PcToRow) { - // Build entries and PC-to-row mapping + // ★★ ROWS GO IN PC ORDER, and the sort is the whole point of this block. + // + // The rows used to come out of `instructions.iter()`, so their order was + // hashbrown's: a function of the hasher, the capacity the map happened to + // grow to, and the insertion sequence. Every one of those is stable for a + // given binary, which is why nothing has ever failed — prover and verifier + // both reach this through `instructions_from_elf`, so they agree with each + // other. What they agree on is a CONSTRUCTION PROCEDURE, not the ELF. + // + // That distinction is about to start mattering. These five columns are + // ELF-derived and their Merkle root is on its way to being a program + // constant pinned in a recursion guest (W1-B). A pinned root must be a + // function of the ELF ALONE: sorted by pc it is, and a hashbrown version + // bump, a capacity change or a reserve added upstream cannot move it. + // Unsorted it is not, and the failure mode is the bad kind — a toolchain + // update silently invalidates the pin, with no ELF change, no code change + // and no test that fails until a verifier rejects a valid proof. + // + // pc is unique (it is the map's key), so the order is total and the sort is + // not merely deterministic but canonical. + let mut entries: Vec<(u64, Instruction)> = instructions + .iter() + .map(|(&pc, &instr)| (pc, instr)) + .collect(); + entries.sort_unstable_by_key(|(pc, _)| *pc); + let mut pc_to_row = PcToRow::default(); pc_to_row.reserve(instructions.len() + 1); - let entries: Vec<_> = instructions - .iter() + let entries: Vec<_> = entries + .into_iter() .enumerate() - .map(|(row_idx, (&pc, &instr))| { + .map(|(row_idx, (pc, instr))| { pc_to_row.insert(pc, row_idx); // instruction_length = 4 (RV64C compressed decode is a separate workstream). DecodeEntry::from_instruction(pc, instr, 4) diff --git a/prover/src/tests/decode_tests.rs b/prover/src/tests/decode_tests.rs index a761ac929..4580f432d 100644 --- a/prover/src/tests/decode_tests.rs +++ b/prover/src/tests/decode_tests.rs @@ -241,9 +241,21 @@ fn decode_commitment_zero_bytes_rejects() { /// commitment as a compile-time constant for its inner program. If the /// AIR or FFT pipeline changes, this drifts and the test fails — /// regenerate via the `print_decode_commitment_for_sub` helper below. +/// +/// ⚠ RE-BASELINED when DECODE rows moved to pc order. It was +/// `e97168d6…5f`, which was the root of a trace whose row order came out of +/// `instructions.iter()` — hashbrown's, a function of the hasher, the map's +/// capacity and the insertion sequence rather than of the ELF. Under the sort +/// it is a function of the ELF alone, which is what a constant pinned in a +/// guest has to be: see +/// [`the_decode_trace_does_not_depend_on_the_map_that_carried_it`]. +/// +/// That is the whole reason this value moved, and the reason it is the LAST +/// time it can move for a reason nobody chose. A future drift means the AIR or +/// the FFT pipeline changed, which is what this constant was always for. const SUB_DECODE_COMMITMENT_BLOWUP_2: [u8; 32] = [ - 0xe9, 0x71, 0x68, 0xd6, 0x2e, 0xb1, 0xf6, 0x56, 0x61, 0x9d, 0x04, 0x6e, 0x65, 0xed, 0x63, 0x4a, - 0x27, 0xa3, 0x4d, 0xcb, 0x6c, 0x02, 0x11, 0xd7, 0x65, 0xc9, 0xc9, 0xfd, 0x59, 0x34, 0x41, 0x5f, + 0x0a, 0x71, 0x0a, 0x9c, 0x8e, 0xbe, 0x1a, 0xbc, 0x32, 0x6a, 0x3d, 0x33, 0xb2, 0x42, 0x13, 0x9a, + 0x33, 0x0c, 0xcb, 0x19, 0x22, 0xe1, 0xf7, 0xca, 0xb7, 0x67, 0x32, 0x8c, 0xf5, 0xb7, 0x29, 0x1b, ]; #[test] @@ -279,3 +291,125 @@ fn print_decode_commitment_for_sub() { eprintln!("SUB_DECODE_COMMITMENT_BLOWUP_2 (sub.elf, blowup=2):"); eprintln!("{c:02x?}"); } + +// ========================================================================= +// Row order is a function of the ELF alone +// ========================================================================= + +/// A distinct instruction per pc, so a permuted trace cannot match a sorted one +/// by accident — every row differs from every other in PACKED_DECODE and IMM. +fn instr_for(pc: u64) -> Instruction { + Instruction::ArithImm { + dst: ((pc / 4) % 30) as u32 + 1, + src: ((pc / 4) % 7) as u32 + 1, + imm: (pc % 2048) as i32 - 1024, + op: ArithOp::Add, + } +} + +/// The same instruction set, reached through two independently-constructed +/// maps: ascending with no reserve, descending with a large one. +/// +/// Different insertion order and different capacity means a different hashbrown +/// bucket layout, hence a different `iter()` order — which is exactly the +/// variation a hashbrown version bump, an added `reserve`, or a change of hasher +/// would introduce, expressed as something a test can construct today. +fn two_maps_of(n: u64) -> (U64HashMap, U64HashMap) { + let pcs: Vec = (0..n).map(|i| 0x1000 + i * 4).collect(); + + let mut ascending: U64HashMap = U64HashMap::default(); + for &pc in &pcs { + ascending.insert(pc, instr_for(pc)); + } + + let mut descending: U64HashMap = U64HashMap::default(); + descending.reserve(1024); + for &pc in pcs.iter().rev() { + descending.insert(pc, instr_for(pc)); + } + + (ascending, descending) +} + +/// ★★★ THE PROPERTY: the DECODE trace is a function of the instruction SET, +/// not of the map that carried it. +/// +/// Sorting by pc is the mechanism; this is the thing that must be true, and it +/// is stated that way on purpose. A test asserting "the rows are sorted" would +/// pass on any total order and would not say why the order matters — whereas a +/// root pinned as a program constant needs exactly this: two parties holding +/// the same ELF, and nothing else in common, produce the same rows. +/// +/// ⚠ This is what fails without the sort. The two maps differ in insertion +/// order and capacity, so `instructions.iter()` walks them differently and the +/// traces come out permuted. Nothing in the system notices today, because +/// prover and verifier both build their map through `instructions_from_elf` and +/// so make the same arbitrary choice — they agree on a construction procedure +/// rather than on the ELF. A hashbrown bump breaks that agreement with no ELF +/// change and no failing test. +#[test] +fn the_decode_trace_does_not_depend_on_the_map_that_carried_it() { + let (ascending, descending) = two_maps_of(300); + + // The premise: the two maps really are walked differently. If hashbrown ever + // made iteration order insertion- and capacity-independent, this test would + // still pass below while testing nothing, so the premise is asserted. + let order_a: Vec = ascending.iter().map(|(&pc, _)| pc).collect(); + let order_b: Vec = descending.iter().map(|(&pc, _)| pc).collect(); + assert_ne!( + order_a, order_b, + "the two maps iterate identically, so this test cannot detect a \ + map-dependent trace — rebuild the maps so they differ" + ); + + let (trace_a, pc_to_row_a) = generate_decode_trace(&ascending); + let (trace_b, pc_to_row_b) = generate_decode_trace(&descending); + + assert_eq!(trace_a.num_rows(), trace_b.num_rows()); + for row in 0..trace_a.num_rows() { + assert_eq!( + trace_a.main_table.get_row(row), + trace_b.main_table.get_row(row), + "row {row} differs between two maps holding the same instructions" + ); + } + + // …and the index agrees too, or `update_multiplicities` would write the + // right counts to the wrong rows. + for (&pc, &row) in pc_to_row_a.iter() { + assert_eq!( + pc_to_row_b.get(&pc), + Some(&row), + "pc {pc:#x} maps to a different row in the two maps" + ); + } +} + +/// ★ THE MECHANISM, pinned separately so the reason stays visible. +/// +/// The property above holds for any canonical order; this says which one, so a +/// future change that keeps determinism but moves the rows has to come here and +/// re-baseline the pins rather than sliding past. +#[test] +fn decode_rows_are_in_ascending_pc_order() { + let (ascending, _) = two_maps_of(64); + let (trace, _) = generate_decode_trace(&ascending); + + // The instruction rows come first, then the CPU padding row, then zeroed + // padding to the next power of two — so only the first `n` are ordered. + let pcs: Vec = (0..64) + .map(|row| { + let lo = *trace.main_table.get(row, cols::PC_0).value(); + let hi = *trace.main_table.get(row, cols::PC_1).value(); + lo | (hi << 32) + }) + .collect(); + + let mut sorted = pcs.clone(); + sorted.sort_unstable(); + assert_eq!( + pcs, sorted, + "the instruction rows are not in ascending pc order" + ); + assert_eq!(pcs[0], 0x1000, "the first row is not the lowest pc"); +} diff --git a/prover/src/tests/mod.rs b/prover/src/tests/mod.rs index 99e3d1177..288b35936 100644 --- a/prover/src/tests/mod.rs +++ b/prover/src/tests/mod.rs @@ -97,6 +97,8 @@ pub mod shape_profile_tests; pub mod shift_tests; #[cfg(test)] +pub mod statement_alignment_tests; +#[cfg(test)] pub mod statement_tests; #[cfg(test)] pub mod static_commitments_tests; @@ -108,3 +110,9 @@ pub mod templates_tests; pub mod trace_builder_tests; #[cfg(test)] pub mod trace_test_helpers; +#[cfg(test)] +pub mod whir_byte_gate; +#[cfg(test)] +pub mod whir_hash_tests; +#[cfg(test)] +pub mod whir_identity_tests; diff --git a/prover/src/tests/multilinear_bench_tests.rs b/prover/src/tests/multilinear_bench_tests.rs index 0b79d1330..6465f9dd3 100644 --- a/prover/src/tests/multilinear_bench_tests.rs +++ b/prover/src/tests/multilinear_bench_tests.rs @@ -18,6 +18,7 @@ use std::time::Instant; use executor::elf::Elf; use executor::vm::execution::Executor; use multilinear::whir_chain::GrindBits; +use multilinear::whir_hash::KeccakWhir; use stark::proof::options::GoldilocksCubicProofOptions; use crate::multilinear_prove; @@ -109,14 +110,20 @@ fn shapes() { continue; } }; - let mut traces = - match Traces::from_elf_and_logs(&elf, &logs, &MaxRowsConfig::default(), &inputs) { - Ok(traces) => traces, - Err(e) => { - println!("{label:<22} trace failed: {e:?}"); - continue; - } - }; + let mut traces = match Traces::from_elf_and_logs( + &elf, + &logs, + &MaxRowsConfig::default(), + &inputs, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) { + Ok(traces) => traces, + Err(e) => { + println!("{label:<22} trace failed: {e:?}"); + continue; + } + }; let table_counts = traces.table_counts(); let airs = crate::VmAirs::new( &elf, @@ -255,6 +262,531 @@ fn whir_against_fri() { /// `LAMBDA_VM_BENCH_EPOCH_LOG2` is the epoch length in cycles, the CLI's /// default (2^20) unless it is set. It is a resource knob, not a property of /// either prover: both sides get the same one. +/// One transcript-counter line, labelled with the WINDOW it covers. +/// +/// ⚠ The window is part of the number. The prover and the verifier each run a +/// transcript, over different work, and only the verifier's is what a recursive +/// verifier replays — so a count quoted without its side cannot be checked +/// against anything. +/// +/// ⚠ `states` is NOT part of `squeezes`: a squeeze is a `finalize_reset` that +/// chains its output back in, a state read finalizes a CLONE and advances +/// nothing. There is one state read per grind check, so on the verify line the +/// states column must equal the grind-check count — two independent instruments +/// on one quantity. +#[cfg(feature = "hash-metrics")] +fn print_transcript_counts(window: &str, c: &crypto::hash_metrics::Counts) { + let (ua, us, ut) = c.transcript_unattributed(); + println!( + "{:<12} transcript absorbs {}/{} · squeezes {}/{} · states {}/{} (keccak/rpx) · unattributed {}/{}/{}", + window, + c.transcript_absorbs_keccak, + c.transcript_absorbs_rpx, + c.transcript_squeezes_keccak, + c.transcript_squeezes_rpx, + c.transcript_states_keccak, + c.transcript_states_rpx, + ua, + us, + ut, + ); +} + +/// ★★ THE TRANSCRIPT PAIR, PINNED — one configuration, both sides. +/// +/// # Why a pair and not just the verify line +/// +/// The verify line alone is the number recursion cares about, but pinning both +/// makes their DIFFERENCE mutation-checkable, and that difference is a derived +/// quantity rather than a measurement: the verifier runs `owed`, the prover does +/// not, and `owed` is `Sum roots.len()` absorbs and `2 x epochs` squeezes. So a +/// change that moves one side without the other fails loudly here instead of +/// silently re-opening a question that took two lanes and a wrong candidate to +/// close. +/// +/// The 2 squeezes per `owed` call are not "two samples, two squeezes": a cubic +/// element is three 8-byte draws from a 32-byte buffer, so the first sample +/// squeezes once and leaves a group over, and the second spends the leftover and +/// squeezes again. Three not dividing four is the whole reason it is two. +/// +/// # WARNING: the guard is the ELF's sha256, not its name +/// +/// Two builds of the same guest, from the same sources, in two worktrees, share +/// a name and differ in bytes. That is not hypothetical: a 0.23% difference in +/// these very counts was read as a model error before it was traced to a +/// different build of "the same" guest — the arms' ELF touches genesis pages +/// (`0x280000`, `0x680000`) that another build does not, which moves the +/// GLOBAL_MEMORY set and with it the chain shapes these counts are made of. +/// +/// So a rebuild must not fail this assert — it must SKIP it, out loud, naming +/// the sha it saw. A silent pass and an absent assert are the same thing. +/// +/// # Configuration is part of the constant +/// +/// The counts are a function of the ELF, the input and the epoch size, so all +/// three are in the guard. Measured on FAST under `cuda,hash-metrics`, and +/// independently reproduced by lane V1's shape-derived closed form, which +/// predicts the verify triple exactly from the table shapes — two derivations, +/// one number. +/// +/// # ...and so is the BRANCH, which is why the absorbs are not one number +/// +/// The absorb columns carry a term that differs between lineages — one `u64` +/// per per-table count, per epoch statement — so they are pinned as a base plus +/// `EPOCHS * NUM_TABLE_KINDS`, with the kind count read from the struct. The +/// bases were taken on the seam branch (`whir/rpx`, the `c73568f4` measurement +/// plus this branch's computed statement padding); the merged lineage, which +/// binds one count more, gets its own totals from the same bases without +/// editing anything here. See [`transcript_pin::PROVE_BASE_ABSORBS`]. +#[cfg(feature = "hash-metrics")] +mod transcript_pin { + /// sha256 of the guest ELF these counts were measured against. + /// + /// MEASURED, and it has to say where: `sha256sum` on the box fixture + /// `ethrex_8f826601.elf`. The previous value agreed with this one for + /// exactly 16 hex characters and was invented for the other 48 — every + /// message that carried the sha carried a 16-char prefix, and the tail was + /// written to look like a measurement. The guard then skipped on the pinned + /// guest itself, and the skip line printed `[..16]` of both sides, which is + /// precisely the width at which a fabricated tail still agrees. + /// + /// A guard on a value nobody measured to full width is a guard on a guess. + /// Anything shortened for a message is a display; the constant is the + /// measurement. + pub const ELF_SHA256: &str = "8f826601776d4085cbb6fbf0302fe8d8d5d1be7940ac1aaca24899c6244ec80a"; + pub const ELF_LEN: usize = 3_948_504; + pub const EPOCH_LOG2: u32 = 21; + + /// Epoch proofs in the pinned run. Part of the measured shape, like the ELF + /// and the epoch size: 2^21 epochs over this guest is fifteen of them, which + /// is also where `OWED`'s thirty squeezes come from (two per epoch call). + pub const EPOCHS: u64 = 15; + + /// ★★ THE ABSORB COUNTS ARE A BASE PLUS A PER-BRANCH TERM, and the term is + /// read from the struct rather than written down. + /// + /// Every epoch statement binds one `u64` per per-table count + /// (`statement::absorb_table_counts`), so the absorb total carries + /// `EPOCHS * NUM_TABLE_KINDS`. That count is **not a constant of the + /// protocol**: the per-table campaign adds `TableCounts::blake3`, so this + /// lineage absorbs fourteen per epoch and the merged one fifteen. The box + /// discovered that as a pin FAILURE (+15 on both sides at the merged base), + /// and the term was legitimate — the BLAKE3 table is conditional and its + /// count is the one entry a verifier cannot derive, which is why per-table + /// bumped both domain tags for it. + /// + /// A pin carrying `583_940` would therefore be a constant describing one + /// branch while claiming to describe the protocol. The bases below are + /// branch-independent; the branch supplies its own kind count. + /// + /// ⚠ PROVENANCE, and one half of it is a PREDICTION. The box measured + /// `583_924 / 584_061` at `c73568f4` (run a2q, both hashes, against the + /// `8f826601` fixture). The computed statement padding adds exactly one + /// absorb per statement — fifteen epoch statements and one cross-epoch + /// statement, sixteen — giving `583_940 / 584_077`, from which + /// `EPOCHS * 14 = 210` is subtracted here. The `+16` has not been measured + /// yet; these constants are what will say so if it is wrong. + pub const PROVE_BASE_ABSORBS: u64 = 583_730; + /// The verify side's base. See [`PROVE_BASE_ABSORBS`]. + pub const VERIFY_BASE_ABSORBS: u64 = 583_867; + + /// Absorbs the per-table counts contribute to a whole continuation proof. + pub const fn table_count_absorbs() -> u64 { + EPOCHS * crate::statement::NUM_TABLE_KINDS as u64 + } + + /// (absorbs, squeezes, states) after `prove_continuation`. + pub const PROVE: (u64, u64, u64) = (PROVE_BASE_ABSORBS + table_count_absorbs(), 183_226, 2_996); + /// ...and after `verify_continuation`. The difference is `owed`, nothing else. + pub const VERIFY: (u64, u64, u64) = + (VERIFY_BASE_ABSORBS + table_count_absorbs(), 183_256, 2_996); + + /// `owed`'s own cost, stated rather than left as a subtraction: 137 absorbs + /// is `Sum roots.len()` over the 15 epoch calls, 30 squeezes is `2 x 15`, + /// and it reads no state. + pub const OWED: (u64, u64, u64) = (137, 30, 0); +} + +/// Whether the pinned counts describe THIS run. +/// +/// Pure, and taking the sha as a string so the refusal can be tested without +/// forging an ELF: a sha that agrees on a prefix and differs in the tail is one +/// `format!` away, which is the case that actually occurred. +#[cfg(feature = "hash-metrics")] +fn pin_applies(sha: &str, len: usize, epoch_size_log2: u32) -> bool { + sha == transcript_pin::ELF_SHA256 + && len == transcript_pin::ELF_LEN + && epoch_size_log2 == transcript_pin::EPOCH_LOG2 +} + +/// The line a skipped pin prints. +/// +/// FULL 64 hex on BOTH sides, never a prefix. The truncated version of this +/// line is why a wrong constant survived a box run: it showed `[..16]` of each, +/// the two agreed there, and the mismatch it existed to report was invisible in +/// its own output. A diagnostic that can agree while the values differ is not a +/// diagnostic. +#[cfg(feature = "hash-metrics")] +fn pin_skip_line(sha: &str, len: usize, epoch_size_log2: u32) -> String { + format!( + "{:<12} transcript pin SKIPPED - elf sha {} ({} bytes, epoch 2^{}); \ + pinned {} ({} bytes, epoch 2^{})", + "WHIR", + sha, + len, + epoch_size_log2, + transcript_pin::ELF_SHA256, + transcript_pin::ELF_LEN, + transcript_pin::EPOCH_LOG2, + ) +} + +/// Asserts the pinned pair, or says out loud why it did not. +#[cfg(feature = "hash-metrics")] +fn check_transcript_pins( + elf: &[u8], + epoch_size_log2: u32, + prove: &crypto::hash_metrics::Counts, + verify: &crypto::hash_metrics::Counts, +) { + use sha2::{Digest, Sha256}; + + let sha: String = Sha256::digest(elf) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + if !pin_applies(&sha, elf.len(), epoch_size_log2) { + // Never silent. A skipped assert that prints nothing is + // indistinguishable from one that passed, which is the failure this + // whole pin exists against. + println!("{}", pin_skip_line(&sha, elf.len(), epoch_size_log2)); + return; + } + + let triple = |c: &crypto::hash_metrics::Counts| { + ( + c.transcript_absorbs, + c.transcript_squeezes, + c.transcript_states, + ) + }; + assert_pinned_pair(triple(prove), triple(verify)); + println!( + "{:<12} transcript pin OK (both sides, and the owed delta)", + "WHIR" + ); +} + +/// The assertions themselves, split from the guard so they can be reached +/// without the pinned guest. +/// +/// ⚠ This split is the point, not tidiness. With the guard and the assertions +/// in one function, the assertions ran on the box and NOWHERE ELSE: a mistyped +/// constant, a swapped pair or a broken delta would have been discovered by a +/// GPU run rather than by `cargo test`. Taking the triples as arguments makes +/// every branch reachable from a laptop, which is why the four tests below +/// exist and why three of them are `should_panic`. +#[cfg(feature = "hash-metrics")] +fn assert_pinned_pair(prove: (u64, u64, u64), verify: (u64, u64, u64)) { + // Only the state columns are destructured: the two lines are compared whole + // against their pins, and the delta between them is a property of the + // CONSTANTS rather than of a measurement — see + // `the_pinned_constants_differ_by_owed`. + let (_, _, pt) = prove; + let (_, _, vt) = verify; + + assert_eq!( + prove, + transcript_pin::PROVE, + "the PROVE-side transcript counts moved" + ); + assert_eq!( + verify, + transcript_pin::VERIFY, + "the VERIFY-side transcript counts moved" + ); + + // ⚠ NO `owed` ASSERTION HERE, and its absence is deliberate. Once both + // lines match their pins, their difference is forced — `OWED` is + // `VERIFY - PROVE` by construction, so a third runtime assertion could + // never fire. Writing its test is what exposed that: the constructed + // counter-example was rejected by the VERIFY assertion two lines up, + // never reaching the delta. + // + // The delta is a statement about the CONSTANTS, not about a measurement, + // so it is checked where it can fail — see + // [`the_pinned_constants_differ_by_owed`]. + + // The control that costs nothing: one `state()` per grind check, so this + // column and the grind count are two instruments on one quantity. + assert_eq!( + pt, vt, + "the two sides disagree on state reads, which are grind checks on both" + ); +} + +/// ★★ The constants are the measurement — asserted against LITERALS. +/// +/// Passing `transcript_pin::PROVE` here would be a check that cannot fail: a +/// mutated constant would move the input and the expectation together and the +/// test would pass on any value. The numbers below are written out so that a +/// constant which drifts, is mistyped, or has its two lines swapped fails here, +/// on a laptop, rather than on the box an hour later. +/// +/// ⚠ The literals are the BASES, and the per-branch term is recomputed from the +/// struct — because that term is what differs between this lineage and the +/// merged one. Writing the totals out would make this test the thing that has +/// to be edited on every branch, which is precisely the property the split +/// removed from the pin. +#[cfg(feature = "hash-metrics")] +#[test] +fn the_pinned_pair_is_the_measurement() { + let counts = 15 * crate::statement::NUM_TABLE_KINDS as u64; + assert_pinned_pair( + (583_730 + counts, 183_226, 2_996), + (583_867 + counts, 183_256, 2_996), + ); +} + +/// ★ One unit on the prove line fails on the prove assertion. +#[cfg(feature = "hash-metrics")] +#[test] +#[should_panic(expected = "the PROVE-side transcript counts moved")] +fn a_prove_count_off_by_one_is_rejected() { + let (a, s, t) = transcript_pin::PROVE; + assert_pinned_pair((a + 1, s, t), transcript_pin::VERIFY); +} + +/// ★ One unit on the verify line fails on the verify assertion. +#[cfg(feature = "hash-metrics")] +#[test] +#[should_panic(expected = "the VERIFY-side transcript counts moved")] +fn a_verify_count_off_by_one_is_rejected() { + let (a, s, t) = transcript_pin::VERIFY; + assert_pinned_pair(transcript_pin::PROVE, (a + 1, s, t)); +} + +/// ★★ The per-branch term is a TERM, not a re-baseline. +/// +/// The pin's absorb totals are `base + EPOCHS * NUM_TABLE_KINDS`, and this says +/// the split is the one the protocol makes: the kind count is the length of the +/// list `absorb_table_counts` walks, and an epoch statement absorbs exactly that +/// many counts. +/// +/// It can fail. `NUM_TABLE_KINDS` is checked against the array +/// `statement::table_count_values` actually returns — which is the one place the +/// destructure of `TableCounts` is written — so a field added to `TableCounts` +/// and pushed into the array without bumping the constant fails to compile, and +/// a constant bumped without the field fails here. +#[cfg(feature = "hash-metrics")] +#[test] +fn the_per_branch_term_is_the_table_kind_count() { + let zero = crate::TableCounts { + cpu: 0, + lt: 0, + memw: 0, + memw_aligned: 0, + load: 0, + mul: 0, + dvrm: 0, + shift: 0, + branch: 0, + memw_register: 0, + eq: 0, + bytewise: 0, + store: 0, + cpu32: 0, + }; + let kinds = crate::statement::table_count_values(&zero).len(); + assert_eq!( + kinds, + crate::statement::NUM_TABLE_KINDS, + "NUM_TABLE_KINDS is not the number of counts a statement absorbs" + ); + assert_eq!( + transcript_pin::table_count_absorbs(), + transcript_pin::EPOCHS * kinds as u64, + "the pin's per-branch term is not `epochs x kinds`" + ); + assert_eq!( + transcript_pin::PROVE.0 - transcript_pin::PROVE_BASE_ABSORBS, + transcript_pin::table_count_absorbs(), + ); + assert_eq!( + transcript_pin::VERIFY.0 - transcript_pin::VERIFY_BASE_ABSORBS, + transcript_pin::table_count_absorbs(), + ); +} + +/// ★★ THE DERIVED DELTA, checked where it can actually fail: on the constants. +/// +/// `owed` is the only thing the verifier does that the prover does not, so the +/// two pinned lines must differ by exactly it — `Sum roots.len()` absorbs, +/// `2 x epochs` squeezes, no state reads. That is a claim about the pair of +/// constants, and it is the claim that survives a re-baseline: if someone +/// measures a new run and updates PROVE and VERIFY together, this fires unless +/// `owed` is still `owed`, forcing them to look at why the difference moved +/// rather than carrying a changed protocol into two numbers that agree with +/// each other. +/// +/// ⚠ It lives here rather than inside [`assert_pinned_pair`] because there it +/// could never fire: with both lines asserted against their pins, their +/// difference is forced. That was found by writing the test — the constructed +/// counter-example never reached the delta, because the VERIFY assertion +/// rejected it first. +#[cfg(feature = "hash-metrics")] +#[test] +fn the_pinned_constants_differ_by_owed() { + let (pa, ps, pt) = transcript_pin::PROVE; + let (va, vs, vt) = transcript_pin::VERIFY; + assert_eq!( + (va - pa, vs - ps, vt - pt), + transcript_pin::OWED, + "the two pinned lines no longer differ by `owed` — one was re-baselined \ + without the other, or the protocol changed" + ); + + // …and `owed` is itself derived, not observed: 137 absorbs is one per root + // over the 15 epoch calls, 30 squeezes is two per call. Stating the shape + // means a future epoch count cannot silently keep the old constant. + let (oa, os, ot) = transcript_pin::OWED; + assert_eq!( + os, + 2 * transcript_pin::EPOCHS, + "`owed` samples twice per epoch call" + ); + assert_eq!(ot, 0, "`owed` reads no transcript state"); + // ⚠ The absorb count gets no assertion of its own. It is `Sum roots.len()` + // over the epochs — data from the table shapes, not something derivable + // here — so any predicate this test could write about it would be either + // circular (comparing the constant to itself) or vacuous. An earlier draft + // had `oa % 1 == 0`, which is true of every integer. It is pinned by + // `VERIFY - PROVE` above and by V1's closed form, which is where it belongs. + let _ = oa; +} + +/// ★★ A sha that agrees on a PREFIX is refused, and the skip line shows why. +/// +/// The defect this is written against: the constant's last 48 hex were +/// fabricated, so the guard skipped on the pinned guest itself — and the skip +/// line printed 16 characters of each side, the exact width at which the real +/// sha and the invented one agreed. The failure was invisible in the output of +/// the thing that existed to report it. +/// +/// Both halves are needed. The refusal alone would pass with a truncated +/// diagnostic; the diagnostic alone would pass with a comparison that only +/// looked at a prefix. +#[cfg(feature = "hash-metrics")] +#[test] +fn a_sha_agreeing_only_on_the_prefix_is_refused_and_says_so() { + // Same first 16 hex, different tail — one `format!`, no ELF to forge. + let near_miss = format!("{}{}", &transcript_pin::ELF_SHA256[..16], "0".repeat(48)); + assert_eq!(near_miss.len(), 64); + assert_eq!( + near_miss[..16], + transcript_pin::ELF_SHA256[..16], + "the near miss must agree on the prefix, or it tests nothing" + ); + assert_ne!(near_miss, transcript_pin::ELF_SHA256); + + assert!( + !pin_applies( + &near_miss, + transcript_pin::ELF_LEN, + transcript_pin::EPOCH_LOG2 + ), + "a sha differing only after position 16 was accepted: the comparison is \ + looking at a prefix" + ); + + // …and the line it prints must make the difference visible. + let line = pin_skip_line( + &near_miss, + transcript_pin::ELF_LEN, + transcript_pin::EPOCH_LOG2, + ); + assert!( + line.contains(&near_miss), + "the skip line does not carry the found sha in full: {line}" + ); + assert!( + line.contains(transcript_pin::ELF_SHA256), + "the skip line does not carry the pinned sha in full: {line}" + ); + + // The property in one assertion: whatever the line shows of each side, the + // two shown values must differ. A prefix display fails here. + let shown: Vec<&str> = line.split_whitespace().filter(|w| w.len() == 64).collect(); + assert_eq!( + shown.len(), + 2, + "expected two 64-hex values in the skip line, found {}: {line}", + shown.len() + ); + assert_ne!( + shown[0], shown[1], + "the skip line shows the same value twice for a genuine mismatch" + ); +} + +/// ★ The pinned constant is a full-width sha, not a truncation. +#[cfg(feature = "hash-metrics")] +#[test] +fn the_pinned_sha_is_full_width() { + assert_eq!( + transcript_pin::ELF_SHA256.len(), + 64, + "a sha256 is 64 hex characters; anything shorter is a display that got \ + pinned" + ); + assert!( + transcript_pin::ELF_SHA256 + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), + "the pinned sha is not lowercase hex" + ); +} + +/// ★ The guard skips rather than fires on a guest that is not the pinned one. +/// +/// Card-free and not `#[ignore]`d, because the skip path is the half that runs +/// on every other invocation of the bench and the half that would fail silently +/// if it were wrong. If the guard were inverted — asserting on the wrong ELF — +/// every run on any other program would panic on counts that were never about +/// it; if it were absent, the pin would be decorative. +/// +/// The counts passed in are deliberately absurd. Reaching the assertions with +/// them would panic, so a test that returns at all proves the guard returned +/// first. +#[cfg(feature = "hash-metrics")] +#[test] +fn the_transcript_pin_skips_a_guest_it_does_not_recognise() { + let nonsense = crypto::hash_metrics::Counts { + transcript_absorbs: 1, + transcript_squeezes: 2, + transcript_states: 3, + ..Default::default() + }; + + // Wrong bytes, wrong length. + check_transcript_pins( + b"not an elf", + transcript_pin::EPOCH_LOG2, + &nonsense, + &nonsense, + ); + + // ⚠ Right length, wrong bytes — the guard must be the sha and not the size, + // which is the whole point of preferring it to the ELF's name. + let same_length = vec![0u8; transcript_pin::ELF_LEN]; + check_transcript_pins( + &same_length, + transcript_pin::EPOCH_LOG2, + &nonsense, + &nonsense, + ); +} + #[test] #[ignore] fn continuations() { @@ -297,6 +829,24 @@ RAYON_NUM_THREADS={threads}, backend={backend}" } if backend != "fri" { + // Held from the prove window to the verify one so the pair - and the + // `owed` delta between them - can be asserted together. Uninitialised + // and assigned exactly once: an `Option` here would carry a `None` the + // compiler can prove is never read, since the assignment dominates the + // use and both sit in this one branch. + #[cfg(feature = "hash-metrics")] + let prove_counts; + // ★ Zeroed per arm, so the counts below belong to THIS prove and not to + // whatever ran before it in the process. + #[cfg(feature = "cuda")] + { + crypto::grinding::reset_gpu_grind_calls(); + multilinear::gpu::reset_call_counters(); + } + // ★ Same reason, for the transcript counters: the line printed below + // must be THIS arm's and not the process's running total. + #[cfg(feature = "hash-metrics")] + crypto::hash_metrics::reset(); let start = Instant::now(); let bundle = crate::multilinear_continuation::prove_continuation( &bytes, @@ -306,14 +856,66 @@ RAYON_NUM_THREADS={threads}, backend={backend}" ) .expect("multilinear continuation"); let prove = start.elapsed(); + + // ★★ READ 0 FOR EVERY WHIR ARM — which dispatches actually reached the + // card. The `★ WHIR HASH:` banner says which hash was SELECTED; these + // say which kernels RAN, and the two are not the same claim. + // + // A measured RPX arm once came in 14.5x slower than keccak with a + // correct, KAT-pinned grind kernel sitting unused, because the host-side + // dispatch had no arm for it. Nothing in this bench's output named the + // cause: prove time, verify time, proof size and epoch count were all + // consistent with "the hash is just expensive". A grind count of ZERO + // beside a commit count of thousands says it in one line. + #[cfg(feature = "cuda")] + println!( + "{:<12} gpu commits {} · host fallbacks {} · keccak grinds {} · rpx grinds {}", + "WHIR", + multilinear::gpu::commit_calls(), + multilinear::gpu::host_fallbacks(), + crypto::grinding::gpu_grind_calls(), + crypto::grinding::gpu_grind_calls_rpx(), + ); + // ★★ WHICH SPONGE THE TRANSCRIPT RAN ON, per arm and on BOTH sides. + // + // The line above says which KERNELS ran; this one says which sponge the + // Fiat-Shamir transcript used, and they are not the same claim. For + // four measured A/Bs the RPX arm ran an RPX Merkle backend, an RPX + // grind and a KECCAK transcript, and nothing printed here disagreed. + // + // Both sides are printed because one is not evidence. A counter on the + // RPX side alone reads "rpx > 0, keccak 0" — and that keccak zero is + // equally true when the keccak transcript ran and nobody instrumented + // it, which is exactly the state that hid. `unattributed` is printed + // for the same reason one level out: a third configuration arriving + // with no counter of its own would otherwise look like silence. + #[cfg(feature = "hash-metrics")] + { + let c = crypto::hash_metrics::snapshot(); + print_transcript_counts("WHIR prove", &c); + prove_counts = c; + } let size = rkyv::to_bytes::(&bundle) .expect("serialize") .len(); let epochs = bundle.num_epochs(); + // ★★ The VERIFY side gets its own window, and it is the side that + // matters for recursion: an LFM replays the VERIFIER's transcript, not + // the prover's. The two are different numbers over different work — + // differencing one against a closed form derived for the other is the + // same category error as comparing a prove stopwatch to a verify one. + #[cfg(feature = "hash-metrics")] + crypto::hash_metrics::reset(); let start = Instant::now(); let ok = crate::multilinear_continuation::verify_continuation(&bytes, &bundle, &opts) .expect("multilinear verify"); assert!(ok, "the multilinear continuation must verify"); + #[cfg(feature = "hash-metrics")] + { + let verify_counts = crypto::hash_metrics::snapshot(); + print_transcript_counts("WHIR verify", &verify_counts); + check_transcript_pins(&bytes, epoch_size_log2, &prove_counts, &verify_counts); + } whir = Some((prove, start.elapsed(), size, epochs)); } @@ -449,6 +1051,7 @@ fn continuation_phases() { #[cfg(feature = "cuda")] for (tag, count) in [ ("gpu commits", multilinear::gpu::commit_calls()), + ("host fallbacks", multilinear::gpu::host_fallbacks()), ("gpu sumchecks", multilinear::gpu::sumcheck_calls()), ("gpu evals", multilinear::gpu::evaluate_calls()), ("gpu trees", multilinear::gpu::tree_calls()), @@ -496,8 +1099,15 @@ fn phases() { let execute = start.elapsed(); let start = Instant::now(); - let mut traces = - Traces::from_elf_and_logs(&elf, &logs, &MaxRowsConfig::default(), &inputs).expect("traces"); + let mut traces = Traces::from_elf_and_logs( + &elf, + &logs, + &MaxRowsConfig::default(), + &inputs, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("traces"); let trace_build = start.elapsed(); let table_counts = traces.table_counts(); @@ -566,7 +1176,20 @@ fn phases() { }) .collect(); let count = tables.len(); - let committed = CommittedTables::commit(tables, &config).expect("commit"); + // ⚠ KECCAK ONLY, and it refuses rather than mislabels. + // + // The committed tables escape into the rest of this function, so the hash + // cannot be a `match` here — both arms would have to return the same type + // and they do not. Rather than print `★ WHIR HASH: rpx256` over keccak's + // seconds, this bench asserts the knob agrees with what it actually runs. + // The hash-arm split lives in `commit_phases`, whose `merkle` pass isolates + // the hashing anyway, which is the number a hash comparison wants. + assert_eq!( + crate::whir_hash_knob::selected(), + crate::whir_hash_knob::Setting::Keccak, + "`phases` commits with keccak; run `commit_phases` for the hash arms" + ); + let committed = CommittedTables::<_, _, KeccakWhir>::commit(tables, &config).expect("commit"); let commit = start.elapsed(); // `multi_prove`'s own body, so the tables' arguments and the one opening @@ -600,7 +1223,7 @@ fn phases() { .iter() .flat_map(|t| t.columns()) .collect(); - let columns = multilinear::stacked_eval::prove::( + let columns = multilinear::stacked_eval::prove::( &committed.groups()[0], &group_columns, None, @@ -648,6 +1271,7 @@ fn phases() { for (tag, count) in [ ("gpu grinds", stark::gpu_lde::gpu_grind_calls()), ("gpu commits", multilinear::gpu::commit_calls()), + ("host fallbacks", multilinear::gpu::host_fallbacks()), ("gpu sumchecks", multilinear::gpu::sumcheck_calls()), ("gpu rounds", multilinear::gpu::sumcheck_rounds()), ("gpu evals", multilinear::gpu::evaluate_calls()), @@ -686,8 +1310,15 @@ fn commit_phases() { .and_then(Executor::run) .expect("run") .logs; - let mut traces = - Traces::from_elf_and_logs(&elf, &logs, &MaxRowsConfig::default(), &inputs).expect("trace"); + let mut traces = Traces::from_elf_and_logs( + &elf, + &logs, + &MaxRowsConfig::default(), + &inputs, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .expect("trace"); let table_counts = traces.table_counts(); let airs = crate::VmAirs::new( &elf, @@ -747,17 +1378,26 @@ fn commit_phases() { let codeword = whir::encode::(&coeffs, &domain).expect("encode"); encode += start.elapsed(); let start = Instant::now(); - let commitment = CodewordCommitment::new( - &codeword, - config - .schedule(poly.num_vars()) - .first() - .copied() - .unwrap_or(0), - ) - .expect("commit"); + // ★ The `merkle` pass is the LEAF AND TREE HASHING, alone — stack, + // lift and encode are clocked apart above. So this line is the hash + // term itself, and it has to follow the knob or the two arms are not + // comparable. + crate::with_whir_hash!(|H| { + let commitment = CodewordCommitment::<_, H>::new( + &codeword, + config + .schedule(poly.num_vars()) + .first() + .copied() + .unwrap_or(0), + ) + .expect("commit"); + // Dropped inside the arm: the commitment's type names `H`, so it + // cannot leave the block. That is also why this is the bench the + // hash arms run through — nothing here escapes. + drop(commitment); + }); merkle += start.elapsed(); - drop(commitment); } let label = if input.is_empty() { &name } else { &input }; @@ -879,8 +1519,15 @@ fn constraint_program_sizes() { .and_then(Executor::run) .unwrap() .logs; - let mut traces = - Traces::from_elf_and_logs(&elf, &logs, &MaxRowsConfig::default(), &inputs).unwrap(); + let mut traces = Traces::from_elf_and_logs( + &elf, + &logs, + &MaxRowsConfig::default(), + &inputs, + #[cfg(feature = "disk-spill")] + stark::storage_mode::StorageMode::Ram, + ) + .unwrap(); let table_counts = traces.table_counts(); let airs = crate::VmAirs::new( &elf, diff --git a/prover/src/tests/multilinear_prove_tests.rs b/prover/src/tests/multilinear_prove_tests.rs index 98aadcaca..2b511959e 100644 --- a/prover/src/tests/multilinear_prove_tests.rs +++ b/prover/src/tests/multilinear_prove_tests.rs @@ -10,6 +10,7 @@ use crate::test_utils::asm_elf_bytes; use stark::proof::options::ProofOptions; use crate::tables::MaxRowsConfig; +use multilinear::whir_hash::KeccakWhir; fn prove(elf: &[u8]) -> MultilinearVmProof { multilinear_prove::prove_with_options( @@ -138,7 +139,7 @@ fn a_forged_preprocessed_column_is_rejected() { let prove = |columns: Vec>>| { let table = CommittedTable::from_layout(layout(), |col| columns[col as usize].clone()).unwrap(); - let committed = CommittedTables::commit(vec![table], &config).unwrap(); + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config).unwrap(); let mut transcript = DefaultTranscript::::new(b"forged"); let proof = multilinear_table::multi_prove(&committed, &config, &mut transcript).unwrap(); ( @@ -162,7 +163,7 @@ fn a_forged_preprocessed_column_is_rejected() { multilinear::whir::Domain, )| { let mut transcript = DefaultTranscript::::new(b"forged"); - multilinear_table::multi_verify( + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( &proof, &[statement], std::slice::from_ref(&stacked), diff --git a/prover/src/tests/multilinear_table_tests.rs b/prover/src/tests/multilinear_table_tests.rs index b288aab8b..9dc7a53f8 100644 --- a/prover/src/tests/multilinear_table_tests.rs +++ b/prover/src/tests/multilinear_table_tests.rs @@ -42,6 +42,7 @@ use crate::test_utils::{ use executor::elf::Elf; use executor::vm::execution::Executor; use executor::vm::logs::Log; +use multilinear::whir_hash::KeccakWhir; type ExtE = FieldElement; @@ -113,7 +114,7 @@ fn argue>( }; // The trace goes in as it is: base-field. Only the challenges are not. let table = CommittedTable::from_layout(layout()?, |col| columns[col as usize].clone())?; - let committed = CommittedTables::commit(vec![table], &config())?; + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config())?; let mut prover = DefaultTranscript::::new(b"vm-table"); let proof = multilinear_table::multi_prove(&committed, &config(), &mut prover)?; @@ -135,7 +136,7 @@ fn argue>( let owed = multilinear_table::contribution(&proof.tables[0].bus_output) .ok_or(multilinear::Error::BusImbalance)?; let mut verifier = DefaultTranscript::::new(b"vm-table"); - multilinear_table::multi_verify( + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( &proof, &[statement], std::slice::from_ref(committed.groups()[0].layout()), @@ -308,7 +309,8 @@ fn prove_and_verify_all_tables(elf: Elf, logs: &[Log]) -> usize { } let count = tables.len(); // Every table's columns in one commitment: 55 of them still open once. - let committed = CommittedTables::commit(tables, &config()).expect("commit every table"); + let committed = + CommittedTables::<_, _, KeccakWhir>::commit(tables, &config()).expect("commit every table"); let mut prover = DefaultTranscript::::new(b"vm-sweep"); let proof = multilinear_table::multi_prove(&committed, &config(), &mut prover) @@ -346,7 +348,7 @@ fn prove_and_verify_all_tables(elf: Elf, logs: &[Log]) -> usize { .expect("the commit fingerprints are invertible"); let mut verifier = DefaultTranscript::::new(b"vm-sweep"); - multilinear_table::multi_verify( + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( &proof, &statements, std::slice::from_ref(&stacked), @@ -425,7 +427,7 @@ fn a_real_table_proof_survives_serialization() { }; let table = CommittedTable::from_layout(layout(), |col| columns[col as usize].clone()).unwrap(); - let committed = CommittedTables::commit(vec![table], &config()).unwrap(); + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config()).unwrap(); let mut prover = DefaultTranscript::::new(b"serialized"); let proof = multilinear_table::multi_prove(&committed, &config(), &mut prover).unwrap(); @@ -446,7 +448,7 @@ fn a_real_table_proof_survives_serialization() { // The roots travel in the proof, so a format that dropped them would // fail here rather than pass on the original's. let mut verifier = DefaultTranscript::::new(b"serialized"); - multilinear_table::multi_verify( + multilinear_table::multi_verify::<_, _, _, KeccakWhir>( round_tripped, &[statement], std::slice::from_ref(committed.groups()[0].layout()), @@ -524,6 +526,6 @@ fn a_real_table_is_one_commitment_for_its_main_columns() { assert_eq!(table.num_committed_columns(), lt::cols::NUM_COLUMNS); // And they all ride in one stacked polynomial, alone or alongside others. - let committed = CommittedTables::commit(vec![table], &config()).unwrap(); + let committed = CommittedTables::<_, _, KeccakWhir>::commit(vec![table], &config()).unwrap(); assert_eq!(committed.roots().len(), 1); } diff --git a/prover/src/tests/statement_alignment_tests.rs b/prover/src/tests/statement_alignment_tests.rs new file mode 100644 index 000000000..e49c9d02e --- /dev/null +++ b/prover/src/tests/statement_alignment_tests.rs @@ -0,0 +1,695 @@ +//! ★★ Every value a WHIR proof absorbs starts on a field element boundary. +//! +//! ```text +//! cargo test -p lambda-vm-prover --lib statement_alignment +//! ``` +//! +//! # The property, and what it is NOT +//! +//! The transcript hashes BYTES. The algebraic configuration's sponge re-slices +//! everything absorbed since its last squeeze into field elements every 8 bytes +//! (`crypto::hash::rpx::sponge_leaf_bytes`), so a value absorbed at an offset +//! that is not a multiple of 8 straddles two of them — which a field-machine +//! verifier replaying the transcript can only reproduce by decomposing bits. +//! +//! The property is therefore about OFFSETS, over every shape: *every absorb +//! after the statement lands at a window offset that is a multiple of 8*. It is +//! **not** "the statement's fixed prefix is a multiple of 8": the roots do not +//! follow the fixed prefix, they follow two variable-length fields, and a +//! statement padded to a round fixed prefix leaves them at +//! `(|public_output| + |table_num_vars|) mod 8` — 2 mod 8 at the shape this +//! system runs. That arithmetic is why the pad is computed from an accumulated +//! length rather than written as a constant, and this file is where the claim +//! can fail. +//! +//! # Two tests, because one of them cannot see the other's failure +//! +//! * [`the_statement_ends_on_a_field_element_boundary`] sweeps the shapes +//! against a recording transcript. It sees every shape and no protocol. +//! * [`every_absorb_of_a_real_prove_is_field_element_aligned`] drives a real +//! `multi_prove` and sees one shape and the whole protocol. It is the one that +//! can report a misalignment the statement padding does not fix. +//! +//! # Why the recorder is validated rather than trusted +//! +//! [`WindowRecorder`] has to know where a window ENDS, and only a squeeze ends +//! one — so it mirrors `DefaultTranscript`'s duplex output buffer instead of +//! guessing. A mirror that drifts would draw different challenges, so the +//! end-to-end test first proves the same fixture twice, once through the +//! production transcript and once through the recorder, and requires the two +//! proofs to serialise to the same bytes. The offsets it reports are then the +//! production stream's offsets, not a model's. + +use digest::Digest; +use math::field::element::FieldElement; +use math::field::traits::HasDefaultTranscript; +use math::traits::AsBytes; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use crypto::fiat_shamir::is_transcript::IsTranscript; +use crypto::fiat_shamir::transcript_hash::{ + HasTranscriptHash, KeccakTranscriptHash, TranscriptHash, +}; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::statement::{FELT_BYTES, NUM_TABLE_KINDS, statement_padding}; +use crate::tables::eq::{EqConstraints, EqOperation, generate_eq_trace}; +use crate::test_utils::{ConcreteVmAir, E, F, create_eq_air}; +use crate::{RuntimePageRange, TableCounts}; + +/// Bytes one squeeze hands the transcript, and therefore the offset a fresh +/// window opens at. `DefaultTranscript::sample` finalize-resets the sponge and +/// re-absorbs its own 32-byte output, so a window never opens empty. +/// +/// It is private to `default_transcript`, so it is mirrored here rather than +/// imported — and a wrong value here does not silently weaken this file: the +/// duplex buffer would refill at the wrong time, the challenge stream would +/// diverge, and +/// [`every_absorb_of_a_real_prove_is_field_element_aligned`]'s byte comparison +/// against the production transcript would fail. +const SQUEEZE_LEN: usize = 32; + +// ------------------------------------------------------------------------- +// The recording transcript +// ------------------------------------------------------------------------- + +/// A transcript that records the window offset of every absorb. +/// +/// Absorption is DELEGATED — the inner `DefaultTranscript` is what hashes, so +/// this is the production sponge with a tape attached. Only the duplex output +/// buffer is mirrored, because that is the only part that tells the recorder +/// when a window ends. +struct WindowRecorder { + inner: DefaultTranscript, + out_buf: [u8; SQUEEZE_LEN], + out_pos: usize, + /// Bytes absorbed since the sponge was last reset by a squeeze. + window: usize, + /// `(window offset, length)` of every absorb, in call order. + absorbs: Vec<(usize, usize)>, +} + +/// Of the absorbs recorded after the first `from` — which is how a caller says +/// "everything after the statement" — the ones that do not start on a field +/// element boundary. +fn misaligned(absorbs: &[(usize, usize)], from: usize) -> Vec<(usize, usize)> { + absorbs[from..] + .iter() + .copied() + .filter(|(offset, _)| !offset.is_multiple_of(FELT_BYTES)) + .collect() +} + +impl WindowRecorder { + fn new() -> Self { + Self { + inner: DefaultTranscript::::new(&[]), + out_buf: [0u8; SQUEEZE_LEN], + out_pos: SQUEEZE_LEN, + window: 0, + absorbs: Vec::new(), + } + } + + fn record(&mut self, len: usize) { + self.absorbs.push((self.window, len)); + self.window += len; + // Same invalidation the inner transcript performs: a challenge drawn + // after an absorb must depend on it. + self.out_pos = SQUEEZE_LEN; + } + + /// `DefaultTranscript::next_sample_u64`, mirrored. The squeeze itself is the + /// inner transcript's, so the bytes are production's; what is duplicated is + /// only the bookkeeping that says when one happens. + fn next_u64(&mut self) -> u64 { + if self.out_pos + 8 > SQUEEZE_LEN { + self.out_buf = self.inner.sample(); + self.out_pos = 0; + // A squeeze finalize-resets the sponge and re-absorbs its own + // output, so the new window opens holding those 32 bytes. + self.window = SQUEEZE_LEN; + } + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(&self.out_buf[self.out_pos..self.out_pos + 8]); + self.out_pos += 8; + u64::from_be_bytes(bytes) + } +} + +impl HasTranscriptHash for WindowRecorder { + type Hash = T; +} + +impl IsTranscript for WindowRecorder { + fn append_bytes(&mut self, new_bytes: &[u8]) { + self.record(new_bytes.len()); + self.inner.append_bytes(new_bytes); + } + + fn append_field_element(&mut self, element: &FieldElement) { + // Counted per `update`, the same unit the inner transcript counts, so a + // serialisation that streamed an element in pieces would show up here + // as several absorbs rather than one. + let mut chunks: Vec = Vec::new(); + element.stream_bytes(&mut |b| chunks.push(b.len())); + for len in chunks { + self.record(len); + } + self.inner.append_field_element(element); + } + + fn state(&self) -> [u8; 32] { + // Finalizes a clone: no reset, no re-absorb, so the window does not move. + self.inner.state() + } + + fn sample_field_element(&mut self) -> FieldElement { + E::sample_field_element_from(|| self.next_u64()) + } + + fn sample_u64(&mut self, upper_bound: u64) -> u64 { + assert!(upper_bound > 0, "upper_bound must be greater than 0"); + let threshold = upper_bound.wrapping_neg() % upper_bound; + loop { + let candidate = self.next_u64(); + if candidate >= threshold { + return candidate % upper_bound; + } + } + } +} + +// ------------------------------------------------------------------------- +// The closed forms, written from the field lists rather than from the code +// ------------------------------------------------------------------------- + +const DIGEST: usize = 32; +const U64: usize = 8; +/// `log_blowup`, `log_folding`, `num_queries`. +const CONFIG: usize = 3 * U64; +/// `grind.folding`, `grind.ood`, `grind.query`, absorbed as one 3-byte value. +const GRIND_TRAILER: usize = 3; + +/// What a statement's fields add up to, what it therefore absorbs in total, and +/// how many times it calls the transcript. +#[derive(Clone, Copy, Debug)] +struct Expected { + /// The statement's own fields, before any padding. + body: usize, + /// `body` plus the pad, which is what the transcript should have taken. + total: usize, + /// Absorb calls, padding included. + calls: usize, +} + +impl Expected { + fn new(body: usize, calls: usize) -> Self { + Self { + body, + total: body + statement_padding(body), + calls, + } + } +} + +/// What an epoch statement absorbs, field by field. +/// +/// Deliberately a SECOND derivation: the production function accumulates its +/// length beside its own absorbs, and this adds up the fields it is supposed to +/// have. The two can only be compared here, and a field added to one and not +/// the other fails here. +fn epoch_expected(public_output: usize, table_num_vars: usize) -> Expected { + let body = crate::multilinear_continuation::MULTILINEAR_EPOCH_TAG.len() + + DIGEST + + U64 // epoch_label + + U64 // |public_output| + + public_output + + NUM_TABLE_KINDS * U64 + + U64 // |table_num_vars| + + table_num_vars + + CONFIG + + GRIND_TRAILER; + let calls = 1 // tag + + 1 // elf digest + + 1 // epoch label + + 1 // |public_output| + + 1 // public_output + + NUM_TABLE_KINDS + + 1 // |table_num_vars| + + 1 // table_num_vars + + 3 // config + + 1 // grind trailer + + 1; // padding, ALWAYS + Expected::new(body, calls) +} + +/// The same for the cross-epoch statement. `page_bases` is eight bytes an entry +/// and cannot move the alignment; `table_num_vars` can. +fn global_expected(page_bases: usize, table_num_vars: usize) -> Expected { + let body = crate::multilinear_continuation::MULTILINEAR_GLOBAL_TAG.len() + + DIGEST + + U64 // num_epochs + + U64 // num_private_input_pages + + U64 // |page_bases| + + page_bases * U64 + + U64 // |table_num_vars| + + table_num_vars + + CONFIG + + GRIND_TRAILER; + let calls = 1 + 1 + 1 + 1 + 1 + page_bases + 1 + 1 + 3 + 1 + 1; + Expected::new(body, calls) +} + +/// And for the monolithic multilinear statement. Its ranges are sixteen bytes +/// an entry, so they too are alignment-neutral. +fn monolithic_expected( + public_output: usize, + runtime_page_ranges: usize, + table_num_vars: usize, +) -> Expected { + let body = crate::statement::MULTILINEAR_TAG.len() + + DIGEST + + U64 // |public_output| + + public_output + + NUM_TABLE_KINDS * U64 + + U64 // num_private_input_pages + + U64 // |runtime_page_ranges| + + runtime_page_ranges * 2 * U64 + + U64 // |table_num_vars| + + table_num_vars + + CONFIG + + GRIND_TRAILER; + let calls = + 1 + 1 + 1 + 1 + NUM_TABLE_KINDS + 1 + 1 + 2 * runtime_page_ranges + 1 + 1 + 3 + 1 + 1; + Expected::new(body, calls) +} + +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + } +} + +fn counts() -> TableCounts { + TableCounts { + cpu: 1, + lt: 2, + memw: 3, + memw_aligned: 4, + load: 5, + mul: 6, + dvrm: 7, + shift: 8, + branch: 9, + memw_register: 10, + eq: 11, + bytewise: 12, + store: 13, + cpu32: 14, + } +} + +/// The shapes the sweep runs, as `(|public_output|, |table_num_vars|)`. +/// +/// `(0, 34)` is what this system actually proves — and the shape a constant pad +/// would have left at 2 mod 8. The others are chosen so the residue of +/// `|public_output| + |table_num_vars|` differs: a sweep in which every shape +/// needed the same pad would pass with the pad hard-coded to that value. +const SHAPES: &[(usize, usize)] = &[(0, 34), (7, 34), (32, 34), (33, 35), (2, 1), (0, 1)]; + +// ------------------------------------------------------------------------- +// (1) The statement, over every shape +// ------------------------------------------------------------------------- + +#[test] +fn the_statement_ends_on_a_field_element_boundary() { + let mut residues_seen = std::collections::BTreeSet::new(); + + for &(po_len, tnv_len) in SHAPES { + let public_output = vec![0xABu8; po_len]; + let table_num_vars = vec![20u8; tnv_len]; + + let mut rec = WindowRecorder::::new(); + crate::multilinear_continuation::absorb_epoch( + &mut rec, + &[7u8; 32], + &public_output, + &counts(), + 3, + &table_num_vars, + &config(), + ); + + let expected = epoch_expected(po_len, tnv_len); + residues_seen.insert(expected.body % FELT_BYTES); + + // ★ The property first, so a mutation reports the property. The two + // assertions under it are corroborating derivations, not the claim. + assert!( + rec.window.is_multiple_of(FELT_BYTES), + "epoch statement at (po {po_len}, tnv {tnv_len}) ended at byte \ + {} = {} mod {FELT_BYTES}: whatever is absorbed next straddles two \ + field elements", + rec.window, + rec.window % FELT_BYTES, + ); + assert_eq!( + rec.window, expected.total, + "epoch statement at (po {po_len}, tnv {tnv_len}) absorbed a different \ + number of BYTES than its field list implies", + ); + assert_eq!( + rec.absorbs.len(), + expected.calls, + "epoch statement at (po {po_len}, tnv {tnv_len}) absorbed a different \ + number of times than its field list implies — a padding absorb that \ + is skipped when the pad is empty shows up here", + ); + + // And the thing that actually follows a statement: the roots. + let before = rec.absorbs.len(); + for _ in 0..4 { + rec.append_bytes(&[0u8; 32]); + } + assert_eq!( + misaligned(&rec.absorbs, before), + Vec::new(), + "a root absorbed after the epoch statement at (po {po_len}, tnv \ + {tnv_len}) does not start on a field element boundary", + ); + } + + // ⚠ A sweep whose shapes all need the same pad would pass with the pad + // written as that constant. This says the sweep is not that sweep. + assert!( + residues_seen.len() > 1, + "every shape in the sweep has the same residue, so the sweep cannot \ + tell a computed pad from a constant one", + ); +} + +#[test] +fn the_cross_epoch_statement_ends_on_a_field_element_boundary() { + for &(pages, tnv_len) in &[(0usize, 15usize), (35, 50), (3, 18), (1, 16), (35, 51)] { + let page_bases: Vec = (0..pages as u64).map(|i| i * 4096).collect(); + let table_num_vars = vec![21u8; tnv_len]; + + let mut rec = WindowRecorder::::new(); + crate::multilinear_continuation::absorb_global( + &mut rec, + &[9u8; 32], + 15, + 0, + &page_bases, + &table_num_vars, + &config(), + ); + + let expected = global_expected(pages, tnv_len); + assert!( + rec.window.is_multiple_of(FELT_BYTES), + "cross-epoch statement at (pages {pages}, tnv {tnv_len}) ended at \ + byte {} = {} mod {FELT_BYTES}", + rec.window, + rec.window % FELT_BYTES, + ); + assert_eq!( + (rec.absorbs.len(), rec.window), + (expected.calls, expected.total), + "cross-epoch statement at (pages {pages}, tnv {tnv_len})", + ); + } +} + +#[test] +fn the_monolithic_statement_ends_on_a_field_element_boundary() { + for &(po_len, ranges, tnv_len) in &[ + (0usize, 0usize, 34usize), + (5, 2, 34), + (33, 1, 35), + (4, 3, 7), + ] { + let public_output = vec![0x5Au8; po_len]; + let runtime_page_ranges: Vec = (0..ranges) + .map(|i| RuntimePageRange { + base: i as u64 * 4096, + count: 1, + }) + .collect(); + let table_num_vars = vec![22u8; tnv_len]; + + let mut rec = WindowRecorder::::new(); + crate::multilinear_prove::absorb( + &mut rec, + &[4u8; 32], + &public_output, + &counts(), + 2, + &runtime_page_ranges, + &table_num_vars, + &config(), + ); + + let expected = monolithic_expected(po_len, ranges, tnv_len); + assert!( + rec.window.is_multiple_of(FELT_BYTES), + "monolithic statement at (po {po_len}, ranges {ranges}, tnv \ + {tnv_len}) ended at byte {} = {} mod {FELT_BYTES}", + rec.window, + rec.window % FELT_BYTES, + ); + assert_eq!( + (rec.absorbs.len(), rec.window), + (expected.calls, expected.total), + "monolithic statement at (po {po_len}, ranges {ranges}, tnv {tnv_len})", + ); + } +} + +/// ★ The pad is a function of the WHOLE length, not of the fixed prefix. +/// +/// This is the arithmetic that killed the constant-pad design, written as an +/// assertion so it cannot be forgotten: at the shape this system runs, padding +/// the fixed prefix to a multiple of 8 leaves the roots at 2 mod 8. +#[test] +fn padding_only_the_fixed_prefix_would_not_align_the_roots() { + // The shape this system runs: no public output, 34 tables. + let (po_len, tnv_len) = (0usize, 34usize); + assert!( + epoch_expected(po_len, tnv_len) + .total + .is_multiple_of(FELT_BYTES) + ); + + // The fixed prefix is the body with both variable fields empty. + let fixed = epoch_expected(0, 0).body; + assert_eq!(fixed, 237, "the epoch statement's fixed prefix"); + let fixed_rounded = fixed + statement_padding(fixed); + assert_eq!(fixed_rounded, 240); + + assert_eq!( + (fixed_rounded + po_len + tnv_len) % FELT_BYTES, + 2, + "a pad computed from the fixed prefix alone leaves the roots at 2 mod \ + {FELT_BYTES} at the shape we run: it would move every pinned constant \ + and align nothing", + ); +} + +// ------------------------------------------------------------------------- +// (2) A real prove, every absorb +// ------------------------------------------------------------------------- + +fn eq_table() -> ( + &'static ConcreteVmAir, + Vec>>, +) { + let ops = vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ]; + let columns: Vec>> = generate_eq_trace(&ops).columns_main(); + // Leaked so the layout borrows nothing from a temporary; this is a test + // binary and the leak is one AIR. + let air: &'static ConcreteVmAir = Box::leak(Box::new(create_eq_air( + &ProofOptions::default_test_options(), + ))); + (air, columns) +} + +fn committed( + air: &'static ConcreteVmAir, + columns: &[Vec>], + cfg: &ChainConfig, +) -> CommittedTables<'static, F, E, H> { + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + columns.len(), + columns[0].len().trailing_zeros() as usize, + Uniforms::default(), + ) + .expect("layout"); + let table = + CommittedTable::from_layout(layout, |col| columns[col as usize].clone()).expect("table"); + CommittedTables::<_, _, H>::commit(vec![table], cfg).expect("commit") +} + +/// The epoch statement this fixture proves under, absorbed into `t`, and how +/// many absorbs it took. +fn seed_with_statement(t: &mut impl IsTranscript, table_num_vars: &[u8], po: &[u8]) -> usize { + crate::multilinear_continuation::absorb_epoch( + t, + &[1u8; 32], + po, + &counts(), + 0, + table_num_vars, + &config(), + ); + epoch_expected(po.len(), table_num_vars.len()).calls +} + +/// ⚠ The trace is generated ONCE and handed to both proves. +/// +/// `generate_eq_trace` emits its rows in `HashMap` iteration order, so two +/// calls produce two row orders, two commitments and two proofs — W1's finding +/// F2, and the reason the byte gate sorts its rows canonically. Regenerating it +/// per prove would make this comparison fail for a reason that has nothing to do +/// with the recorder, which is exactly what it did on the first run of this test. +fn prove_through_recorder( + air: &'static ConcreteVmAir, + columns: &[Vec>], + po: &[u8], +) -> (Vec, WindowRecorder) +where + WindowRecorder: IsTranscript, +{ + let cfg = config(); + let num_vars = columns[0].len().trailing_zeros() as u8; + let committed = committed::(air, columns, &cfg); + + let mut rec = WindowRecorder::::new(); + let statement_absorbs = seed_with_statement(&mut rec, &[num_vars], po); + assert_eq!(rec.absorbs.len(), statement_absorbs); + + let proof = multilinear_table::multi_prove(&committed, &cfg, &mut rec).expect("prove"); + let bytes = rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec(); + (bytes, rec) +} + +fn prove_through_production( + air: &'static ConcreteVmAir, + columns: &[Vec>], + po: &[u8], +) -> Vec { + let cfg = config(); + let num_vars = columns[0].len().trailing_zeros() as u8; + let committed = committed::(air, columns, &cfg); + + let mut t = DefaultTranscript::::new(&[]); + seed_with_statement(&mut t, &[num_vars], po); + + let proof = multilinear_table::multi_prove(&committed, &cfg, &mut t).expect("prove"); + rkyv::to_bytes::(&proof) + .expect("serialize") + .to_vec() +} + +/// ★★ The end-to-end claim, on the production stream. +/// +/// Run under both configurations. The offsets are a property of the CALL +/// SEQUENCE rather than of the sponge, so the two arms are expected to agree — +/// which is exactly why running both is worth its seconds: a disagreement would +/// mean one configuration absorbs something the other does not. +#[test] +fn every_absorb_of_a_real_prove_is_field_element_aligned() { + // Two public-output lengths, so the statement ends at a different length in + // each and the pad differs: at `[]` the pad is 2, at `[0, 0]` it is 0 — the + // shape where the padding absorb is empty and must still be made. + let (air, columns) = eq_table(); + + for po in [vec![], vec![0u8, 0u8]] { + for arm in ["keccak", "rpx"] { + let (recorded, rec, produced) = match arm { + "keccak" => { + let (b, r) = prove_through_recorder::(air, &columns, &po); + ( + b, + r.absorbs.clone(), + prove_through_production::(air, &columns, &po), + ) + } + _ => { + let (b, r) = prove_through_recorder::(air, &columns, &po); + ( + b, + r.absorbs.clone(), + prove_through_production::(air, &columns, &po), + ) + } + }; + + // The recorder is production's sponge with a tape attached, and this + // is what says so. A mirrored duplex buffer that drifted would draw + // different challenges and land here. + assert_eq!( + keccak_line(&recorded), + keccak_line(&produced), + "the {arm} arm's recorded prove is not the production prove \ + (po {} bytes): the recorder's mirror of the duplex buffer has \ + drifted, so its offsets describe some other transcript", + po.len(), + ); + + let statement_absorbs = epoch_expected(po.len(), 1).calls; + let bad = misaligned(&rec, statement_absorbs); + assert!( + bad.is_empty(), + "the {arm} arm absorbed {} of {} values at an offset that is not \ + a multiple of {FELT_BYTES} (po {} bytes). First few \ + (offset, len): {:?}", + bad.len(), + rec.len() - statement_absorbs, + po.len(), + &bad[..bad.len().min(8)], + ); + + // A read that can fail in the other direction: a prove that absorbed + // nothing after the statement would satisfy the emptiness above. + // "None misaligned" is a statement only beside the count it is out of. + assert!( + rec.len() > statement_absorbs + 8, + "the {arm} arm absorbed only {} values in total, so the \ + alignment assertion above is about almost nothing", + rec.len(), + ); + println!( + "ALIGNED {arm} po={} {} absorbs after the statement, 0 misaligned", + po.len(), + rec.len() - statement_absorbs, + ); + } + } +} + +fn keccak_line(bytes: &[u8]) -> String { + crypto::hash::platform_keccak::PlatformKeccak256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} diff --git a/prover/src/tests/whir_byte_gate.rs b/prover/src/tests/whir_byte_gate.rs new file mode 100644 index 000000000..fc8691a34 --- /dev/null +++ b/prover/src/tests/whir_byte_gate.rs @@ -0,0 +1,250 @@ +//! ★ THE BYTE GATE — the WHIR identity line over a canonically sorted EQ trace. +//! +//! A printing measurement, not an assertion. It exists to be run on two +//! revisions and have its output compared, which is a thing a test harness +//! cannot do for you, so it prints and is `#[ignore]`d. +//! +//! ```text +//! cargo test --release -p lambda-vm-prover --lib \ +//! the_whir_identity_line_over_a_canonically_sorted_eq_trace \ +//! -- --ignored --nocapture +//! ``` +//! +//! # (a) The sort is MEASUREMENT-ONLY +//! +//! The rows are canonically ordered HERE, after `generate_eq_trace` returns, +//! and none of the six trace builders is touched. Row order is free to the +//! argument — the bus is a multiset — so a sorted trace is still a valid EQ +//! trace that proves and verifies; it is simply a reproducible one. +//! +//! # (b) Why it exists at all, which is a mistake worth not repeating +//! +//! The obvious version of this measurement — hash the proof of +//! `generate_eq_trace`'s output — **is not reproducible across processes**, and +//! quoting it across two revisions produces a number that looks like evidence +//! and is not. `generate_eq_trace` deduplicates through a +//! `std::collections::HashMap` and lays its rows out in iteration order +//! (`prover/src/tables/eq.rs:128`), which `RandomState` randomises per map; +//! five sibling generators do the same (`bytewise.rs:107`, `branch.rs:166`, +//! `dvrm.rs:298`, `lt.rs:168`, `mul.rs:306`). +//! +//! Four consecutive runs of the unsorted version, on ONE unchanged tree and one +//! unchanged binary, observed 2026-09-15: +//! +//! ```text +//! 9147a1b9ad34e92248608c997506b4b6c06228654fb8717eca04c09f17236bc5 +//! 7b8afea2618350600e99bb67200bb4447d962f753b6e858ee0982336436e6dd3 +//! f3c9671a29d0f8fe02aaaf9d1ea85e86e13577989489f914c5c89802cb70970e +//! d778a2de322c96e1733757cfa7668ea7180907914af82d79cf6c6a72e81ceca2 +//! ``` +//! +//! Two of those agreeing by chance across two revisions is roughly a one-in-ten +//! event, and it happened. **Run any instrument twice on one tree before +//! quoting it across two.** +//! +//! # (c) The expected value +//! +//! With the sort, three consecutive runs agree, and the line is identical at +//! PR #988's head and at every commit of this branch: +//! +//! ```text +//! 307d7c00 7b8afea2618350600e99bb67200bb4447d962f753b6e858ee0982336436e6dd3 6880 bytes +//! bcdd3dd2 7b8afea2618350600e99bb67200bb4447d962f753b6e858ee0982336436e6dd3 6880 bytes +//! 29fbb45d 7b8afea2618350600e99bb67200bb4447d962f753b6e858ee0982336436e6dd3 6880 bytes +//! ``` +//! +//! ✓ Machine-independent: three runs on a 9950X + RTX 5090 box gave the same +//! line as this laptop. +//! +//! Under `LAMBDA_VM_WHIR_HASH=rpx` the line is +//! `dcc0e8d52a80a6c9ee4ed9911d54b41e7df132d209fbf91e43018b0ac01c4985` — a +//! DIFFERENT digest at the SAME 6880 bytes, which is the whole claim of the +//! seam in one line: the hash moved, the format did not. +//! +//! ⚠ The RPX line was `5226e4cf…031adb` until W1-A2b wired the Fiat-Shamir +//! transcript to the configuration. Until then the RPX arm ran an RPX Merkle +//! backend, an RPX grind and a KECCAK sponge, and this line was the digest of +//! that mixture. Both lines are now PINNED; see [`RPX_LINE`]. +//! +//! That is the gate: **the keccak arm's bytes do not move.** A commit that +//! changes this line has changed the proof PR #988 produces, and owes an +//! explanation. +//! +//! # ⛔ AND THE TRAP THIS TEST ITSELF FELL INTO +//! +//! The first version pinned `KeccakWhir` at its prove site, so it never reached +//! a dispatch. Run under `LAMBDA_VM_WHIR_HASH=rpx` on the box it printed **the +//! keccak line and no banner** — a measurement wearing the wrong arm's label, +//! the third instance of that class on this branch and the first inside the +//! instrument meant to catch it. Two lessons, both now enforced below rather +//! than described: a bench that does not print `★ WHIR HASH:` **did not reach a +//! dispatch**, and an arm that cannot produce a different answer is not an arm. +//! +//! # What it does NOT cover +//! +//! Grinding is off, so no nonce reaches the transcript and nothing here +//! exercises the proof-of-work path — see `whir_identity_tests` for why a +//! ground proof cannot be gated this way at all. One table, one group, one +//! stacked polynomial: this is a canary for the seam, not a block-level +//! measurement. +//! +//! ⛔ **AND NO STATEMENT.** The fixture builds its own transcript from the seed +//! `b"whir-identity"` and calls `multi_prove`, which begins at the roots. So +//! nothing here absorbs an epoch, cross-epoch or monolithic statement, and a +//! change to one of those — the computed statement padding, for instance — +//! CANNOT move these lines. When such a change lands, "the gate is unmoved" is +//! the assertion it owes, not evidence that it did nothing; the instruments that +//! see it are the transcript pair pin's absorb counts and +//! `tests::statement_alignment_tests`. It also means this fixture's own first +//! window (13 seed bytes, then roots) is NOT field element aligned and must +//! never be quoted as a witness that a real proof's is. + +use digest::Digest; +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::tables::eq::{EqOperation, generate_eq_trace}; +use crate::test_utils::{ConcreteVmAir, create_eq_air}; + +/// The EQ trace's columns with the ROWS sorted into a canonical order. +/// +/// Sorted by the whole row read as canonical `u64`s, so the result does not +/// depend on the incoming order — which is the entire point. +fn canonically_sorted_columns() -> Vec>> { + let ops = vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ]; + let columns: Vec>> = generate_eq_trace(&ops).columns_main(); + let rows = columns[0].len(); + + let key = |r: usize| -> Vec { columns.iter().map(|c| *c[r].value()).collect() }; + let mut order: Vec = (0..rows).collect(); + order.sort_by_key(|r| key(*r)); + + columns + .iter() + .map(|c| order.iter().map(|r| c[*r]).collect()) + .collect() +} + +/// The keccak arm's line. It has never moved and must not. +const KECCAK_LINE: &str = "7b8afea2618350600e99bb67200bb4447d962f753b6e858ee0982336436e6dd3"; + +/// ★★ The RPX arm's line, PINNED rather than merely required to differ. +/// +/// It was `5226e4cf…031adb` from the seam landing until the transcript was +/// wired to the configuration (W1-A2b). That earlier value is worth keeping in +/// view, because of how the missing wiring was found: W1-A removed the squeeze +/// reversal for RPX, which alters the challenge stream from the first squeeze +/// onward and therefore HAD to move this line — and it did not. The line's +/// refusal to move is what carried the information. +/// +/// An `assert_ne!` against [`KECCAK_LINE`], which is what this arm had before, +/// would have passed on that run and said nothing. A pinned constant is the +/// difference between an instrument that can report a surprise and one that can +/// only report a category. +const RPX_LINE: &str = "dcc0e8d52a80a6c9ee4ed9911d54b41e7df132d209fbf91e43018b0ac01c4985"; + +/// The serialized length, which neither arm may move: 32-byte digests either +/// way and no proof struct gains a field. +const SERIALIZED_LEN: usize = 6880; + +/// Prints the identity line and the serialized length, and ASSERTS what each +/// arm owes. See the module header. +#[test] +#[ignore = "a printing measurement: run it on two revisions and compare the output"] +fn the_whir_identity_line_over_a_canonically_sorted_eq_trace() { + let config = ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::default(), + }; + + let options = ProofOptions::default_test_options(); + let air: ConcreteVmAir<_> = create_eq_air(&options); + let columns = canonically_sorted_columns(); + let num_vars = columns[0].len().trailing_zeros() as usize; + + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + columns.len(), + num_vars, + Uniforms::default(), + ) + .expect("layout"); + let table = CommittedTable::from_layout(layout, |col| columns[col as usize].clone()) + .expect("committed table"); + + // ★ Through the knob, like every production site. `MultiProof` does not + // mention the hash in its type — that is the whole point of a 32-byte + // digest either way — so the proof can leave the dispatch arm and both + // arms unify. Reaching a dispatch is also what makes the `★ WHIR HASH:` + // banner print, and its absence from a log is how this test's own trap was + // found. + let proof = crate::with_whir_hash!(|H| { + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config).expect("commit"); + let mut transcript = DefaultTranscript::< + Ext, + ::Transcript, + >::new(b"whir-identity"); + multilinear_table::multi_prove(&committed, &config, &mut transcript).expect("prove") + }); + + let bytes = rkyv::to_bytes::(&proof).expect("serialize"); + let line: String = crypto::hash::platform_keccak::PlatformKeccak256::digest(bytes.as_ref()) + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + + println!("IDENTITY-LINE {line}"); + println!("IDENTITY-LEN {}", bytes.len()); + + // ★★ The assertions that make this an arm rather than a print. + // + // The length is the strict one: it may not move under either hash, because + // a hash swap is not a proof-format change. The line is the opposite — it + // MUST move, or the rpx label is on a keccak proof, which is exactly what + // this test printed before it went through a dispatch. + assert_eq!( + bytes.len(), + SERIALIZED_LEN, + "a hash swap must not change the serialized length" + ); + match crate::whir_hash_knob::selected() { + crate::whir_hash_knob::Setting::Keccak => assert_eq!( + line, KECCAK_LINE, + "the keccak arm's bytes moved: this commit changed the proof PR #988 produces" + ), + crate::whir_hash_knob::Setting::Rpx => { + // Both, and in this order: the pin is the real assertion, and the + // inequality below is what makes a failure legible when the two + // arms collapse into one. + assert_ne!( + line, KECCAK_LINE, + "the rpx arm produced KECCAK's line — the proof never reached the RPX hash" + ); + assert_eq!( + line, RPX_LINE, + "the rpx arm's bytes moved. If that was intended, say which \ + change moved them and re-pin; if not, the configuration \ + reaching the prove is not the one this constant was taken from" + ); + } + } +} diff --git a/prover/src/tests/whir_hash_tests.rs b/prover/src/tests/whir_hash_tests.rs new file mode 100644 index 000000000..327a41068 --- /dev/null +++ b/prover/src/tests/whir_hash_tests.rs @@ -0,0 +1,279 @@ +//! ★★ The hash seam, end to end, on a real VM table — the four propositions the +//! byte gate is made of, in the one place they can all be checked in process. +//! +//! This is the miniature of the A/B the box will run, and it is deliberately +//! shaped the same way: **one process, the trace built once, proved once per +//! hash.** Two processes could not be compared, because six of this VM's trace +//! generators lay their rows out in `HashMap` iteration order +//! (`prover/src/tables/eq.rs:128` and five siblings), so the same program gives +//! different traces run to run for reasons that have nothing to do with the +//! hash. +//! +//! | | proposition | why it can fail | +//! |---|---|---| +//! | **G1** | the keccak arm is the proof PR #988 produces | a threading mistake would move a byte | +//! | **G2** | the RPX arm proves and verifies, and its proof DIFFERS | a seam that silently kept hashing keccak would pass a verify and fail this | +//! | **G3** | prove under one hash, verify under the other ⇒ REJECTED | this is what makes G2 non-vacuous: prover and verifier agree on a wrong hash too | +//! | **B3** | the two arms serialize to the SAME LENGTH, to the byte | a digest-width or field change would move it; a hash change must not | +//! +//! G3 is the load-bearing one. "Prover and verifier agree" is worth nothing on +//! its own — they would agree on a hash that returned its input. What says the +//! hash is really in the proof is that a verifier told to expect the other one +//! rejects, and that the KATs in `crypto::hash::rpx::tests` pin the digest to +//! numbers this repository did not produce. + +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, MultiProof, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::tables::eq::{EqOperation, generate_eq_trace}; +use crate::test_utils::{ConcreteVmAir, create_eq_air}; + +type Proof = MultiProof; +type Columns = Vec>>; + +/// Grinding ON: the grind is one of the three hash consumers the seam names, so +/// an arm that left it on keccak would be a half-flip and this fixture has to +/// be able to see it. Four bits, so the search costs nothing. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::uniform(4), + } +} + +fn fixture_columns() -> Columns { + let ops = vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ]; + generate_eq_trace(&ops).columns_main() +} + +fn air() -> ConcreteVmAir> { + create_eq_air(&ProofOptions::default_test_options()) +} + +fn layout(columns: &Columns) -> TableLayout<'static, Fp, Ext> { + // The AIR is rebuilt per call so the layout borrows nothing from a temporary. + let a = Box::leak(Box::new(air())); + TableLayout::::new( + a.constraint_program(), + a.constraints_meta(), + a.bus_interactions(), + columns.len(), + columns[0].len().trailing_zeros() as usize, + Uniforms::default(), + ) + .expect("layout") +} + +/// Prove the fixed trace under `H`. +fn prove(columns: &Columns) -> Proof { + let table = CommittedTable::from_layout(layout(columns), |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + let mut transcript = DefaultTranscript::::new(b"whir-hash-seam"); + multilinear_table::multi_prove(&committed, &config(), &mut transcript).expect("prove") +} + +/// Verify `proof` under `H`. Returns the verifier's verdict rather than +/// unwrapping, because half these calls are supposed to fail. +fn verify(proof: &Proof, columns: &Columns) -> Result<(), multilinear::Error> { + let table = CommittedTable::from_layout(layout(columns), |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + + let owed = multilinear_table::contribution(&proof.tables[0].bus_output) + .ok_or(multilinear::Error::BusImbalance)?; + let verifier_layout = layout(columns); + let statement = verifier_layout.statement(); + let mut transcript = DefaultTranscript::::new(b"whir-hash-seam"); + multilinear_table::multi_verify::<_, _, _, H>( + proof, + &[statement], + std::slice::from_ref(committed.groups()[0].layout()), + std::slice::from_ref(committed.groups()[0].domain()), + committed.sizes(), + &owed, + &config(), + &mut transcript, + ) +} + +fn serialized(proof: &Proof) -> Vec { + rkyv::to_bytes::(proof) + .expect("serialize") + .to_vec() +} + +/// (G2, first half) The RPX arm proves and verifies. On its own this says very +/// little — see the module header — which is why it is one line of four. +#[test] +fn the_rpx_arm_proves_and_verifies() { + let columns = fixture_columns(); + let proof = prove::(&columns); + verify::(&proof, &columns).expect("an RPX proof must verify under RPX"); +} + +/// The keccak arm still does, unchanged. +#[test] +fn the_keccak_arm_proves_and_verifies() { + let columns = fixture_columns(); + let proof = prove::(&columns); + verify::(&proof, &columns).expect("a keccak proof must verify under keccak"); +} + +/// (G2, second half) ★ The two arms produce DIFFERENT proofs. +/// +/// Checked at the root, not at the whole blob: a root is the one field whose +/// difference can only come from the Merkle hash, so this distinguishes "the +/// hash moved" from "some challenge moved". +#[test] +fn the_two_hashes_produce_different_roots() { + let columns = fixture_columns(); + let keccak = prove::(&columns); + let rpx = prove::(&columns); + + assert_eq!(keccak.roots.len(), 1, "the fixture commits one group"); + assert_ne!( + keccak.roots, rpx.roots, + "a seam that kept hashing keccak under RpxWhir would land here" + ); +} + +/// (G3) ★★ **The arm that must FAIL.** A proof made under one hash must be +/// rejected by a verifier expecting the other, in both directions. +#[test] +fn a_proof_made_under_one_hash_is_rejected_under_the_other() { + let columns = fixture_columns(); + + let keccak = prove::(&columns); + assert!( + verify::(&keccak, &columns).is_err(), + "an RPX verifier accepted a keccak proof" + ); + + let rpx = prove::(&columns); + assert!( + verify::(&rpx, &columns).is_err(), + "a keccak verifier accepted an RPX proof" + ); +} + +/// (B3) ★ The serialized length is EQUAL to the byte. +/// +/// The sharper of the two byte statements: the digest is 32 bytes under either +/// hash and no proof struct gains a field, so the length is not allowed to move +/// even though every byte inside it does. A difference here is a defect in the +/// seam, not a property of the hash. +#[test] +fn the_two_hashes_serialize_to_the_same_length() { + let columns = fixture_columns(); + let keccak = serialized(&prove::(&columns)); + let rpx = serialized(&prove::(&columns)); + + assert_eq!( + keccak.len(), + rpx.len(), + "a hash swap must not be a proof-format change" + ); + assert_ne!(keccak, rpx, "…but the bytes themselves must differ"); +} + +/// ✓ The configurations name themselves, and differently — the string the +/// banner prints and the KATs are filed under. +#[test] +fn the_two_configurations_have_distinct_names() { + assert_eq!(KeccakWhir::NAME, "keccak256"); + assert_eq!(RpxWhir::NAME, "rpx256"); + assert_ne!(KeccakWhir::NAME, RpxWhir::NAME); +} + +/// ★ The GRIND follows the configuration, not a default. +/// +/// The grind is the seam's third consumer and the easiest to leave behind, +/// because it is reached through a free function rather than through a type. A +/// nonce valid under one configuration's digest is invalid under the other's +/// with overwhelming probability, so this is a direct read of which hash the +/// proof-of-work actually ran on. +#[test] +fn the_grind_follows_the_configuration() { + use crypto::grinding::{generate_nonce_smallest, is_valid_nonce}; + use multilinear::whir_hash::GrindingDigest; + + let seed = [7u8; 32]; + let factor = 12u8; + + let k = generate_nonce_smallest::>(&seed, factor).expect("nonce"); + let r = generate_nonce_smallest::>(&seed, factor).expect("nonce"); + + assert!(is_valid_nonce::>( + &seed, k, factor + )); + assert!(is_valid_nonce::>(&seed, r, factor)); + assert_ne!( + k, r, + "the same seed must not grind to the same nonce under two different hashes" + ); + assert!( + !is_valid_nonce::>(&seed, k, factor), + "keccak's nonce must not satisfy RPX's predicate" + ); + assert!( + !is_valid_nonce::>(&seed, r, factor), + "RPX's nonce must not satisfy keccak's predicate" + ); +} + +/// ★ The two transcript TYPES draw different challenges. +/// +/// ⚠ RENAMED. This was called `the_transcript_follows_the_configuration`, which +/// is a claim about the PROVER — and this body never mentions the prover. It +/// constructs both transcript types itself and compares them, so it was true +/// for the whole period in which no WHIR call site built an RPX transcript at +/// all, and it would have stayed true if none ever did. +/// +/// What it does check is worth keeping: that the two configurations are not +/// accidentally the same sponge. The claim its old name made is now checked two +/// ways — by the compiler, via the `HasTranscriptHash` bound on `multi_prove` +/// and `multi_verify`, which makes a mismatched transcript unspellable rather +/// than merely untested; and at runtime by +/// `prover/tests/whir_transcript_configuration.rs`, which runs a real prove and +/// reads the Fiat-Shamir counters afterwards. +#[test] +fn the_two_transcript_types_draw_different_challenges() { + use crypto::fiat_shamir::is_transcript::IsTranscript; + use crypto::fiat_shamir::transcript_hash::TranscriptHash; + + type KeccakT = DefaultTranscript::Transcript>; + type RpxT = DefaultTranscript::Transcript>; + + let mut k = KeccakT::new(b"same-seed"); + let mut r = RpxT::new(b"same-seed"); + k.append_bytes(b"same-absorbed-bytes"); + r.append_bytes(b"same-absorbed-bytes"); + + assert_ne!(k.state(), r.state(), "two hashes, two sponge states"); + assert_ne!( + k.sample_field_element(), + r.sample_field_element(), + "two hashes, two challenge streams" + ); + assert_eq!(::Transcript::NAME, "keccak256"); + assert_eq!(::Transcript::NAME, "rpx256"); +} diff --git a/prover/src/tests/whir_identity_tests.rs b/prover/src/tests/whir_identity_tests.rs new file mode 100644 index 000000000..fb438ad53 --- /dev/null +++ b/prover/src/tests/whir_identity_tests.rs @@ -0,0 +1,375 @@ +//! ★ The byte gate for the WHIR hash seam, and the proof that it is a gate. +//! +//! `whir_identity::identity_line` is the instrument the coordinator diffs +//! across arms, so what it can and cannot see has to be pinned rather than +//! described. Five propositions, each constructed rather than observed: +//! +//! 1. the line is STABLE when the proof is — two independent runs of the same +//! prover on the same input give the same line; +//! 2. the line is SENSITIVE — moving one byte of one Merkle root moves it, and +//! so does moving one opened codeword value; +//! 3. the line is BLIND TO NONCES, and to nothing else — changing only a +//! grinding nonce leaves it alone, which is exactly the exclusion the +//! instrument claims; +//! 4. the SERIALIZED LENGTH is unaffected by the nonce normalisation, so a +//! length comparison between two arms measures the proof and not the +//! instrument; +//! 5. ⚠ **the normalisation does NOT make a GROUND proof reproducible** — the +//! thing this gate was first designed to do, which it cannot. +//! +//! Proposition 2 is what makes 1 and 3 worth anything: a digest that could not +//! change would satisfy 1 and 3 vacuously. +//! +//! # ⚠ The correction proposition 5 records +//! +//! The gate was pre-registered as "hash the proof with every grinding nonce +//! zeroed", on the reasoning that the nonce is the only nondeterministic field. +//! That reasoning is wrong, and the test below is what found it: the nonce is +//! **absorbed into the transcript** (`multilinear::whir_chain::grind`), so every +//! challenge drawn after the first grind depends on which valid nonce the +//! search returned. Two honest runs diverge in every root and every opening +//! from that point on, and the divergence is not in the nonce fields, so +//! zeroing them cannot remove it. +//! +//! What a byte gate needs instead is a search that returns the SAME valid nonce +//! — `crypto::grinding::generate_nonce_smallest`, reached in production by +//! setting `LAMBDA_VM_DETERMINISTIC_GRIND`. The nonce normalisation is kept +//! anyway, because it costs nothing and because it makes the line insensitive +//! to the one field that still legitimately varies between a CPU arm and a +//! device arm. +//! +//! # ⚠ A SECOND cause of irreproducibility, found the same way +//! +//! Even with no proof of work at all, two `generate_eq_trace` calls on the same +//! operations produce DIFFERENT TRACES: the generator deduplicates through a +//! `std::collections::HashMap` and lays the rows out in iteration order +//! (`prover/src/tables/eq.rs:128`), which `RandomState` randomises per map. +//! Five sibling generators do the same — BYTEWISE, BRANCH, DVRM, LT and MUL. +//! Row order is free to the argument (the bus is a multiset), so this is not a +//! soundness defect, but it means **a proof of the same program is not +//! byte-reproducible across processes for reasons that have nothing to do with +//! the hash**. +//! +//! So the fixture below builds its trace ONCE and proves it twice. What these +//! tests pin is the property the seam is responsible for — the WHIR prove path +//! is a function of its input — and not a property of the trace builders, which +//! is someone else's to fix. + +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; + +use crypto::fiat_shamir::default_transcript::DefaultTranscript; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::KeccakWhir; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, MultiProof, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +use crate::tables::eq::{EqOperation, generate_eq_trace}; +use crate::test_utils::{ConcreteVmAir, create_eq_air}; +use crate::whir_identity::{identity_line, serialized_len}; + +type Proof = MultiProof; + +/// Grinding ON, deliberately: the nonces are what the instrument normalises +/// away, so a fixture without them could not exercise propositions 3 and 5. +/// Four bits, so the search costs nothing. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::uniform(4), + } +} + +/// The same posture with no proof of work — the deterministic control. +fn config_unground() -> ChainConfig { + ChainConfig { + grind: GrindBits::default(), + ..config() + } +} + +fn eq_operations() -> Vec { + vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ] +} + +/// The trace, built ONCE — see the header: the generator's row order follows a +/// `HashMap`, so building it per call would make every comparison below a test +/// of the trace builder instead of a test of the prover. +fn fixture_columns() -> Vec>> { + generate_eq_trace(&eq_operations()).columns_main() +} + +/// One real table, proved end to end — the smallest thing that has roots, +/// openings and nonces all at once. +fn prove_once(seed: &[u8], columns: &[Vec>]) -> Proof { + prove_with(seed, columns, &config()) +} + +fn prove_once_unground(seed: &[u8], columns: &[Vec>]) -> Proof { + prove_with(seed, columns, &config_unground()) +} + +fn prove_with(seed: &[u8], columns: &[Vec>], config: &ChainConfig) -> Proof { + let options = ProofOptions::default_test_options(); + let air: ConcreteVmAir<_> = create_eq_air(&options); + let num_main = columns.len(); + let num_vars = columns[0].len().trailing_zeros() as usize; + + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + num_main, + num_vars, + Uniforms::default(), + ) + .expect("layout"); + let table = CommittedTable::from_layout(layout, |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = + CommittedTables::<_, _, KeccakWhir>::commit(vec![table], config).expect("commit"); + + let mut transcript = DefaultTranscript::::new(seed); + multilinear_table::multi_prove(&committed, config, &mut transcript).expect("prove") +} + +/// The fixture is worth using only if it actually carries what the propositions +/// are about. +fn assert_fixture_is_not_degenerate(proof: &Proof) { + assert!(!proof.roots.is_empty(), "the fixture has no Merkle root"); + let rounds = proof + .columns + .iter() + .flat_map(|stacked| &stacked.polys) + .flat_map(|chain| &chain.rounds) + .count(); + assert!(rounds > 0, "the fixture has no WHIR rounds"); + let ground: u64 = proof + .columns + .iter() + .flat_map(|stacked| &stacked.polys) + .flat_map(|chain| &chain.rounds) + .map(|r| r.nonces.folding | r.nonces.ood | r.nonces.query) + .fold(0, |a, b| a | b); + assert!( + ground != 0, + "the fixture ground no nonce, so the normalisation cannot be exercised" + ); + let opened: usize = proof + .columns + .iter() + .flat_map(|stacked| &stacked.polys) + .map(|chain| chain.opened_elements()) + .sum(); + assert!(opened > 0, "the fixture opened no codeword value"); +} + +/// (1) The same input gives the same line, twice, when the proof itself is +/// deterministic. +#[test] +fn the_identity_line_is_stable_across_runs() { + let columns = fixture_columns(); + let a = prove_once_unground(b"whir-identity", &columns); + assert!(!a.roots.is_empty(), "the control fixture has no root"); + let b = prove_once_unground(b"whir-identity", &columns); + + assert_eq!( + identity_line(&a).unwrap(), + identity_line(&b).unwrap(), + "two runs of the same prover on the same input must give one line" + ); +} + +/// (5) ⚠ **The correction, pinned so nobody re-derives the wrong gate.** +/// +/// With grinding on, two honest runs differ in far more than their nonces, and +/// the normalised line differs too. This is the proposition that killed the +/// original byte-gate design; it is here so the next reader is told by a test +/// rather than by a comment. +/// +/// It compares the two proofs' NON-nonce content directly rather than asserting +/// two digests differ: the digests differing is the consequence, the roots +/// differing is the cause, and a test that asserted only the consequence would +/// pass for the wrong reason if the instrument broke. If the search ever became +/// deterministic by default, this assertion would fail LOUDLY and name the +/// reason — which is the correct outcome, not a flake. +#[test] +fn zeroing_the_nonces_does_not_make_a_ground_proof_reproducible() { + let columns = fixture_columns(); + let a = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&a); + let b = prove_once(b"whir-identity", &columns); + + let nonces_of = |p: &Proof| { + p.columns + .iter() + .flat_map(|s| &s.polys) + .flat_map(|c| &c.rounds) + .map(|r| r.nonces) + .collect::>() + }; + if nonces_of(&a) == nonces_of(&b) { + // The two searches happened to agree — at four bits that is common. + // Nothing is being claimed about this run. + return; + } + + // The COMMITTED trace's root is drawn before any grind, so it does not + // move — naming that explicitly, because it is the thing that makes this + // failure mode easy to miss. What moves is everything the transcript + // produced after the first grind. + assert_eq!( + a.roots, b.roots, + "the trace commitment precedes the first grind and cannot depend on it" + ); + + let after_the_grind = |p: &Proof| { + p.columns + .iter() + .flat_map(|s| &s.polys) + .map(|c| { + ( + c.final_value, + c.rounds + .iter() + .map(|r| (r.next_root, r.ood_value)) + .collect::>(), + ) + }) + .collect::>() + }; + assert_ne!( + after_the_grind(&a), + after_the_grind(&b), + "the nonce is absorbed, so a different nonce must move every challenge \ + drawn after it — the successor roots and the out-of-domain values" + ); + assert_ne!( + identity_line(&a).unwrap(), + identity_line(&b).unwrap(), + "and the normalised line moves with them: the nonce fields are not where \ + the divergence lives, so zeroing them cannot remove it" + ); +} + +/// (2a) One byte of one root moves the line. +#[test] +fn the_identity_line_moves_when_a_root_moves() { + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + let before = identity_line(&proof).unwrap(); + + let mut tampered = proof.clone(); + tampered.roots[0][0] ^= 1; + + assert_ne!( + before, + identity_line(&tampered).unwrap(), + "a changed Merkle root must change the line" + ); +} + +/// (2b) One opened codeword value moves the line. +/// +/// The roots are the obvious field; the openings are the bulk of the bytes and +/// the thing a leaf-hash defect would corrupt, so they are checked separately. +#[test] +fn the_identity_line_moves_when_an_opened_value_moves() { + use multilinear::whir_chain::RoundOpenings; + + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + let before = identity_line(&proof).unwrap(); + + let mut tampered = proof.clone(); + let round = &mut tampered.columns[0].polys[0].rounds[0]; + match &mut round.openings { + RoundOpenings::Base(p) => p.current[0].values[0] += FieldElement::::one(), + RoundOpenings::Extension(p) => { + p.current[0].values[0] += FieldElement::::one(); + } + } + + assert_ne!( + before, + identity_line(&tampered).unwrap(), + "a changed opened codeword value must change the line" + ); +} + +/// (3) ★ The exclusion, stated exactly: a nonce and nothing but a nonce. +#[test] +fn the_identity_line_is_blind_to_grinding_nonces_and_to_nothing_else() { + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + let before = identity_line(&proof).unwrap(); + + // A different, arbitrary nonce in every slot of one round. + let mut renonced = proof.clone(); + { + let nonces = &mut renonced.columns[0].polys[0].rounds[0].nonces; + nonces.folding ^= 0xdead_beef; + nonces.ood ^= 0x0bad_f00d; + nonces.query ^= 0xfeed_face; + } + assert_ne!( + renonced.columns[0].polys[0].rounds[0].nonces, proof.columns[0].polys[0].rounds[0].nonces, + "the tamper must actually have changed the nonces" + ); + assert_eq!( + before, + identity_line(&renonced).unwrap(), + "the line excludes grinding nonces, by construction" + ); + + // …and the very next field along does move it, so the blindness is a + // targeted exclusion rather than a broken digest. + let mut moved = proof.clone(); + moved.columns[0].polys[0].final_value += FieldElement::::one(); + assert_ne!( + before, + identity_line(&moved).unwrap(), + "only the nonces are excluded" + ); +} + +/// (4) The nonce normalisation does not change how long the proof is, so a +/// length comparison across arms measures the proof. +#[test] +fn the_serialized_length_does_not_depend_on_the_nonces() { + let columns = fixture_columns(); + let proof = prove_once(b"whir-identity", &columns); + assert_fixture_is_not_degenerate(&proof); + + let mut renonced = proof.clone(); + for stacked in &mut renonced.columns { + for chain in &mut stacked.polys { + for round in &mut chain.rounds { + round.nonces.folding ^= u64::MAX; + round.nonces.ood ^= u64::MAX; + round.nonces.query ^= u64::MAX; + } + } + } + + assert_eq!( + serialized_len(&proof).unwrap(), + serialized_len(&renonced).unwrap(), + "a nonce is a fixed-width u64; the length cannot depend on its value" + ); +} diff --git a/prover/src/whir_hash_knob.rs b/prover/src/whir_hash_knob.rs new file mode 100644 index 000000000..2d7e21f0f --- /dev/null +++ b/prover/src/whir_hash_knob.rs @@ -0,0 +1,186 @@ +//! ★ `LAMBDA_VM_WHIR_HASH` — which hash the multilinear path commits, +//! transcripts and grinds with. +//! +//! ```text +//! LAMBDA_VM_WHIR_HASH=keccak (default, and what PR #988 produces) +//! LAMBDA_VM_WHIR_HASH=rpx the algebraic arm, for a proof headed into +//! another proof +//! ``` +//! +//! Read ONCE per process and cached, so a run cannot change hash halfway +//! through and produce a proof no single configuration describes. +//! +//! # Three decisions worth stating +//! +//! **An unknown value ABORTS, loudly.** `LAMBDA_VM_WHIR_HASH=rpx256` — a +//! plausible typo, since that is what the configuration calls itself — would +//! otherwise fall through to keccak and produce a perfectly valid proof under +//! the hash the operator was trying to move away from. A measurement taken that +//! way is worse than no measurement: it looks like the RPX arm and is not. The +//! cost of aborting is a failed run with a message naming the accepted values; +//! the cost of defaulting is a number nobody can tell is wrong. +//! +//! **The banner prints on EVERY setting, including the default.** A banner that +//! only appeared for RPX could not be distinguished from a banner that did not +//! appear because this code was never reached — which is exactly what a byte +//! gate comparing two arms needs to rule out. Its absence in a log is therefore +//! a fact about the run, not an ambiguity. +//! +//! **It is read here and nowhere else.** The seam it selects is a type +//! parameter, so every consumer gets the hash through [`with_whir_hash`] rather +//! than by asking the environment again. + +use std::sync::OnceLock; + +/// Which configuration this process proves and verifies under. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Setting { + /// Keccak-256, the default. + Keccak, + /// RPX256 (XHash12). + Rpx, +} + +impl Setting { + /// The name the configuration itself reports — what the banner prints and + /// the KATs are filed under. + pub const fn name(self) -> &'static str { + match self { + Self::Keccak => "keccak256", + Self::Rpx => "rpx256", + } + } +} + +/// The environment variable that selects it. +pub const ENV: &str = "LAMBDA_VM_WHIR_HASH"; + +/// What `ENV` accepts, and what an error message lists. +const ACCEPTED: &[(&str, Setting)] = &[ + ("keccak", Setting::Keccak), + ("keccak256", Setting::Keccak), + ("rpx", Setting::Rpx), + ("rpx256", Setting::Rpx), +]; + +/// ★ The setting for this process, read once and cached. +/// +/// Prints the banner on the first call. Aborts on an unrecognised value — see +/// the module header for why that is better than defaulting. +pub fn selected() -> Setting { + static SETTING: OnceLock = OnceLock::new(); + *SETTING.get_or_init(|| { + let setting = match std::env::var(ENV) { + Err(_) => Setting::Keccak, + Ok(raw) => parse(raw.trim()).unwrap_or_else(|| { + let accepted: Vec<&str> = ACCEPTED.iter().map(|(name, _)| *name).collect(); + // eprintln then abort rather than a panic: this is a + // configuration error at startup, and the operator needs the + // accepted values, not a backtrace through the prover. + eprintln!( + "{ENV}={raw:?} is not a hash this path knows. Accepted: {}.", + accepted.join(", ") + ); + std::process::abort() + }), + }; + // Always, including the default — see the module header. + println!("★ WHIR HASH: {}", setting.name()); + setting + }) +} + +/// The accepted spellings, case-insensitively. +fn parse(raw: &str) -> Option { + let lowered = raw.to_ascii_lowercase(); + ACCEPTED + .iter() + .find(|(name, _)| *name == lowered) + .map(|(_, setting)| *setting) +} + +/// ★ Run `$body` with `$h` bound to the configuration [`selected`] names. +/// +/// The seam is a type parameter, so the dispatch has to happen where a type can +/// be named — one `match` per call site, each arm monomorphising the body at +/// its own hash. That is also why this is a macro rather than a function: a +/// function cannot return a type. +/// +/// ⚠ Both arms are always compiled, which doubles the monomorphisations of +/// everything below the call. That is deliberate: a feature gate would make the +/// RPX arm unreachable in a default build, and then the knob would be a control +/// that does nothing on exactly the binary most people run. +#[macro_export] +macro_rules! with_whir_hash { + (|$h:ident| $body:block) => { + match $crate::whir_hash_knob::selected() { + $crate::whir_hash_knob::Setting::Keccak => { + #[allow(non_camel_case_types)] + type $h = multilinear::whir_hash::KeccakWhir; + $body + } + $crate::whir_hash_knob::Setting::Rpx => { + #[allow(non_camel_case_types)] + type $h = multilinear::whir_hash::RpxWhir; + $body + } + } + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every accepted spelling maps where it says, case-insensitively. + /// + /// Tested through [`parse`] rather than through [`selected`]: the latter + /// caches in a process-global `OnceLock` and aborts on a bad value, so it + /// can be exercised exactly once per process and never with a bad input. + /// What is testable is the decision it makes, which is this function. + #[test] + fn every_accepted_spelling_maps_to_its_configuration() { + assert_eq!(parse("keccak"), Some(Setting::Keccak)); + assert_eq!(parse("keccak256"), Some(Setting::Keccak)); + assert_eq!(parse("rpx"), Some(Setting::Rpx)); + assert_eq!(parse("rpx256"), Some(Setting::Rpx)); + assert_eq!(parse("RPX"), Some(Setting::Rpx)); + assert_eq!(parse("Keccak256"), Some(Setting::Keccak)); + } + + /// ★ And the near-misses do NOT. This is the list that would otherwise + /// default to keccak and report itself as the RPX arm. + #[test] + fn a_near_miss_is_not_silently_accepted() { + for raw in [ + "rpx-256", + "rpx_256", + "xhash12", + "algebraic", + "blake3", + "", + "kecak", + ] { + assert_eq!(parse(raw), None, "{raw:?} must not parse"); + } + } + + /// The two settings name themselves the way the configurations do, so a + /// banner and a KAT cannot disagree about which arm ran. + #[test] + fn the_names_match_the_configurations() { + use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; + assert_eq!(Setting::Keccak.name(), KeccakWhir::NAME); + assert_eq!(Setting::Rpx.name(), RpxWhir::NAME); + } + + /// ✓ The macro really does bind a different type per arm — checked through + /// the configuration's own name, so a macro that expanded both arms to + /// keccak would fail here rather than silently proving under one hash. + #[test] + fn the_macro_binds_the_configuration_the_setting_names() { + use multilinear::whir_hash::WhirHash; + let name = with_whir_hash!(|H| { H::NAME }); + assert_eq!(name, selected().name()); + } +} diff --git a/prover/src/whir_identity.rs b/prover/src/whir_identity.rs new file mode 100644 index 000000000..3231a705a --- /dev/null +++ b/prover/src/whir_identity.rs @@ -0,0 +1,108 @@ +//! ★ The IDENTITY line for a WHIR proof — the byte gate the hash seam is +//! measured against. +//! +//! # What it is for +//! +//! The control for every arm of the hash work is "with the parameter unset, the +//! proof is the one PR #988 produced". Hashing the serialized proof would be +//! the obvious way to say that, and it does not work: **grinding nonces are +//! nondeterministic.** `crypto::grinding::generate_nonce` searches with rayon's +//! `find_any` under the `parallel` feature and returns whichever valid nonce a +//! worker reached first; on the device arm the kernel returns the smallest in +//! the range it scanned. Neither is a contract — the verifier accepts any nonce +//! passing `is_valid_nonce` — so two honest runs of the same prover on the same +//! input produce different proof BYTES, and a raw digest of them reports a +//! difference that means nothing. +//! +//! So the digest below is taken over the proof with **every grinding nonce +//! zeroed**, and over nothing else that has been excluded. Everything a hash +//! swap actually moves — every Merkle root, every out-of-domain value, every +//! sumcheck coefficient, every opened codeword value and every authentication +//! path — is inside it. +//! +//! # What it therefore does NOT cover +//! +//! - **The nonces themselves.** A defect that produced a valid-but-wrong nonce +//! would not show here. It is not invisible: `check_grind` rejects an invalid +//! nonce at verify time, and that is the gate for this property. +//! - **Anything outside the `MultiProof`.** Table heights, page ranges, public +//! output and the epoch bookends live in the enclosing proof structs; the +//! caller hashes those separately if it wants them bound. +//! - **Serialized LENGTH is checked separately** by +//! [`serialized_len`], because that is the sharper of the two statements: a +//! hash swap must leave the length equal to the byte (32-byte digests either +//! way, no proof struct gains a field), while the digest is *expected* to +//! change under a different hash. + +use multilinear::whir_chain::{ChainProof, RoundNonces}; + +use crate::test_utils::{E, F}; + +/// The keccak the identity line itself is taken with. +/// +/// Deliberately fixed, and deliberately NOT the proof's own hash: this is a +/// measuring instrument, not part of the protocol. If it followed the +/// configuration then the keccak arm and the RPX arm would be hashed by +/// different functions, and "the digests differ" would no longer distinguish a +/// changed proof from a changed instrument. +type Line = crypto::hash::platform_keccak::PlatformKeccak256; + +/// A proof with every grinding nonce zeroed — the form the identity line is +/// taken over. +fn without_nonces(proof: &MultiProof) -> MultiProof { + let mut out = proof.clone(); + for stacked in &mut out.columns { + for chain in &mut stacked.polys { + zero_nonces(chain); + } + } + out +} + +fn zero_nonces(chain: &mut ChainProof) { + for round in &mut chain.rounds { + round.nonces = RoundNonces::default(); + } +} + +/// The `MultiProof` this VM's multilinear path produces. +pub type MultiProof = stark::multilinear_table::MultiProof; + +/// ★ The identity line: keccak-256 over the rkyv bytes of the proof with every +/// grinding nonce zeroed. +/// +/// Two runs of the same prover on the same input must produce the same line. +/// Two runs under different hash configurations must produce different ones — +/// see [`crate::tests::whir_identity_tests`], where both halves are asserted, +/// because a digest that could not change is not a gate. +pub fn identity_line(proof: &MultiProof) -> Result<[u8; 32], String> { + use digest::Digest; + + let normalised = without_nonces(proof); + let bytes = rkyv::to_bytes::(&normalised) + .map_err(|e| format!("the proof did not serialize: {e}"))?; + let digest = Line::digest(bytes.as_ref()); + Ok(digest.into()) +} + +/// The identity line as lowercase hex — what a box run prints and a coordinator +/// diffs. +pub fn identity_hex(proof: &MultiProof) -> Result { + Ok(identity_line(proof)? + .iter() + .map(|b| format!("{b:02x}")) + .collect()) +} + +/// The serialized length of the proof, in bytes. +/// +/// ★ Checked SEPARATELY from the digest and held to a stricter standard: the +/// digest is expected to move when the hash moves, the length is not. Every +/// [`multilinear::whir_hash::WhirHash`] has a 32-byte commitment and no proof +/// struct gains a field, so a length difference between two hash arms is a +/// defect in the seam rather than a property of the hash. +pub fn serialized_len(proof: &MultiProof) -> Result { + Ok(rkyv::to_bytes::(proof) + .map_err(|e| format!("the proof did not serialize: {e}"))? + .len()) +} diff --git a/prover/tests/rpx_grind_device.rs b/prover/tests/rpx_grind_device.rs new file mode 100644 index 000000000..0be5343bb --- /dev/null +++ b/prover/tests/rpx_grind_device.rs @@ -0,0 +1,158 @@ +//! ★★ The test that fails without the RPX device grind arm. +//! +//! Needs a GPU, like every other test in this file's family: +//! +//! ```text +//! cargo test -p lambda-vm-prover --release --features cuda --test rpx_grind_device -- --nocapture +//! ``` +//! +//! # What it is for +//! +//! `generate_nonce_maybe_gpu` dispatches the nonce search to a device kernel +//! chosen by the digest. While that dispatch read "is `D` platform keccak", an +//! RPX configuration silently took the HOST search — ~2^20 RPX permutations per +//! grind, thousands of grinds per block proof. A measured WHIR block arm came in +//! at 571 s against keccak's 39 s, ~510 s of it on this line, with a correct, +//! KAT-pinned `rpx_grind_search` sitting unused in the cubin. Nothing failed; +//! the proof was valid; only the clock said so. +//! +//! So the assertions here are about WHICH KERNEL RAN, not about whether a grind +//! succeeded — a grind succeeds either way, which is exactly why the regression +//! was invisible. +//! +//! # Two counters, not one +//! +//! A single "device grinds" counter is satisfied by the keccak kernel firing +//! under an RPX configuration, which is the precise failure the dispatch exists +//! to prevent. So the RPX arm must show `rpx > 0 && keccak == 0` and the keccak +//! arm `keccak > 0 && rpx == 0`; either counter alone would pass on a dispatch +//! that ignored its key. +#![cfg(feature = "cuda")] + +use crypto::grinding::{ + generate_nonce_maybe_gpu, gpu_grind_calls, gpu_grind_calls_rpx, is_valid_nonce, + reset_gpu_grind_calls, +}; +use multilinear::whir_hash::{GrindingDigest, KeccakWhir, RpxWhir}; + +type Rpx = GrindingDigest; +type Keccak = GrindingDigest; + +/// The production grind depth. Also comfortably above `GRIND_MIN_FACTOR`, below +/// which the dispatch keeps the search on the CPU on purpose. +const FACTOR: u8 = 20; + +fn seed_of(byte: u8) -> [u8; 32] { + [byte; 32] +} + +/// ★★ An RPX grind runs on the RPX kernel, and the nonce it returns is one the +/// host accepts. +/// +/// Remove the `Arm::Rpx256` branch from `crypto::grinding`'s dispatch and this +/// reads `rpx grinds 0` and fails — the search still finds a nonce, on the host, +/// which is the whole point. +#[test] +fn an_rpx_grind_runs_on_the_rpx_kernel() { + let seed = seed_of(0xA7); + reset_gpu_grind_calls(); + + let nonce = generate_nonce_maybe_gpu::(&seed, FACTOR).expect("a nonce exists"); + + println!( + "rpx grinds {} · keccak grinds {} · nonce {nonce}", + gpu_grind_calls_rpx(), + gpu_grind_calls() + ); + assert!( + gpu_grind_calls_rpx() > 0, + "the RPX grind did not reach the device: the dispatch has no RPX arm, \ + or the kernel returned a nonce the host rejected and it fell back" + ); + assert_eq!( + gpu_grind_calls(), + 0, + "the KECCAK kernel ran under an RPX configuration" + ); + assert!( + is_valid_nonce::(&seed, nonce, FACTOR), + "the device nonce must satisfy the host predicate" + ); +} + +/// The keccak arm still runs on keccak's kernel — so the change above moved a +/// dispatch rather than replacing one. +#[test] +fn a_keccak_grind_still_runs_on_the_keccak_kernel() { + let seed = seed_of(0x5C); + reset_gpu_grind_calls(); + + let nonce = generate_nonce_maybe_gpu::(&seed, FACTOR).expect("a nonce exists"); + + println!( + "keccak grinds {} · rpx grinds {} · nonce {nonce}", + gpu_grind_calls(), + gpu_grind_calls_rpx() + ); + assert!(gpu_grind_calls() > 0, "the keccak grind left the device"); + assert_eq!( + gpu_grind_calls_rpx(), + 0, + "the RPX kernel ran under a keccak configuration" + ); + assert!(is_valid_nonce::(&seed, nonce, FACTOR)); +} + +/// ★ The device reproduces the ORACLE table, nonce for nonce. +/// +/// The three rows of `RPX_GRIND_VECTORS` from +/// `crypto/math-cuda/tests/host_kat/rpx_kat_vectors.h` — the per-table branch's +/// host implementation, which this repository did not produce. The kernel +/// `atomicMin`s, so it returns the SMALLEST valid nonce, which is the column +/// recorded there. +/// +/// This is what makes the two counter tests more than launch counts: they say a +/// kernel ran, this says it computed the right hash. +#[test] +fn the_device_reproduces_the_oracle_grind_vectors() { + for (byte, factor, want) in [(90u8, 12u8, 1342u64), (17, 13, 300), (32, 14, 705)] { + let seed = seed_of(byte); + reset_gpu_grind_calls(); + let got = generate_nonce_maybe_gpu::(&seed, factor).expect("a nonce exists"); + assert!( + gpu_grind_calls_rpx() > 0, + "seed 0x{byte:02x}: the search did not reach the RPX kernel" + ); + assert_eq!( + got, want, + "seed 0x{byte:02x}, factor {factor}: the device must return the oracle's nonce" + ); + } +} + +/// ✓ The kill switch still works, and its effect is visible in the counter — +/// so a zero counter in a real run can be read as "no device grind" rather than +/// ambiguously. +/// +/// `LAMBDA_VM_NO_GPU_GRIND` is read once into a `OnceLock`, so this cannot be +/// toggled mid-process; it is asserted structurally instead, by requiring the +/// counter and the search to agree about whether a device was used. +#[test] +fn a_device_grind_is_exactly_what_the_counter_reports() { + let seed = seed_of(0x11); + reset_gpu_grind_calls(); + assert_eq!(gpu_grind_calls_rpx(), 0, "reset must zero the counter"); + + let n = 3; + for i in 0..n { + let mut s = seed; + s[0] = i as u8; + let nonce = generate_nonce_maybe_gpu::(&s, FACTOR).expect("a nonce exists"); + assert!(is_valid_nonce::(&s, nonce, FACTOR)); + } + assert_eq!( + gpu_grind_calls_rpx(), + n, + "the counter must be one per successful device grind, not a flag" + ); +} diff --git a/prover/tests/whir_transcript_configuration.rs b/prover/tests/whir_transcript_configuration.rs new file mode 100644 index 000000000..e361af2c3 --- /dev/null +++ b/prover/tests/whir_transcript_configuration.rs @@ -0,0 +1,194 @@ +//! ★★★ A prove under `H` squeezes `H`'s sponge, and no other. +//! +//! ```text +//! cargo test -p lambda-vm-prover --features hash-metrics \ +//! --test whir_transcript_configuration +//! ``` +//! +//! # The claim a test finally makes +//! +//! `whir_hash_tests` has carried a test called +//! `the_transcript_follows_the_configuration` since the seam landed. It +//! constructed both transcript types itself and showed they draw different +//! challenges — true, and it never mentioned the prover. It was therefore true +//! throughout the period in which **no WHIR call site built an RPX transcript +//! at all**, and would have stayed true if none ever did. It is renamed to what +//! its body asserts; this file is the claim its old name made. +//! +//! Here the transcript is the production one: a real `multi_prove` through the +//! same entry point the prover uses, with the counters read afterwards. What is +//! observed is the prover's choice, not the test's. +//! +//! # Why every assertion is two-sided +//! +//! A single "squeezes" number cannot tell "the other hash ran" from "nothing +//! ran", and under the defect this exists to catch the RPX arm squeezed +//! thousands of times — all of them keccak. So each arm asserts both that its +//! own counters moved AND that the other's are zero: +//! +//! * keccak: every squeeze and absorb is keccak's; +//! * rpx: transcript work happened AND none of it was keccak. +//! +//! Without the second half the RPX assertion passed before the wiring landed. +//! Without the first, a prover that stopped squeezing would pass it too. +//! +//! # ⚠ Its own binary +//! +//! The counters are process-global and the prover's lib-test binary runs 600+ +//! tests in parallel, most of which hash. An early version of this test lived +//! there and read `215` squeezes of which `200` were keccak — the other 15 were +//! a neighbour's. A binary of its own, and a lock within it, are both needed. + +#![cfg(feature = "hash-metrics")] + +use math::field::element::FieldElement; +use math::field::{ + extensions_goldilocks::Degree3GoldilocksExtensionField as Ext, + goldilocks::GoldilocksField as Fp, +}; + +use crypto::hash_metrics; +use lambda_vm_prover::tables::eq::{EqOperation, generate_eq_trace}; +use lambda_vm_prover::test_utils::create_eq_air; +use multilinear::whir_chain::{ChainConfig, GrindBits}; +use multilinear::whir_hash::{KeccakWhir, RpxWhir, WhirHash}; +use stark::multilinear_air::Uniforms; +use stark::multilinear_table::{self, CommittedTable, CommittedTables, TableLayout}; +use stark::proof::options::ProofOptions; +use stark::traits::AIR; + +/// Counters are global; tests here take turns. +static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn serialise() -> std::sync::MutexGuard<'static, ()> { + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Grinding ON: the grind is one of the three hash consumers the seam names, +/// so a fixture that left it out could not see a half-flip. Four bits, so the +/// search costs nothing. +fn config() -> ChainConfig { + ChainConfig { + log_blowup: 2, + log_folding: 2, + num_queries: 3, + grind: GrindBits::uniform(4), + } +} + +/// One real prove under `H`, through the production entry point. +fn prove() { + let ops = vec![ + EqOperation::new(7, 7, false), + EqOperation::new(7, 9, false), + EqOperation::new(3, 3, true), + EqOperation::new(3, 5, true), + ]; + let columns: Vec>> = generate_eq_trace(&ops).columns_main(); + + // Leaked so the layout borrows nothing from a temporary, as in the sibling + // fixture; this is a test binary that exits immediately after. + let air = Box::leak(Box::new(create_eq_air( + &ProofOptions::default_test_options(), + ))); + let layout = TableLayout::::new( + air.constraint_program(), + air.constraints_meta(), + air.bus_interactions(), + columns.len(), + columns[0].len().trailing_zeros() as usize, + Uniforms::default(), + ) + .expect("layout"); + + let table = CommittedTable::from_layout(layout, |col| columns[col as usize].clone()) + .expect("committed table"); + let committed = CommittedTables::<_, _, H>::commit(vec![table], &config()).expect("commit"); + + let mut transcript = crypto::fiat_shamir::default_transcript::DefaultTranscript::< + Ext, + H::Transcript, + >::new(b"whir-transcript-configuration"); + multilinear_table::multi_prove(&committed, &config(), &mut transcript).expect("prove"); +} + +#[test] +fn a_keccak_prove_squeezes_only_keccak() { + let _serialised = serialise(); + + hash_metrics::reset(); + prove::(); + let c = hash_metrics::snapshot(); + + assert!( + c.transcript_squeezes > 0 && c.transcript_absorbs > 0, + "the prove did no transcript work at all ({} absorbs, {} squeezes), so \ + the zeros below would mean nothing", + c.transcript_absorbs, + c.transcript_squeezes + ); + assert_eq!( + (c.transcript_absorbs_keccak, c.transcript_squeezes_keccak), + (c.transcript_absorbs, c.transcript_squeezes), + "a keccak prove squeezed something that was not keccak" + ); + assert_eq!( + (c.transcript_absorbs_rpx, c.transcript_squeezes_rpx), + (0, 0) + ); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +#[test] +fn an_rpx_prove_squeezes_only_rpx() { + let _serialised = serialise(); + + hash_metrics::reset(); + prove::(); + let c = hash_metrics::snapshot(); + + assert!( + c.transcript_squeezes > 0 && c.transcript_absorbs > 0, + "the prove did no transcript work at all, so the keccak zero below is \ + not evidence of anything" + ); + assert_eq!( + (c.transcript_absorbs_rpx, c.transcript_squeezes_rpx), + (c.transcript_absorbs, c.transcript_squeezes), + "an RPX prove did transcript work on some other sponge" + ); + assert_eq!( + (c.transcript_absorbs_keccak, c.transcript_squeezes_keccak), + (0, 0), + "the prover ran a KECCAK transcript under an RPX configuration — {} \ + absorbs and {} squeezes of it. This is the defect this file exists \ + for; it stood through four measured A/Bs.", + c.transcript_absorbs_keccak, + c.transcript_squeezes_keccak + ); + assert_eq!(c.transcript_unattributed(), (0, 0, 0)); +} + +/// ★ The two configurations do the same amount of transcript WORK. +/// +/// Only the bucket may differ. If a hash swap changed the absorb or squeeze +/// count, the per-arm bench line would report a protocol difference as a hash +/// difference, and a reader comparing arms would misattribute it. +#[test] +fn the_two_configurations_do_the_same_transcript_work() { + let _serialised = serialise(); + + hash_metrics::reset(); + prove::(); + let k = hash_metrics::snapshot(); + + hash_metrics::reset(); + prove::(); + let r = hash_metrics::snapshot(); + + assert_eq!( + (k.transcript_absorbs, k.transcript_squeezes), + (r.transcript_absorbs, r.transcript_squeezes), + "the two arms do different amounts of transcript work" + ); +}