STARK recursion with BLAKE3 on the batched CPU pipeline: the August configuration, 104.2 min / 358.2 GiB

Makefile

@@ -3,7 +3,7 @@ compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-s
 clean-recursion-elfs clean test test-asm \
 test-rust test-ethrex test-ethrex-offline test-executor test-syscalls test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \
 test-profile-recursion-block recursion-profile-block-input \
-test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-cuda-integration test-cuda-fallback \
+test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-blake3-host-kat test-blake3-second-source test-cuda-integration test-cuda-fallback \
 test-prover-cuda test-prover-comprehensive-cuda \
 bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \
 update-ethrex-fixture-checksums check-ethrex-fixture-checksums ethrex-real-block-fixture \
@@ -234,17 +234,24 @@ $(RECURSION_ARTIFACTS_DIR)/%.elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_
 # dir "recursion") or copy-paste (presets list is the single source of truth).
 # $(1) is the preset; the recipe uses $$ so `$$(call build_guest_elf,...)`
 # expands at recipe-run time (where $@ is defined).
+#
+# `absorb` rides along with every preset: it routes the verifier legs' coalesced
+# leaf hashing through the chained-absorb ecall, which the prover's BLAKE3 table
+# answers with its absorb mode. It is a feature rather than a default of the
+# guest crate so the no-absorb leg stays buildable for A/B cycle measurement —
+# drop it from RECURSION_GUEST_FEATURES to rebuild that leg.
+RECURSION_GUEST_FEATURES := absorb
 define recursion_verifier_rule
 $(RECURSION_ARTIFACTS_DIR)/recursion-$(1).elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR)
-	$$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-$(1)-bench,--features $(1))
+	$$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-$(1)-bench,--features "$(1) $$(RECURSION_GUEST_FEATURES)")
 endef
 $(foreach preset,$(RECURSION_VERIFIER_PRESETS),$(eval $(call recursion_verifier_rule,$(preset))))
 
 # Continuation variants: same crate, `continuation` feature on top of the preset
 # feature -> recursion-cont-<preset>-bench -> recursion-cont-<preset>.elf.
 define recursion_cont_verifier_rule
 $(RECURSION_ARTIFACTS_DIR)/recursion-cont-$(1).elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR)
-	$$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-cont-$(1)-bench,--features "continuation $(1)")
+	$$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-cont-$(1)-bench,--features "continuation $(1) $$(RECURSION_GUEST_FEATURES)")
 endef
 $(foreach preset,$(RECURSION_CONT_PRESETS),$(eval $(call recursion_cont_verifier_rule,$(preset))))
 
@@ -578,6 +585,79 @@ GPU_TEST_TIMEOUT := timeout -k 30 2700
 test-math-cuda:
 	$(GPU_TEST_TIMEOUT) cargo test -p math-cuda --release
 
+# Known-answer tests for the BLAKE3 device kernels, run on the HOST. No GPU, no
+# nvcc, no cargo — a couple of seconds.
+#
+# This exists because `test-math-cuda` above, which is the authority on these
+# kernels, runs only where a GPU does, and the per-PR CI runners have none (GPU
+# jobs are merge_group-only). Without this the kernels have no per-PR gate: an
+# edit to blake3.cu that broke the hash would reach the merge queue before
+# anything caught it. `crypto/math-cuda/tests/host_kat/` compiles the real kernel
+# source as host C++ through a shim and runs the official BLAKE3 vectors, the
+# canonical 6-round table, the official multi-block vectors against the
+# `Blake3Chain` construction, and every leaf kernel's byte stream through it.
+#
+# BOTH ROUND COUNTS are built and run. The 6-round arm is the one the campaign
+# ships and the one no other CI job compiles (risk R10), and the round count is a
+# compile-time knob, so a single-arm run would leave the shipping configuration
+# ungated. The two arms differ only in `-DBLAKE3_ROUNDS`, exactly as build.rs
+# drives the cubin from the `blake3-6round` feature.
+#
+# It checks arithmetic ONLY. Whether nvcc accepts the file, and everything about
+# execution rather than arithmetic — grid indexing, the Merkle tail's barriers,
+# device alignment, register pressure — stays with `test-math-cuda`. Necessary,
+# never sufficient.
+HOST_KAT_DIR := crypto/math-cuda/tests/host_kat
+HOST_KAT_CXXFLAGS := -std=c++17 -O2 -Wall -Wno-unknown-pragmas \
+    -I$(HOST_KAT_DIR) -Icrypto/math-cuda/kernels
+test-blake3-host-kat:
+	@mkdir -p target/host_kat
+	$(CXX) $(HOST_KAT_CXXFLAGS) \
+	    -o target/host_kat/blake3_host_kat $(HOST_KAT_DIR)/blake3_host_kat.cpp
+	./target/host_kat/blake3_host_kat
+	@echo
+	@echo "=== rebuilding for the 6-round arm (BLAKE3_ROUNDS=6) ==="
+	$(CXX) $(HOST_KAT_CXXFLAGS) -DBLAKE3_ROUNDS=6 \
+	    -o target/host_kat/blake3_host_kat_6r $(HOST_KAT_DIR)/blake3_host_kat.cpp
+	./target/host_kat/blake3_host_kat_6r
+
+# SECOND-SOURCE validation of the 6-round vectors the KAT above trusts.
+#
+# `test-blake3-host-kat` checks the KERNEL against the committed tables. This
+# checks the TABLES, against upstream BLAKE3's own portable C with its round loop
+# parameterised (`thoughts/blake3/reference-impl/`, a 2 KB reviewable diff in
+# PARAMETERISATION.diff). That reference is a different language and author and —
+# the part that matters — a different message-schedule CONSTRUCTION: it indexes a
+# precomputed MSG_SCHEDULE table where our Rust and CUDA compose one permutation
+# between rounds. A bug in the iterative composition is exactly what a single
+# source cannot catch, and check [C] compares the two constructions directly.
+#
+# It exists because 6-round BLAKE3 is computed by nothing else in the world
+# (assumption A6R), so the 6-round column of every table here rests on oracle
+# agreement rather than on a published vector.
+#
+# ⚠ Checks [D] and [E] were SILENTLY DEAD from P-a Stage 1 — which moved
+# CANONICAL_VECTORS out of `prover/src/lfm/blake3.rs` into `crypto` — until
+# 2026-08-15, because nothing ever ran this: it had no target. That is why it has
+# one now. A ~1 second C compile plus a few seconds of Python; no cargo, no GPU.
+test-blake3-second-source:
+	thoughts/blake3/reference-impl/build.sh
+	python3 thoughts/blake3/reference-impl/check.py
+
+# z3/SMT gate for the BLAKE3 chained-absorb mode's row-local constraints: the
+# compression equivalence under the absorb framing, the flags schedule, the mode
+# gating, the countdown/END logic, and the byte-width assumptions.
+#
+# Read `formal_verification/blake3_absorb/README.md` before extending it. The 6
+# rounds are verified ONE AT A TIME because the composed query does not close —
+# commit 89aeeb8c measured 145 minutes of `unknown` — and the negative controls
+# are what make a green board mean anything.
+#
+# z3's Python bindings are the only dependency. No cargo, no GPU, ~25 seconds.
+test-blake3-absorb-fv:
+	python3 formal_verification/blake3_absorb/test_ref.py
+	python3 formal_verification/blake3_absorb/z3_absorb_verify.py
+
 # End-to-end cuda dispatch coverage (requires NVIDIA GPU + nvcc).
 # Asserts the R1-R4 GPU dispatch counters fired on a real prove.
 # --test-threads=1: these tests reset and assert on process-global GPU call
@@ -640,6 +720,33 @@ clippy:
 	cargo clippy --workspace --all-targets -- -D warnings -A clippy::op_ref
 	cargo clippy --workspace --all-targets --no-default-features --features lambda-vm-prover/debug-checks -- -D warnings -A clippy::op_ref
 	cargo clippy --workspace --all-targets --features lambda-vm-prover/disk-spill -- -D warnings -A clippy::op_ref
+	# BLAKE3 at SEVEN rounds — the standard, externally anchored arm.
+	#
+	# This pass used to be the six-round one. Six is the default now (P-a Stage 6
+	# put the commitment aliases on BLAKE3, and every blessed constant is six), so
+	# the pass that needs writing down is the other one: without this, nothing in
+	# the matrix compiles `hash::blake3` at seven and the arm the KATs anchor
+	# against could rot unnoticed.
+	#
+	# Scoped to `crypto`, and it has to be. A workspace-level
+	# `--no-default-features` does NOT turn six off: `lambda-vm-prover` depends on
+	# `crypto` with default features, and feature unification takes the union, so
+	# the flag applies to the packages and the dependency edge puts it back. Only
+	# building `crypto` alone leaves the edge out of the graph.
+	#
+	# The lockstep between `crypto`'s knob and `math-cuda`'s (host primitive vs
+	# compiled cubin) still holds and is now expressed as both defaulting to six,
+	# which `math_cuda::blake3::device_rounds` makes assertable rather than
+	# discoverable as a wrong root.
+	cargo clippy -p crypto --all-targets --no-default-features --features std,asm -- -D warnings -A clippy::op_ref
+	# The chained-absorb guest arm's feature graph. Nothing else resolves it: it is
+	# not a default and no dependency names it, so a typo in the forwarding chain
+	# would go unnoticed. ⚠ This does NOT compile the guest absorb arm — that arm
+	# is `cfg(all(target_arch = "riscv64", feature = "blake3-absorb"))` and this is
+	# a host pass, so what compiles is its `cfg(not(...))` stub. The riscv64 arm
+	# has no CI coverage at all (RESUME-PA-STAGE4 §4.2); closing that needs a
+	# guest-target build, not a clippy feature.
+	cargo clippy --workspace --all-targets --features lambda-vm-prover/blake3-absorb -- -D warnings -A clippy::op_ref
 
 fmt:
 	cargo fmt --all
@@ -650,6 +757,33 @@ lint:
 	cargo clippy --workspace --all-targets -- -D warnings -A clippy::op_ref
 	cargo clippy --workspace --all-targets --no-default-features --features lambda-vm-prover/debug-checks -- -D warnings -A clippy::op_ref
 	cargo clippy --workspace --all-targets --features lambda-vm-prover/disk-spill -- -D warnings -A clippy::op_ref
+	# BLAKE3 at SEVEN rounds — the standard, externally anchored arm.
+	#
+	# This pass used to be the six-round one. Six is the default now (P-a Stage 6
+	# put the commitment aliases on BLAKE3, and every blessed constant is six), so
+	# the pass that needs writing down is the other one: without this, nothing in
+	# the matrix compiles `hash::blake3` at seven and the arm the KATs anchor
+	# against could rot unnoticed.
+	#
+	# Scoped to `crypto`, and it has to be. A workspace-level
+	# `--no-default-features` does NOT turn six off: `lambda-vm-prover` depends on
+	# `crypto` with default features, and feature unification takes the union, so
+	# the flag applies to the packages and the dependency edge puts it back. Only
+	# building `crypto` alone leaves the edge out of the graph.
+	#
+	# The lockstep between `crypto`'s knob and `math-cuda`'s (host primitive vs
+	# compiled cubin) still holds and is now expressed as both defaulting to six,
+	# which `math_cuda::blake3::device_rounds` makes assertable rather than
+	# discoverable as a wrong root.
+	cargo clippy -p crypto --all-targets --no-default-features --features std,asm -- -D warnings -A clippy::op_ref
+	# The chained-absorb guest arm's feature graph. Nothing else resolves it: it is
+	# not a default and no dependency names it, so a typo in the forwarding chain
+	# would go unnoticed. ⚠ This does NOT compile the guest absorb arm — that arm
+	# is `cfg(all(target_arch = "riscv64", feature = "blake3-absorb"))` and this is
+	# a host pass, so what compiles is its `cfg(not(...))` stub. The riscv64 arm
+	# has no CI coverage at all (RESUME-PA-STAGE4 §4.2); closing that needs a
+	# guest-target build, not a clippy feature.
+	cargo clippy --workspace --all-targets --features lambda-vm-prover/blake3-absorb -- -D warnings -A clippy::op_ref
 	# The cuda feature gates whole modules + cuda-only integration tests. build.rs emits empty
 	# cubin stubs when nvcc is absent, so this checks on a GPU-less host (CI lint runner, dev laptop)
 	# too — no GPU required. Catches cuda-gated breakage that the non-cuda passes above miss.