Bump ethrex to 26.0.0 and rebuild the benchmark workload

.github/workflows/bench-abba.yml

@@ -8,28 +8,41 @@ name: Bench ABBA tiebreaker
 #
 # Syntax: "/bench-abba [N]", where N is the pair count (default 12).
 #
-# Resolution, from the paired t-test (resolvable 95% delta = t* x sd / sqrt(N)). The
-# pair-delta sd on this runner is NOT yet measured; the two columns bracket it between
-# 1.0% (the GPU box's measured 0.64% plus margin) and 2.0% (sqrt(2) x this runner's
-# measured 1.43% single-run CV):
+# Resolution, from the paired t-test (resolvable 95% delta = 1.96 x sd / sqrt(N), the
+# normal approximation; the exact t* at these pair counts is 10-20% larger, so the
+# cells below are floors). The
+# two columns are not a guess bracketing an unknown: they are the same runner under
+# two conditions, because its variance is contention and not prover jitter. Over a
+# 14-prove baseline (tooling/ethrex-fixtures/README.md) wall time and the share of CPU
+# a prove actually got correlate at -0.98, perfectly monotonic, so the single-run CV is
+# 0.34% across the proves that got the most CPU and 1.26% across all of them. sd of a
+# pair delta is sqrt(2) x that:
 #
-#   pairs   wall      resolves (sd 2.0% / sd 1.0%)
-#    8      ~50 min    1.7% / 0.8%
-#   12      ~72 min    1.3% / 0.6%   <- default
-#   20      ~1h55m     0.9% / 0.5%
-#   32      ~3h        0.7% / 0.4%
+#   pairs   wall      resolves (quiet box, sd 0.49% / shared, sd 1.78%)
+#    8      ~41 min    0.34% / 1.24%
+#   12      ~58 min    0.28% / 1.01%   <- default
+#   20      ~1h31m     0.21% / 0.78%
+#   32      ~2h22m     0.16% / 0.62%
 #
-# Wall is 2 x 158.8 s per pair (the runner's measured prove at epoch 2^22) plus ~8 min
-# of setup. The FIRST run measures the sd — it is the `sd` field of the paired-t line
-# in the result comment — and this table should be re-pinned to it then.
+# Wall is 2 x ~125 s per pair (the runner's measured prove at epoch 2^22) plus ~8 min of
+# setup. Read the column the run earned, not the one you want: the script prints an
+# exclusivity line reporting the CPU share of every prove and flagging any that came in
+# under 90% of the batch's best. A flagged batch belongs in the right-hand column.
+#
+# The workload rebuild did tighten this, but not as far as first recorded: a
+# 5-prove baseline put the CV at 0.57%, and its 95% interval was [0.34%, 1.64%] — wide
+# enough that the point estimate was not worth pinning a table to, which is what the
+# 14-prove follow-up then showed. Numbers from before the bump measured a different
+# fixture (block 25368371 as an rkyv ProgramInput) and are not comparable.
 on:
   issue_comment:
     types: [created]
 
 env:
   # The epoch /bench proves this block at, so the tiebreaker resolves the same
   # configuration and not just the same block. Memory picks it: this runner peaks at
-  # ~52 GB here against its >=64 GiB floor. See benchmark-pr.yml's REAL_BLOCK_EPOCH_LOG2.
+  # 44.81 GiB here against its >=64 GiB floor, and 2^23 would take that past 50 GiB
+  # for ~7% of wall. See benchmark-pr.yml's REAL_BLOCK_EPOCH_LOG2.
   ABBA_REAL_EPOCH_LOG2: "22"
 
 concurrency:
@@ -50,8 +63,8 @@ jobs:
       startsWith(github.event.comment.body, '/bench-abba') &&
       contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)
     runs-on: [self-hosted, bench]
-    # Hang guardrail, not expected duration: a real-block pair is 2 x 158.8 s = ~5.3 min,
-    # so the default 12 pairs runs ~72 min and the 40-pair clamp ~3.7 hr, plus up to
+    # Hang guardrail, not expected duration: a real-block pair is 2 x 125.17 s = ~4.2 min,
+    # so the default 12 pairs runs ~58 min and the 40-pair clamp ~2.9 hr, plus up to
     # ~30 min of two-sided build on a cold cache.
     timeout-minutes: 360
     steps:
@@ -117,9 +130,9 @@ jobs:
             await github.rest.issues.createComment({
               owner: context.repo.owner, repo: context.repo.repo,
               issue_number: context.issue.number,
-              // A pair is TWO proves at the runner's measured 158.8 s, so ~5.3 min/pair,
-              // plus ~8 min of checkout, two-sided build and fixture fetch.
-              body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} — a pair is 2 proves at ~158.8 s, so roughly ${Math.round(8 + Number(process.env.PAIRS) * 5.3)} min. Pass a smaller pair count for a quicker, coarser run. The bench server is occupied until it finishes.`
+              // A pair is TWO proves at the runner's measured 125.17 s, so ~4.2 min/pair,
+              // plus ~8 min of checkout, two-sided build and fixture build.
+              body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} — a pair is 2 proves at ~125 s, so roughly ${Math.round(8 + Number(process.env.PAIRS) * 4.2)} min. Pass a smaller pair count for a quicker, coarser run. The bench server is occupied until it finishes.`
             });
 
       - name: Checkout (full history for ref resolution)
@@ -153,7 +166,7 @@ jobs:
           # interleaved pairs, and prints the paired-t CI + exact Wilcoxon test.
           # Pass the head SHA (pinned above) so fork PRs resolve.
           scripts/bench_abba.sh "$HEAD_SHA" origin/main "$PAIRS" 2>&1 | tee /tmp/abba_out.txt
-          sed -n '/=== ABBA paired result/,$p' /tmp/abba_out.txt > /tmp/abba_result.txt
+          sed -n '/^==> Exclusivity:/,$p' /tmp/abba_out.txt > /tmp/abba_result.txt
 
       - name: Post result
         if: always()

.github/workflows/benchmark-gpu.yml

@@ -9,7 +9,7 @@ name: Benchmark GPU (PR)
 # Triggered by a "/bench-gpu [N]" comment on a PR (N = pair count, default 14) or via
 # workflow_dispatch.
 #
-# Workload: the real block (see tooling/ethrex-block-converter/README.md), proven with
+# Workload: the real block (see tooling/ethrex-fixtures/README.md), proven with
 # --continuations at the calibrated epoch size below.
 # Orchestration runs on a GitHub-hosted runner; all GPU work happens on the rented
 # Vast box (provisioned by the template onstart).
@@ -57,7 +57,10 @@ env:
   # Continuation epoch for the REAL-BLOCK path, from the RTX 5090 calibration on
   # 2026-07-31 against main @9ccdaf2 (raw traces:
   # ~/workspace/lambda_vm_bench_cache/gpu_epoch_calib_2026-07-31/, PROVENANCE.txt).
-  # Measured on the 32,607 MiB card, same fixture and CLI, one prove per setting:
+  # Measured on the 32,607 MiB card, one prove per setting, on the fixture of the day:
+# block 25368371 as an rkyv ProgramInput, which the ethrex 26 bump retired. The VRAM
+# and epoch figures follow the epoch size and carry over; the wall times and proof
+# sizes are that block's.
   #
   #   2^21   70.52 s wall   19,193 MiB VRAM (58.9%)   25 epochs   1.65 GB proof
   #   2^22   59.87 s wall   23,193 MiB VRAM (71.1%)   13 epochs   1.12 GB proof
@@ -70,7 +73,7 @@ env:
   # GPU PATH ONLY, and deliberately so. It is NOT pushed into bench_abba.sh's default
   # (2^20, which the CPU /bench-abba uses) nor into the CLI's
   # DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2 (also 20): 2^22 needs ~32 GiB of HOST memory on
-  # a CPU build (measured peak RSS on the 124 GiB calibration box; the CUDA path's host
+  # a CPU build for the block that was current then (the rebuilt one peaks at 44.81 GiB) (measured peak RSS on the 124 GiB calibration box; the CUDA path's host
   # peak is a different number, ~36 GB — see the cpu_ram floor below), which would break
   # laptops. VRAM is the binding constraint here and host RAM is the binding constraint
   # there, so the two defaults are not the same question.
@@ -176,7 +179,15 @@ jobs:
             // Per-prove wall varies with the rented host's CPU
             // (the prover is partly host-CPU-bound), so the slope is the measured one
             // and the intercept carries slack for a colder box.
-            const mins = 12 + Number(process.env.PAIRS) * 4;
+            // The intercept also has to cover the real-block fixture, which this box
+            // BUILDS rather than downloads -- it is generated from the block's replay
+            // cache, and a rented box is always a fresh checkout -- i.e. a cold cargo
+            // build of tooling/ethrex-fixtures, ethrex host tree and C dependencies
+            // included, at CARGO_BUILD_JOBS=8. bench_abba.sh backgrounds it so it runs
+            // under the dual cuda build rather than before it, so what reaches the ETA is
+            // only the part that outlasts that build. Neither has been measured, which is
+            // why this is 18 and not a number with a decimal point.
+            const mins = 18 + Number(process.env.PAIRS) * 4;
             const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) of ${process.env.WORKLOAD} on the CUDA prover path. Rough ETA ~${mins} min. The result will replace this comment.`;
             const comments = await github.paginate(github.rest.issues.listComments, {
               owner: context.repo.owner, repo: context.repo.repo,
@@ -229,11 +240,20 @@ jobs:
           # because vast can't numerically compare the driver_version string server-side.
           MIN_DRIVER: "580"
         run: |
-          # cpu_ram filter is in GB. Floor 48 GB: the real block at epoch 2^22 peaks at
-          # ~36 GB host RSS on the CUDA path (measured, main vintage) — ~25% headroom.
+          # cpu_ram filter is in GB. Floor 64 GB, raised from 48 with the rebuilt
+          # workload: that block peaks at 44.81 GiB (48.1 GB) at epoch 2^22 on the CPU
+          # path, which is more than a 48 GB box has -- the old floor would match boxes
+          # that cannot run the benchmark at all, never mind survive a rental with a
+          # hungrier allocator. 64 GB restores the ~25% headroom the 48 GB floor had for
+          # the old block (~36 GB host RSS on the CUDA path, measured at main vintage)
+          # and matches the CPU runner's own floor. It costs
+          # offer availability: fewer boxes qualify, so PRICE_CAP may need a look if
+          # the query starts coming back empty. Peak is set by the epoch size, not the
+          # block, so dropping GPU_REAL_EPOCH_LOG2 to 21 is the other lever (~32.8 GiB host,
+          # +14% wall).
           # Continuation peak is set by the epoch size, not the block, so bigger blocks
-          # don't move it; raising the epoch would (see the calibration tables in
-          # tooling/ethrex-block-converter/README.md).
+          # don't move it; raising the epoch would (see the epoch table in
+          # tooling/ethrex-fixtures/README.md).
           # gpu_frac=1 requires a WHOLE-MACHINE offer (you rent every GPU on the host), so
           # Vast places no other tenant on the box: CPU cores, RAM/memory bandwidth, PCIe,
           # and NVMe are fully dedicated. Without it the "most expensive" sort below lands on
@@ -246,7 +266,7 @@ jobs:
           # at the toolchain sanity gate. `reliability` is the queryable field (the
           # `reliability2` in the response schema is display-only, not filterable).
           # Over-strict just yields no offers, surfaced by the retry loop's "No offer".
-          QUERY="gpu_name=${GPU_NAME} num_gpus=1 gpu_frac=1 cpu_cores_effective>=16 cpu_cores_effective<=32 cpu_ram>=48 disk_space>=64 verified=true rentable=true reliability>=0.95 cuda_max_good>=12.8 dph_total<=${PRICE_CAP}"
+          QUERY="gpu_name=${GPU_NAME} num_gpus=1 gpu_frac=1 cpu_cores_effective>=16 cpu_cores_effective<=32 cpu_ram>=64 disk_space>=64 verified=true rentable=true reliability>=0.95 cuda_max_good>=12.8 dph_total<=${PRICE_CAP}"
           echo "Query: $QUERY (+ client-side driver_version major >= $MIN_DRIVER)"
           # Keep only offers whose driver major >= MIN_DRIVER, then most expensive first
           # (within the price cap). Within the now whole-machine pool, price just tracks
@@ -268,7 +288,7 @@ jobs:
             sleep "$OFFER_INTERVAL"
           done
           if [ -z "$OFFER_ID" ]; then
-            echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (whole-machine gpu_frac=1, 16-32 cores, >=48GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, reliability>=0.95, cuda_max_good>=12.8, <= \$${PRICE_CAP}/hr). Full query echoed above."
+            echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (whole-machine gpu_frac=1, 16-32 cores, >=64GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, reliability>=0.95, cuda_max_good>=12.8, <= \$${PRICE_CAP}/hr). Full query echoed above."
             exit 1
           fi
           echo "id=$OFFER_ID" >> "$GITHUB_OUTPUT"
@@ -495,7 +515,7 @@ jobs:
           # That surfaces as the OOM killer reaping a rustc ("signal: 9") or as an
           # allocation failure mid-compile. (Distinct from the toolchain gate's concern
           # above, which is a host that is broken before any load is applied.)
-          # 8 leaves ~6 GB/job on the >=48 GB floor; the build is a one-time per-bench
+          # 8 leaves ~8 GB/job on the >=64 GB floor; the build is a one-time per-bench
           # cost, and the job timeout above has ample room for it.
           REMOTE="set -e; cd /workspace/lambda_vm; \
             command -v python3 >/dev/null || { apt-get update -qq && apt-get install -y -qq python3; }; \
@@ -525,7 +545,7 @@ jobs:
           set -o pipefail
           $SSH "bash -lc \"$REMOTE\"" | tee "$RUNNER_TEMP/abba_out.txt"
           # Extract the result section for the PR comment (same marker bench-abba.yml uses).
-          sed -n '/=== ABBA paired result/,$p' "$RUNNER_TEMP/abba_out.txt" > "$RUNNER_TEMP/abba_result.txt"
+          sed -n '/^==> Exclusivity:/,$p' "$RUNNER_TEMP/abba_out.txt" > "$RUNNER_TEMP/abba_result.txt"
 
       - name: Write run summary
         # Always run so a failure (incl. workflow_dispatch, which has no PR comment step) is